diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000000..81a9cc6398 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } +git lfs pre-push "$@" diff --git a/CMakeLists.txt b/CMakeLists.txt index 6029ed49b9..e69de29bb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,834 +0,0 @@ -################################################################################ -# -# MIT License -# -# Copyright (c) 2017 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ -cmake_minimum_required( VERSION 3.15 ) - -if (POLICY CMP0074) - cmake_policy(SET CMP0074 NEW) -endif() - -macro(set_var_to_condition var) - if(${ARGN}) - set(${var} TRUE) - else() - set(${var} FALSE) - endif() -endmacro() - -macro(set_if_bools_are_different var in1 in2) - set(${var} FALSE) - if(${in1}) - if(NOT ${in2}) - set(${var} TRUE) - endif() - else() - if(${in2}) - set(${var} TRUE) - endif() - endif() -endmacro() - -get_property(MIOPEN_GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) - -# This has to be initialized before the project() command appears -# Set the default of CMAKE_BUILD_TYPE to be release, unless user specifies with -D. -if(MIOPEN_GENERATOR_IS_MULTI_CONFIG) - if (NOT CMAKE_CONFIGURATION_TYPES) - set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel" CACHE STRING - "Available build types (configurations) on multi-config generators") - endif() -else() - if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING - "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel.") - endif() -endif() - -# Default installation path -if(NOT WIN32) - set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "") -endif() - -project ( MIOpen C CXX ) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -include(CTest) - -find_package(Threads REQUIRED) -find_package(ROCM 0.7.3 REQUIRED PATHS /opt/rocm) - -include(ROCMInstallTargets) -include(ROCMPackageConfigHelpers) -include(ROCMSetupVersion) -include(ROCMInstallSymlinks) -include(ROCMCreatePackage) -include(CheckCXXCompilerFlag) -include(ROCMHeaderWrapper) - -# Build library with Beta APIs -add_definitions("-DMIOPEN_BETA_API=1") - -set(MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK On CACHE BOOL "Enable AI-based fallback for Immediate Mode") -set(MIOPEN_ENABLE_AI_KERNEL_TUNING On CACHE BOOL "Enable AI heuristic for kernel tuning") -set(MIOPEN_ENABLE_SQLITE On CACHE BOOL "") -# Use SQLITE for compiled kernels, when turned off this will use raw files -set(MIOPEN_ENABLE_SQLITE_KERN_CACHE On CACHE BOOL "") - -# By default build shared libraries -option(BUILD_SHARED_LIBS "Create shared libraries" ON) - -if(MIOPEN_ENABLE_SQLITE) - # MIOpen now depends on SQLite as well - find_package(SQLite3 REQUIRED) -endif() -find_package(BZip2 REQUIRED) -find_package(nlohmann_json 3.9.1 REQUIRED) -if(MIOPEN_ENABLE_SQLITE_KERN_CACHE AND NOT MIOPEN_ENABLE_SQLITE) - message(FATAL_ERROR "MIOPEN_ENABLE_SQLITE_KERN_CACHE requires MIOPEN_ENABLE_SQLITE") -endif() -set(MIOPEN_LOG_FUNC_TIME_ENABLE Off CACHE BOOL "") -set(MIOPEN_ENABLE_SQLITE_BACKOFF On CACHE BOOL "") - -option( BUILD_DEV "Build for development only" OFF) -option(MIOPEN_ENABLE_FIN "Enable the fin driver for MIOpen" OFF) -option(MIOPEN_STRIP_SYMBOLS "Strip symbols in release mode" ON) - -option(MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM "Workaround: Use boost::filesystem instead of std::filesystem" OFF) -message(STATUS "MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM ${MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM}") - -# Strip symbols for release -if(MIOPEN_STRIP_SYMBOLS AND NOT WIN32 AND NOT APPLE) - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -s") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s") -endif() - -rocm_setup_version(VERSION 3.3.0) - -list( APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) -include(TargetFlags) - -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.3") - message(FATAL_ERROR "MIOpen requires at least gcc 5.3") - endif() -endif() - -############################################################ -# OPTION - MIOpen Backend -# - OpenCL -# - HIP -check_cxx_compiler_flag("--cuda-host-only -x hip" HAS_HIP) -if(HAS_HIP) - set(MIOPEN_DEFAULT_BACKEND "HIP") -else() - set(MIOPEN_DEFAULT_BACKEND "OpenCL") -endif() - -if(NOT WIN32 AND NOT MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM) - include(CheckCXXLinkerFlag) - check_cxx_linker_flag(-lstdc++fs HAS_LIB_STD_FILESYSTEM) -endif() - -list(APPEND CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/llvm ${CMAKE_INSTALL_PREFIX}/hip /opt/rocm /opt/rocm/llvm /opt/rocm/hip) - -option(ENABLE_HIP_WORKAROUNDS Off) -set(MIOPEN_INSTALL_CXX_HEADERS Off CACHE BOOL "Install MIOpen's C++ header interface") - -set_var_to_condition(MIOPEN_OFFLINE_COMPILER_PATHS_V2_DEFAULT FALSE) -option(MIOPEN_OFFLINE_COMPILER_PATHS_V2 "Use rocm-core to find offline GPU compiler" ${MIOPEN_OFFLINE_COMPILER_PATHS_V2_DEFAULT}) -message( STATUS "MIOPEN_OFFLINE_COMPILER_PATHS_V2: ${MIOPEN_OFFLINE_COMPILER_PATHS_V2}" ) - -# Embedded Build Configuration -set(MIOPEN_EMBED_DB "" CACHE STRING "Semi-colon separated list of architecture to embed on-disk DBs in the binary. Example gfx906_60;gfx900_56") -if(NOT MIOPEN_EMBED_DB STREQUAL "") - option(MIOPEN_DISABLE_SYSDB "Disable sys database access" Off) -else() - option(MIOPEN_DISABLE_SYSDB "Disable sys database access" ${MIOPEN_EMBED_BUILD}) -endif() -set(MIOPEN_BINCACHE_PATH "" CACHE STRING "URL or path containing binary cache files to embed") -option(MIOPEN_EMBED_BINCACHE "Embed Binary Cache or KDB" Off) -option(MIOPEN_EMBED_BUILD "Build with the set of embed flags." Off) -option(MIOPEN_DISABLE_USERDB "Disable user database access" ${MIOPEN_EMBED_BUILD}) - -# MIOPEN_USE_HIP_KERNELS is a Workaround for COMgr issues -if(MIOPEN_EMBED_BUILD) - set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build as a shared library" FORCE) - option(MIOPEN_USE_HIP_KERNELS "Use HIP kernels." Off) -else() - option(MIOPEN_USE_HIP_KERNELS "Use HIP kernels." On) -endif() - -if(MIOPEN_EMBED_BUILD) - if(MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) - message(FATAL_ERROR "AI-based fallback for Immediate Mode cannot be used \ - with database embedding") - endif() - if(MIOPEN_ENABLE_AI_KERNEL_TUNING) - message(FATAL_ERROR "AI Kernel tuning cannot be used with database embedding") - endif() -endif() - -set( MIOPEN_BACKEND ${MIOPEN_DEFAULT_BACKEND} CACHE STRING - "Which of MIOpens's backends to use?" ) -set_property( CACHE MIOPEN_BACKEND PROPERTY STRINGS - OpenCL HIP HIPOC HIPNOGPU) - -set_var_to_condition(MIOPEN_BUILD_DRIVER_DEFAULT (NOT MIOPEN_EMBED_BUILD) AND (NOT (MIOPEN_BACKEND STREQUAL "HIPNOGPU"))) -option(MIOPEN_BUILD_DRIVER "Build MIOpenDriver (and use it in tests)" ${MIOPEN_BUILD_DRIVER_DEFAULT}) -message(STATUS "MIOPEN_BUILD_DRIVER: ${MIOPEN_BUILD_DRIVER}" ) - -# OpenCL 1.2 -if( MIOPEN_BACKEND STREQUAL "OpenCL") - set(MIOPEN_BACKEND_OPENCL 1) - find_package( OpenCL REQUIRED ) - find_program(MIOPEN_HIP_COMPILER clang++ - PATH_SUFFIXES bin - PATHS - /opt/rocm/llvm - ${CMAKE_INSTALL_PREFIX}/llvm - ) - if(MIOPEN_HIP_COMPILER) - message(STATUS "hip compiler: ${MIOPEN_HIP_COMPILER}") - else() - message(FATAL_ERROR "hip compiler not found") - endif() - - # TODO (priority_low) Use to build HIP and ASM kernels. - if(MIOPEN_USE_COMGR) - message(FATAL_ERROR "comgr cannot be used with OpenCL backend") - endif() - - # This is to pass all necessary build flags to HIP compiler - # for device code compilation. Used within "find_package(hip...". - # See https://github.com/ROCm-Developer-Tools/HIP/pull/2035#issuecomment-616861118. - set (HIP_CXX_COMPILER ${MIOPEN_HIP_COMPILER}) -endif() - -# HIP SDK on Windows does not detect platform correctly, defaulting to "amd" -if(NOT HIP_PLATFORM AND CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(HIP_PLATFORM "amd") -endif() - -# HIP is always required -find_package(hip REQUIRED PATHS /opt/rocm) -message(STATUS "Build with HIP ${hip_VERSION} ${hip_DIR}") - -# Override HIP version in config.h, if necessary. -# The variables set by find_package() can't be overwritten, -# therefore let's use intermediate variables. -set(MIOPEN_hip_VERSION_MAJOR "${hip_VERSION_MAJOR}") -set(MIOPEN_hip_VERSION_MINOR "${hip_VERSION_MINOR}") -set(MIOPEN_hip_VERSION_PATCH "${hip_VERSION_PATCH}") -if( DEFINED MIOPEN_OVERRIDE_HIP_VERSION_MAJOR ) - set(MIOPEN_hip_VERSION_MAJOR "${MIOPEN_OVERRIDE_HIP_VERSION_MAJOR}") - message(STATUS "MIOPEN_hip_VERSION_MAJOR overriden with ${MIOPEN_OVERRIDE_HIP_VERSION_MAJOR}") -endif() -if( DEFINED MIOPEN_OVERRIDE_HIP_VERSION_MINOR ) - set(MIOPEN_hip_VERSION_MINOR "${MIOPEN_OVERRIDE_HIP_VERSION_MINOR}") - message(STATUS "MIOPEN_hip_VERSION_MINOR overriden with ${MIOPEN_OVERRIDE_HIP_VERSION_MINOR}") -endif() -if( DEFINED MIOPEN_OVERRIDE_HIP_VERSION_PATCH ) - set(MIOPEN_hip_VERSION_PATCH "${MIOPEN_OVERRIDE_HIP_VERSION_PATCH}") - message(STATUS "MIOPEN_hip_VERSION_PATCH overriden with ${MIOPEN_OVERRIDE_HIP_VERSION_PATCH}") -endif() - -# Depend on Composable Kernels -option(MIOPEN_USE_COMPOSABLEKERNEL "Enable MIOpen to use composable kernels for various operations" On) -if(MIOPEN_BACKEND_OPENCL) - set(MIOPEN_USE_COMPOSABLEKERNEL OFF) -endif() -message(STATUS "Enable Composable Kernels: ${MIOPEN_USE_COMPOSABLEKERNEL}") - -set_var_to_condition(MIOPEN_USE_COMGR_DEFAULT (NOT DEFINED MIOPEN_BACKEND_OPENCL) AND (NOT (MIOPEN_BACKEND STREQUAL "HIPNOGPU"))) -option(MIOPEN_USE_COMGR "Use comgr to build kernels instead of offline tools" ${MIOPEN_USE_COMGR_DEFAULT}) - -set(MIOPEN_hip_VERSION ${MIOPEN_hip_VERSION_MAJOR}.${MIOPEN_hip_VERSION_MINOR}.${MIOPEN_hip_VERSION_PATCH}) - -# Do not enable HIPRTC by default for older ROCm versions in order to avoid -# build time errors, because HIPRTC is a relatively new component. -set_var_to_condition(MIOPEN_USE_HIPRTC_DEFAULT MIOPEN_USE_COMGR) -option(MIOPEN_USE_HIPRTC "Use HIPRTC to build HIP kernels instead of COMGR" ${MIOPEN_USE_HIPRTC_DEFAULT}) - -set_if_bools_are_different(MIOPEN_CONFIGURATION_ERROR_COMGR_HIPRTC MIOPEN_USE_COMGR MIOPEN_USE_HIPRTC) -if(MIOPEN_CONFIGURATION_ERROR_COMGR_HIPRTC) - message(FATAL_ERROR "MIOPEN_USE_COMGR (${MIOPEN_USE_COMGR}) and MIOPEN_USE_HIPRTC (${MIOPEN_USE_HIPRTC}) should be set to the same value") -endif() - -# Do not append system include directories to HIP compiler flags when HIPRTC is used -set_var_to_condition(MIOPEN_HIP_COMPILER_USE_SYSTEM_INCLUDE_DIRECTORIES_DEFAULT - (NOT (MIOPEN_USE_HIPRTC AND (MIOPEN_hip_VERSION VERSION_GREATER_EQUAL 6.1.40091)))) -option(MIOPEN_HIP_COMPILER_USE_SYSTEM_INCLUDE_DIRECTORIES "Append include directories to compiler flags" - ${MIOPEN_HIP_COMPILER_USE_SYSTEM_INCLUDE_DIRECTORIES_DEFAULT}) - -target_flags(HIP_COMPILER_FLAGS hip::device) -# Remove cuda arch flags -string(REGEX REPLACE --cuda-gpu-arch=[a-z0-9]+ "" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") -string(REGEX REPLACE --offload-arch=[a-z0-9:+-]+ "" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") -# Skip library paths since hip will incorrectly treat it as a source file -string(APPEND HIP_COMPILER_FLAGS " ") -foreach(_unused RANGE 2) - string(REGEX REPLACE " /[^ ]+\\.(a|so) " " " HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") -endforeach() - -# WORKAROUND_SWDEV_413293 -# Assume that any HIP kernel can be launched with non-uniform block size; otherwise -# the "Failed to launch kernel: invalid argument" error may happen at run time. -# References: SWDEV-413293 and https://reviews.llvm.org/D155213 effective HIP_FLAT_VERSION 500723302 on Linux. -# This may lead to perf drops in the future therefore https://github.com/ROCm/MIOpen/issues/2708 is opened. -if(HAS_HIP) - # HIP version is unreliable on Windows and on Fedora, so we use compiler flag detection, - # if this is possible. See issue 2734 and PR 2719. - check_cxx_compiler_flag("-x hip -fno-offload-uniform-block" MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK) -else() - # CXX compiler is not HIP compiler, let's analyze HIP version. - set(MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK Off) - if(MIOPEN_hip_VERSION_FLAT GREATER_EQUAL 500723302) - set(MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK On) - endif() - message(STATUS "MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK: ${MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK}") -endif() -if(MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK) - string(APPEND HIP_COMPILER_FLAGS " -fno-offload-uniform-block ") -endif() - -if(WIN32) - string(REPLACE "\\" "/" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") -endif() - -message(STATUS "Hip compiler flags: ${HIP_COMPILER_FLAGS}") - -add_compile_definitions($<$:HIP_COMPILER_FLAGS=${HIP_COMPILER_FLAGS}>) - -# HIP -if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") - if(MIOPEN_USE_COMPOSABLEKERNEL) - find_package(composable_kernel 1.0.0 COMPONENTS device_other_operations device_gemm_operations device_conv_operations device_reduction_operations) - endif() - if( MIOPEN_BACKEND STREQUAL "HIPNOGPU") - set(MIOPEN_MODE_NOGPU 1) - endif() - set(MIOPEN_BACKEND_HIP 1) - - find_program(HIP_OC_COMPILER NAMES amdclang clang - PATH_SUFFIXES bin - PATHS - /opt/rocm - ${CMAKE_INSTALL_PREFIX} - ENV HIP_PATH - ) - if(HIP_OC_COMPILER) - message(STATUS "OpenCL compiler: ${HIP_OC_COMPILER}") - set(HIP_OC_COMPILER "${HIP_OC_COMPILER}") - else() - message(STATUS "OpenCL compiler not found") - endif() - - # Hcc's clang always defines __HCC__ even when not using hcc driver - add_definitions(-U__HCC__) - - set(MIOPEN_HIP_COMPILER ${CMAKE_CXX_COMPILER} CACHE PATH "") - - # rocblas - set(MIOPEN_USE_ROCBLAS ON CACHE BOOL "") - if(MIOPEN_USE_ROCBLAS) - find_package(rocblas REQUIRED PATHS /opt/rocm) - message(STATUS "Build with rocblas ${rocblas_VERSION} ${rocblas_DIR}") - else() - message(STATUS "Build without rocblas") - endif() - - # hipblaslt - set_var_to_condition(MIOPEN_USE_HIPBLASLT_DEFAULT NOT WIN32) - option(MIOPEN_USE_HIPBLASLT "Use hipBlasLt" ${MIOPEN_USE_HIPBLASLT_DEFAULT}) - if(MIOPEN_USE_HIPBLASLT) - find_package(hipblas REQUIRED PATHS /opt/rocm $ENV{HIP_PATH}) - message(STATUS "Build with hipbBLAS ${hipblas_VERSION} ${hipblas_DIR}") - find_package(hipblaslt REQUIRED PATHS /opt/rocm $ENV{HIP_PATH}) - message(STATUS "Build with hipbBLASLt ${hipblaslt_VERSION} ${hipblaslt_DIR}") - else() - message(STATUS "Build without hipbBLASLt") - endif() -else() - #CK is only enabled when HIP backend is selected - set(MIOPEN_USE_COMPOSABLEKERNEL Off) - if(MIOPEN_USE_HIPRTC) - message(FATAL_ERROR "HIPRTC cannot be used without HIP backend") - endif() -endif() -message( STATUS "${MIOPEN_BACKEND} backend selected." ) - -# look for and register clang-offload-bundler -if(MIOPEN_HIP_COMPILER MATCHES ".*clang\\+\\+.*") - find_program(MIOPEN_OFFLOADBUNDLER_BIN clang-offload-bundler - PATH_SUFFIXES bin - PATHS - /opt/rocm/llvm - ${CMAKE_INSTALL_PREFIX}/llvm - ) -endif() -if(MIOPEN_OFFLOADBUNDLER_BIN) - message(STATUS "clang-offload-bundler found: ${MIOPEN_OFFLOADBUNDLER_BIN}") - set(MIOPEN_OFFLOADBUNDLER_BIN "${MIOPEN_OFFLOADBUNDLER_BIN}") -else() - message(STATUS "clang-offload-bundler not found") -endif() - -set_var_to_condition(MIOPEN_USE_MLIR_DEFAULT NOT (NOT ${BUILD_SHARED_LIBS} AND ${MIOPEN_USE_COMGR})) -option(MIOPEN_USE_MLIR "Use MLIR compilation backend" ${MIOPEN_USE_MLIR_DEFAULT}) - -if(MIOPEN_USE_MLIR) - if(NOT ${BUILD_SHARED_LIBS} AND ${MIOPEN_USE_COMGR}) - message(FATAL_ERROR "Potential symbol conflict between mlir and comgr in static build") - endif() - if(WIN32) - # Windows does not support earlier ROCm versions hence no fallback to MLIRMIOpen. - find_package(rocMLIR 1.0.0 CONFIG REQUIRED) - else() - # Try to find package rocMLIR - # REQUIRED is omitted since we do not want cmake to abort if the package is not found - find_package(rocMLIR 1.0.0 CONFIG) - if(NOT rocMLIR_FOUND) - message(STATUS "Falling back to find library libMLIRMIOpen") - # Backward compatibility with ROCm 5.3 - # If the rocMLIR package is not found, try to find the library libMLIRMIOpen directly - find_library(LIBMLIRMIOPEN MLIRMIOpen REQUIRED) - if(NOT LIBMLIRMIOPEN) - message(FATAL_ERROR "library libMLIRMIOpen not found, please reinstall dependencies. \ - Refer to https://github.com/ROCm/MIOpen#installing-the-dependencies") - else() - message(STATUS "Build with library libMLIRMIOpen: " ${LIBMLIRMIOPEN}) - set(rocMLIR_VERSION 0.0.1) - endif() - endif() - endif() - message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION} ${rocMLIR_DIR}") -endif() - -# Update HIP Runtime Package Dependency -if(ENABLE_ASAN_PACKAGING) - set(DEPENDS_HIP_RUNTIME "hip-runtime-amd-asan" ) -else() - set(DEPENDS_HIP_RUNTIME "hip-runtime-amd" ) -endif() -set(MIOPEN_PACKAGE_REQS "${DEPENDS_HIP_RUNTIME}") - -# Online assembler -find_program(MIOPEN_AMDGCN_ASSEMBLER - NAMES clang - PATHS - ${MIOPEN_AMDGCN_ASSEMBLER_PATH} - /opt/rocm - /opt/rocm/llvm - ${CMAKE_INSTALL_PREFIX} - ${CMAKE_INSTALL_PREFIX}/llvm - PATH_SUFFIXES - /opencl/bin/x86_64 - /opencl/bin - /bin -) -message(STATUS "AMDGCN assembler: ${MIOPEN_AMDGCN_ASSEMBLER}") - -if(MIOPEN_USE_COMGR) - find_package(amd_comgr REQUIRED CONFIG) - message(STATUS "Build with amd_comgr ${amd_comgr_VERSION} ${amd_comgr_DIR}") - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, comgr") -endif() - -if(MIOPEN_USE_HIPRTC) - if(NOT MIOPEN_USE_COMGR) - message(FATAL_ERROR "HIPRTC can be used only together with COMGR") - endif() - find_package(hiprtc REQUIRED) - message(STATUS "Build with hiprtc ${hiprtc_VERSION} ${hiprtc_DIR}") -endif() - -option(Boost_USE_STATIC_LIBS "Use boost static libraries" ON) -set(BOOST_COMPONENTS filesystem) -if(MIOPEN_BUILD_DRIVER) - # boost core is a header only component that can't be found by find_package - # list(APPEND BOOST_COMPONENTS core) -endif() - -# The FindBoost module has been removed since CMake 3.30. Use the Boost 1.83 configuration file instead. -add_definitions(-DBOOST_ALL_NO_LIB=1) -find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS} CONFIG) - -find_path(HALF_INCLUDE_DIR half/half.hpp) -message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") - -option( MIOPEN_DEBUG_FIND_DB_CACHING "Use system find-db caching" ON) - -# FOR HANDLING ENABLE/DISABLE OPTIONAL BACKWARD COMPATIBILITY for FILE/FOLDER REORG -option(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY "Build with file/folder reorg with backward compatibility enabled" OFF) - -if(WIN32) - set( DATA_INSTALL_DIR bin ) - set( DATABASE_INSTALL_DIR ${DATA_INSTALL_DIR}) -else() - set( MIOPEN_INSTALL_DIR miopen) - set( DATA_INSTALL_DIR ${CMAKE_INSTALL_DATADIR}/${MIOPEN_INSTALL_DIR} ) - set( DATABASE_INSTALL_DIR ${DATA_INSTALL_DIR}/db ) -endif() - -if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) - find_package(frugally-deep CONFIG REQUIRED) - message(STATUS "Build with frugally-deep ${frugally-deep_VERSION} ${frugally-deep_DIR}") - find_package(Eigen3 REQUIRED) - message(STATUS "Build with Eigen3 ${Eigen3_VERSION} ${Eigen3_DIR}") -endif() - -if(WIN32) - set(KERNELS_BINARY_DIR ${PROJECT_BINARY_DIR}/bin) -else() - set(KERNELS_BINARY_DIR ${PROJECT_BINARY_DIR}/${DATABASE_INSTALL_DIR}) -endif() - -set(MIOPEN_GPU_SYNC Off CACHE BOOL "") -if(BUILD_DEV) - set(MIOPEN_BUILD_DEV 1) - set(MIOPEN_SYSTEM_DB_PATH "${KERNELS_BINARY_DIR}" CACHE PATH "Default path of system db files") - set(MIOPEN_USER_DB_PATH "${KERNELS_BINARY_DIR}" CACHE PATH "Default path of user db files") - set(MIOPEN_USER_DB_SUFFIX "${MIOPEN_BACKEND}.${MIOpen_VERSION_MAJOR}_${MIOpen_VERSION_MINOR}_${MIOpen_VERSION_PATCH}" CACHE PATH "Filename suffix for the user find-db files") - set(MIOPEN_CACHE_DIR "" CACHE STRING "") -else() - set(MIOPEN_BUILD_DEV 0) - if(WIN32) - set(MIOPEN_USER_DB_PATH "$USERPROFILE\\\\.miopen\\\\db\\\\" CACHE STRING "Default path to user db files") - set(MIOPEN_CACHE_DIR "$USERPROFILE\\\\.miopen\\\\cache\\\\" CACHE STRING "") - else() - set(MIOPEN_USER_DB_PATH "~/.config/miopen/" CACHE STRING "Default path of user db files") - set(MIOPEN_CACHE_DIR "~/.cache/miopen/" CACHE STRING "") - endif() - set(MIOPEN_USER_DB_SUFFIX "${MIOPEN_BACKEND}.${MIOpen_VERSION_MAJOR}_${MIOpen_VERSION_MINOR}_${MIOpen_VERSION_PATCH}_${MIOpen_VERSION_TWEAK}" CACHE PATH "Filename suffix for the user find-db files") -endif() -set(MIOPEN_SYSTEM_FIND_DB_SUFFIX "${MIOPEN_BACKEND}" CACHE PATH "Filename suffix for the system find-db files") - -# PR-2391 Add the ability to log function calls to roctx. -# This allows attached profilers to see which MIOpen calls are being called by application and which kernels are being invoked by MIOpen. -# Enabled via the MIOPEN_ENABLE_LOGGING_ROCTX env var. -set(MIOPEN_USE_ROCTRACER ON CACHE BOOL "") -if(NOT WIN32 AND MIOPEN_USE_ROCTRACER) - find_library(rocTracer roctx64) - if(rocTracer) - MESSAGE(STATUS "Build with rocTracer: " ${rocTracer}) - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, roctracer") - else() - message(WARNING "rocTracer cannot be found! Build without rocTracer") - set(MIOPEN_USE_ROCTRACER OFF) - endif() -else() - message(STATUS "Build without rocTracer") - set(MIOPEN_USE_ROCTRACER OFF) -endif() - -if(MIOPEN_USE_ROCBLAS) - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, rocblas") -endif() - -if(MIOPEN_USE_HIPBLASLT) - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, hipblaslt") -endif() - -if(MIOPEN_OFFLINE_COMPILER_PATHS_V2) - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, rocm-core") -endif() - -if(MIOPEN_BUILD_DRIVER) - # PR #2785 MIOpenDriver to use rocrand to init buffers - find_package(rocrand REQUIRED) - message(STATUS "Build with rocrand ${rocrand_VERSION} ${rocrand_DIR}") - set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, rocrand") -endif() - -if(MIOPEN_BACKEND STREQUAL "HIP") - set(CPACK_DEBIAN_PACKAGE_DEPENDS "${MIOPEN_PACKAGE_REQS}") - set(CPACK_RPM_PACKAGE_REQUIRES "${MIOPEN_PACKAGE_REQS}") - - # Make backends explicitly conflict - set(CPACK_DEBIAN_PACKAGE_CONFLICTS miopen-opencl) - set(CPACK_RPM_PACKAGE_CONFLICTS miopen-opencl) - -elseif(MIOPEN_BACKEND STREQUAL "OpenCL") - set(CPACK_DEBIAN_PACKAGE_DEPENDS "${MIOPEN_PACKAGE_REQS}, rocm-opencl-dev") - set(CPACK_RPM_PACKAGE_REQUIRES "${MIOPEN_PACKAGE_REQS}, rocm-opencl-devel") - - # Make backends explicitly conflict - set(CPACK_DEBIAN_PACKAGE_CONFLICTS miopen-hip) - set(CPACK_RPM_PACKAGE_CONFLICTS miopen-hip) -endif() - -set(KERNELS_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/kernels) - -find_program(UNZIPPER bzip2 REQUIRED) -file(MAKE_DIRECTORY ${KERNELS_BINARY_DIR}) - -add_custom_target(generate_kernels ALL) - -set(MIOPEN_USE_SQLITE_PERFDB Off CACHE BOOL "Use sqlite perfdb instead of text-based.") -if(MIOPEN_USE_SQLITE_PERFDB) - set(PERFDB_SUFFIX "") -else() - set(PERFDB_SUFFIX ".txt") -endif() - -function(unpack_db db_bzip2_file) - get_filename_component(__fname ${db_bzip2_file} NAME_WLE) - add_custom_command(OUTPUT ${KERNELS_BINARY_DIR}/${__fname} - COMMAND ${UNZIPPER} -dc -k ${db_bzip2_file} > ${KERNELS_BINARY_DIR}/${__fname}) - string(REPLACE "." "_" __tname ${__fname}) - add_custom_target(generate_${__tname} ALL DEPENDS ${KERNELS_BINARY_DIR}/${__fname}) - - get_filename_component(__extension ${__fname} LAST_EXT) - - if(NOT MIOPEN_USE_SQLITE_PERFDB AND __extension STREQUAL ".db") - add_custom_command(OUTPUT ${KERNELS_BINARY_DIR}/${__fname}.txt - DEPENDS sqlite2txt generate_${__tname} - COMMAND $ ${KERNELS_BINARY_DIR}/${__fname} ${KERNELS_BINARY_DIR}/${__fname}.txt - ) - add_custom_target(generate_${__tname}_txt ALL DEPENDS ${KERNELS_BINARY_DIR}/${__fname}.txt) - add_dependencies(generate_kernels generate_${__tname}_txt) - set(__fname ${__fname}.txt) - else() - add_dependencies(generate_kernels generate_${__tname}) - endif() - set(__fname ${__fname} PARENT_SCOPE) -endfunction() - -file(GLOB PERF_DB_BZIP_FILES CONFIGURE_DEPENDS "${KERNELS_SOURCE_DIR}/*.db.bz2") -file(GLOB FIND_DB_BZIP_FILES CONFIGURE_DEPENDS "${KERNELS_SOURCE_DIR}/*.fdb.txt.bz2") - -foreach(DB_BZIP_FILE ${PERF_DB_BZIP_FILES} ${FIND_DB_BZIP_FILES}) - unpack_db(${DB_BZIP_FILE}) - if(MIOPEN_EMBED_DB STREQUAL "" AND NOT MIOPEN_DISABLE_SYSDB AND NOT ENABLE_ASAN_PACKAGING) - install(FILES ${KERNELS_BINARY_DIR}/${__fname} - DESTINATION ${DATABASE_INSTALL_DIR}) - endif() -endforeach() - -# Begin KDB package creation - -function(install_kdb FILE_NAME COMPONENT_NAME) - file(READ ${FILE_NAME} __contents LIMIT 7) - get_filename_component(__fname ${FILE_NAME} NAME_WLE) - if(__contents MATCHES "version") - list(APPEND LFS_MISSING_FILES ${__fname}) - else() - unpack_db(${FILE_NAME}) - if( NOT ENABLE_ASAN_PACKAGING ) - if( NOT MIOPEN_TEST_DBSYNC ) - set(__component_name COMPONENT ${COMPONENT_NAME}) - endif() - rocm_install(FILES ${KERNELS_BINARY_DIR}/${__fname} - DESTINATION ${DATABASE_INSTALL_DIR} - ${__component_name}) - endif() - endif() - set(LFS_MISSING_FILES ${LFS_MISSING_FILES} PARENT_SCOPE) -endfunction() - -# Both the lists below should be in sync always -set(KDB_BZ2_FILES gfx90a.kdb.bz2 gfx1030.kdb.bz2 gfx908.kdb.bz2 gfx906.kdb.bz2 gfx900.kdb.bz2) -set(COMPONENT_LST gfx90akdb gfx1030kdb gfx908kdb gfx906kdb gfx900kdb) - -if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) - foreach(__file __component IN ZIP_LISTS KDB_BZ2_FILES COMPONENT_LST) - install_kdb(${KERNELS_SOURCE_DIR}/${__file} ${__component}) - endforeach() -else() - # TODO: Upgrade minimum CMake to version 3.17+ and use IN ZIP_LISTS instead - list(LENGTH KDB_BZ2_FILES __length) - math(EXPR __high "${__length} - 1") - foreach(__index RANGE ${__high}) - list(GET COMPONENT_LST ${__index} __component) - list(GET KDB_BZ2_FILES ${__index} __file) - install_kdb(${KERNELS_SOURCE_DIR}/${__file} ${__component}) - endforeach() -endif() - -if(LFS_MISSING_FILES) - string(REPLACE ";" ", " __lfs_missing_files "${LFS_MISSING_FILES}") - message(WARNING "GIT LFS files not pulled down, skipped: ${__lfs_missing_files}") - set(MIOPEN_NO_LFS_PULLED TRUE CACHE INTERNAL "") -else() - set(CPACK_COMPONENTS_ALL ${COMPONENT_LST}) -endif() - -# End KDB package creation - -if(NOT MIOPEN_TEST_DISCRETE) -list(APPEND CPACK_COMPONENTS_ALL client) -endif() - -rocm_create_package( - NAME MIOpen-${MIOPEN_BACKEND} - DESCRIPTION "AMD DNN Library" - MAINTAINER "MIOpen Maintainer " - LDCONFIG - # DEPENDS rocm-opencl hip-rocclr tinygemm -) - -include(EnableCompilerWarnings) -set(MIOPEN_TIDY_ERRORS ERRORS * -readability-inconsistent-declaration-parameter-name) -if(CMAKE_CXX_COMPILER MATCHES ".*clang\\+\\+") - set(MIOPEN_TIDY_CHECKS -modernize-use-override -readability-non-const-parameter) -# Enable tidy on hip -elseif(MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") - set(MIOPEN_TIDY_ERRORS ALL) -endif() - -include(ClangTidy) -enable_clang_tidy( - CHECKS - ${MIOPEN_TIDY_CHECKS} - ${MIOPEN_TIDY_ERRORS} - HEADER_FILTER - "\.hpp$" - EXTRA_ARGS - -DMIOPEN_USE_CLANG_TIDY -) - -include(CppCheck) -enable_cppcheck( - CHECKS - warning - style - performance - portability - SUPPRESS - ConfigurationNotChecked - constStatement - # There is no ODR violation because of using separate executables, - # but cppcheck doesn't understand that as it assumes everything - # will be compiled together in one binary. - ctuOneDefinitionRuleViolation:*test/* - ctuOneDefinitionRuleViolation:*src/composable_kernel/composable_kernel/*/* - ctuOneDefinitionRuleViolation:*src/composable_kernel/host/*/* - # There are many FPs with this, let's disable this (ditto in MIGraphX) - ctuPointerArith:*test/* - duplicateCondition - noExplicitConstructor - passedByValue - # preprocessorErrorDirective - shadowVariable - unusedFunction - unusedPrivateFunction - unusedStructMember - # Ignore initializer lists in the tests - useInitializationList:*test/*.cpp - *:*src/sqlite/*.cpp - *:*.cl - *:*src/kernels/*.h - knownConditionTrueFalse:*src/kernels/static_composable_kernel/*/* - redundantAssignment:*src/kernels/static_composable_kernel/*/* - unreadVariable:*src/kernels/static_composable_kernel/*/* - unusedScopedObject:*src/kernels/static_composable_kernel/*/* - wrongPrintfScanfArgNum:*src/kernels/static_composable_kernel/*/* - knownConditionTrueFalse:*src/composable_kernel/composable_kernel/*/* - identicalConditionAfterEarlyExit:*src/composable_kernel/composable_kernel/*/* - duplicateExpression:*src/composable_kernel/composable_kernel/*/* - multiCondition:*src/composable_kernel/composable_kernel/*/* - unreadVariable:*src/composable_kernel/composable_kernel/*/* - unreadVariable:*src/composable_kernel/host/*/* - unreadVariable:*src/composable_kernel/external/*/* - unmatchedSuppression - ################################################################### - # TODO Code Quality WORKAROUND ROCm 5.3 && - # Ubuntu 22.04 && C++17 && cppcheck 2.9 update - ################################################################### - constParameter - constVariable - variableScope - missingReturn - cstyleCast - uselessCallsSubstr - uninitMemberVar - overlappingWriteUnion - operatorEqVarError - returnTempReference - objectIndex - integerOverflowCond - rethrowNoCurrentException - mismatchingContainers - unreadVariable - CastIntegerToAddressAtReturn - knownConditionTrueFalse - shadowFunction - moduloofone - ################################################################### - # TODO Code Quality WORKAROUND ROCm 6.0 && - # Ubuntu 22.04 && cppcheck 2.12.1 update - ################################################################### - duplInheritedMember - constParameterCallback - constParameterReference - constParameterPointer - constVariableReference - constVariablePointer - useStlAlgorithm - uselessOverride - unusedScopedObject - FORCE - SOURCES - addkernels/ - tools/sqlite2txt/ - # driver/ - include/ - src/ - test/ - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_BINARY_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/src/include - DEFINE - CPPCHECK=1 - __linux__=1 -) - - -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) - -if(NOT MIOPEN_USE_SQLITE_PERFDB) - add_subdirectory(tools/sqlite2txt) -endif() -add_subdirectory(addkernels) -add_subdirectory(src) -if(MIOPEN_BUILD_DRIVER) - add_subdirectory(driver) -endif() - -if(BUILD_TESTING) - add_subdirectory(test) - add_subdirectory(speedtests) -endif() - -add_subdirectory(utils) -if(MIOPEN_ENABLE_FIN) - add_subdirectory(fin) -endif() diff --git a/Jenkinsfile b/Jenkinsfile index ac3cfd1a0c..e69de29bb2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,1853 +0,0 @@ -def rocmnode(name) { - return '(rocmtest || miopen) && (' + name + ')' -} - -def miopenCheckout() -{ - checkout([ - $class: 'GitSCM', - branches: scm.branches, - doGenerateSubmoduleConfigurations: true, - extensions: scm.extensions + [[$class: 'SubmoduleOption', parentCredentials: true]], - userRemoteConfigs: scm.userRemoteConfigs - ]) -} - -def show_node_info() { - sh """ - echo "NODE_NAME = \$NODE_NAME" - lsb_release -sd - uname -r - cat /sys/module/amdgpu/version - ls /opt/ -la - """ -} - -//default -// CXX=/opt/rocm/llvm/bin/clang++ CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=Off -DCMAKE_PREFIX_PATH=/usr/local -DBUILD_DEV=On -DCMAKE_BUILD_TYPE=release .. -// -def cmake_build(Map conf=[:]){ - - def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") - def make_targets = conf.get("make_targets","check") - def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") - def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") - def prefixpath = conf.get("prefixpath","/opt/rocm") - def build_type_debug = (conf.get("build_type",'release') == 'debug') - def code_conv_enabled = conf.get("codecov", false) - - def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") - // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. - if (build_type_debug || code_conv_enabled) - { - mlir_args = " -DMIOPEN_USE_MLIR=OFF" - } - - def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","") - def build_fin = conf.get("build_fin", "OFF") - - setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} " - - //cmake_env can overwrite default CXX variables. - def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","") - - def package_build = (conf.get("package_build","") == "true") - - if (package_build == true) { - make_targets = "miopen_gtest package miopen_gtest_check" - setup_args = " -DMIOPEN_TEST_DISCRETE=OFF " + setup_args - } - - def miopen_install_path = "${env.WORKSPACE}/install" - if(conf.get("build_install","") == "true") - { - make_targets = 'install ' + make_targets - setup_args = " -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=${miopen_install_path}" + setup_args - } else if(package_build == true) { - setup_args = ' -DBUILD_DEV=Off' + setup_args - } else { - setup_args = ' -DBUILD_DEV=On' + setup_args - } - - // test_flags = ctest -> MIopen flags - def test_flags = conf.get("test_flags","") - - if (conf.get("vcache_enable","") == "true"){ - def vcache = conf.get(vcache_path,"/var/jenkins/.cache/miopen/vcache") - build_envs = " MIOPEN_VERIFY_CACHE_PATH='${vcache}' " + build_envs - } else{ - test_flags = " --disable-verification-cache " + test_flags - } - - if(code_conv_enabled){ //Need - setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args - }else if(build_type_debug){ - setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args - }else{ - setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args - } - - if(test_flags != ""){ - setup_args = "-DMIOPEN_TEST_FLAGS='${test_flags}'" + setup_args - } - - if(conf.containsKey("find_mode")) - { - def fmode = conf.get("find_mode", "") - setup_args = " -DMIOPEN_DEFAULT_FIND_MODE=${fmode} " + setup_args - } - if(env.CCACHE_HOST) - { - setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args - } - - if ( build_fin == "ON" ) - { - setup_args = " -DMIOPEN_INSTALL_CXX_HEADERS=On " + setup_args - } - - def pre_setup_cmd = """ - echo \$HSA_ENABLE_SDMA - ulimit -c unlimited - rm -rf build - mkdir build - rm -rf install - mkdir install - rm -f src/kernels/*.ufdb.txt - rm -f src/kernels/miopen*.udb - cd build - """ - def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args} .. ") - // WORKAROUND_SWDEV_290754 - // It seems like this W/A is not required since 4.5. - def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${make_targets}") - def execute_cmd = conf.get("execute_cmd", "") - - def cmd = conf.get("cmd", """ - ${pre_setup_cmd} - ${setup_cmd} - ${build_cmd} - """) - - if ( build_fin == "ON" ) - { - def fin_build_cmd = cmake_fin_build_cmd(miopen_install_path) - cmd += """ - export RETDIR=\$PWD - cd ${env.WORKSPACE}/fin - ${fin_build_cmd} - cd \$RETDIR - """ - } - - cmd += """ - ${execute_cmd} - """ - - echo cmd - sh cmd - - // Only archive from master or develop - if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master" || - env.BRANCH_NAME == env.MIOPEN_GOLDEN_PERF_BRANCH || params.PERF_TEST_BRANCH_OVERRIDE)) { - archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true - archiveArtifacts artifacts: "build/*.rpm", allowEmptyArchive: true, fingerprint: true - stash includes: "build/*tar.gz", name: 'miopen_tar' - } -} - -def cmake_fin_build_cmd(prefixpath){ - def flags = "-DCMAKE_INSTALL_PREFIX=${prefixpath} -DCMAKE_BUILD_TYPE=release" - def compiler = 'clang++' - def make_targets = "install" - def compilerpath = "/opt/rocm/llvm/bin/" + compiler - def configargs = "" - if (prefixpath != "") - { - configargs = "-DCMAKE_PREFIX_PATH=${prefixpath}" - } - - def fin_cmd = """ - echo \$HSA_ENABLE_SDMA - ulimit -c unlimited - rm -rf build - mkdir build - cd build - CXX=${compilerpath} cmake ${configargs} ${flags} .. - dumb-init make -j\$(nproc) ${make_targets} - """ - return fin_cmd -} - -def getDockerImageName(dockerArgs) -{ - sh "echo ${dockerArgs} > factors.txt" - def image = "${env.MIOPEN_DOCKER_IMAGE_URL}" - sh "md5sum Dockerfile requirements.txt dev-requirements.txt >> factors.txt" - def docker_hash = sh(script: "md5sum factors.txt | awk '{print \$1}' | head -c 6", returnStdout: true) - sh "rm factors.txt" - echo "Docker tag hash: ${docker_hash}" - image = "${image}:ci_${docker_hash}" - if(params.DOCKER_IMAGE_OVERRIDE != '') - { - echo "Overriding the base docker image with ${params.DOCKER_IMAGE_OVERRIDE}" - image = "${params.DOCKER_IMAGE_OVERRIDE}" - } - return image - -} - -def getDockerImage(Map conf=[:]) -{ - env.DOCKER_BUILDKIT=1 - def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm - def gpu_arch = "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages - def mlir_build = conf.get("mlir_build", "ON") // always ON - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_TARGETS='${gpu_arch}' --build-arg USE_MLIR='${mlir_build}' " - if(env.CCACHE_HOST) - { - def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() - if(check_host == "+PONG") - { - echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" - } - else - { - echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" - } - dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " - env.CCACHE_DIR = """/tmp/ccache_store""" - env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" - } - echo "Docker Args: ${dockerArgs}" - - def image = getDockerImageName(dockerArgs) - - def dockerImage - try{ - echo "Pulling down image: ${image}" - dockerImage = docker.image("${image}") - dockerImage.pull() - } - catch(org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } - catch(Exception ex) - { - dockerImage = docker.build("${image}", "${dockerArgs} .") - withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) { - dockerImage.push() - } - } - return [dockerImage, image] -} - -def buildHipClangJob(Map conf=[:]){ - show_node_info() - miopenCheckout() - env.HSA_ENABLE_SDMA=0 - env.CODECOV_TOKEN="aec031be-7673-43b5-9840-d8fb71a2354e" - env.DOCKER_BUILDKIT=1 - def image - def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" - if (conf.get("enforce_xnack_on", false)) { - dockerOpts = dockerOpts + " --env HSA_XNACK=1" - } - - def variant = env.STAGE_NAME - - def codecov = conf.get("codecov", false) - def needs_gpu = conf.get("needs_gpu", true) - def lfs_pull = conf.get("lfs_pull", false) - - def retimage - gitStatusWrapper(credentialsId: "${env.miopen_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'MIOpen') { - try { - (retimage, image) = getDockerImage(conf) - if (needs_gpu) { - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES') - { - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' - } - } - } - } - catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } - catch(Exception ex) { - (retimage, image) = getDockerImage(conf) - if (needs_gpu) { - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES') - { - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' - } - } - } - } - - withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 420, unit:'MINUTES') - { - if (lfs_pull) { - sh "git lfs pull --exclude=" - } - - cmake_build(conf) - - if (codecov) { - sh ''' - cd build - lcov --directory . --capture --output-file $(pwd)/coverage.info - lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info - lcov --list $(pwd)/coverage.info - curl -s https://codecov.io/bash | bash - echo "Uploaded" - ''' - } - } - } - } - return retimage -} - -def reboot(){ - build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),] -} - -def buildHipClangJobAndReboot(Map conf=[:]){ - try{ - buildHipClangJob(conf) - cleanWs() - } - catch(e){ - echo "throwing error exception for the stage" - echo 'Exception occurred: ' + e.toString() - throw e - } - finally{ - if (conf.get("needs_reboot", true)) { - reboot() - } - } -} - -def RunPerfTest(Map conf=[:]){ - def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" - try { - (retimage, image) = getDockerImage(conf) - withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 600, unit: 'MINUTES') - { - unstash 'miopen_tar' - sh "tar -zxvf build/miopen-hip-*-Linux-runtime.tar.gz" - ld_lib="${env.WORKSPACE}/opt/rocm/lib" - def filename = conf.get("filename", "") - if (env.BRANCH_NAME == env.MIOPEN_GOLDEN_PERF_BRANCH || params.PERF_TEST_BRANCH_OVERRIDE){ - if(params.PERF_TEST_OVERRIDE != '') - { - echo "Appending MIOpenDriver cmd env vars: ${params.PERF_TEST_OVERRIDE}" - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm --override ${params.PERF_TEST_OVERRRIDE}" - }else - { - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm" - } - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm" - jenkins_url = "${env.artifact_path}/${env.MIOPEN_GOLDEN_PERF_BRANCH}/lastSuccessfulBuild/artifact" - try { - sh "rm -rf ${env.WORKSPACE}/opt/rocm/bin/old_results/" - sh "wget -P ${env.WORKSPACE}/opt/rocm/bin/old_results/ ${jenkins_url}/opt/rocm/bin/perf_results/${filename}" - } - catch (Exception err){ - currentBuild.result = 'SUCCESS' - } - } - - archiveArtifacts artifacts: "opt/rocm/bin/perf_results/${filename}", allowEmptyArchive: true, fingerprint: true - try{ - if (env.BRANCH_NAME != env.MIOPEN_GOLDEN_PERF_BRANCH){ - sh "${env.WORKSPACE}/opt/rocm/bin/test_perf.py --compare_results --old_results_path ${env.WORKSPACE}/opt/rocm/bin/old_results --filename ${filename}" - } - } - catch (Exception err){ - currentBuild.result = 'SUCCESS' - } - cleanWs() - } - } - } - catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } -} - - -def CheckPerfDbValid(Map conf=[:]){ - def pdb_image = buildHipClangJob(conf) - pdb_image.inside(){ - dir(path: "$WORKSPACE"){ - sh "ls install/bin/" - sh "MIOPEN_LOG_LEVEL=4 LD_LIBRARY_PATH='install/lib:/opt/rocm/lib/' install/bin/fin -i fin/tests/pdb_check_all.json -o pdb_valid_err.json" - archiveArtifacts "pdb_valid_err.json" - sh "grep clear pdb_valid_err.json" - def has_error = sh ( - script: "echo \$?", - returnStdout: true - ).trim() - assert has_error.toInteger() == 0 - } - } -} - -/// Stage name format: -/// [DataType] Backend[/Compiler] BuildType [TestSet] [Target] -/// -/// The only mandatory elements are Backend and BuildType; others are optional. -/// -/// DataType := { Fp16 | Bf16 | Int8 | Fp32 } -/// Backend := { Hip | HipNoGPU} -/// Compiler := { Clang* | GCC* } -/// * "Clang" is the default for the Hip backend, and implies hip-clang compiler. -/// * The default compiler is usually not specified. -/// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ] -/// * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find -/// NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... } -/// TestSet := { All | Smoke* | | Build-only } [ Codecov ] -/// * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On". -/// * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified. -/// * "Codecov" is optional code coverage analysis. -/// * "Performance Dataset" is a performance test with a specified dataset. -/// Target := { gfx908 | gfx90a | Vega20 | Vega10 | Vega* | gfx1030 } [ Xnack+ ] -/// * "Vega" (gfx906 or gfx900) is the default and usually not specified. - - -pipeline { - agent none - options { - parallelsAlwaysFailFast() - // disable stage-wise timeout due to long wait with queue (limited resources) - // timeout(time: 90, unit:'MINUTES') - } - parameters { - booleanParam( - name: "BUILD_DOCKER", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_STATIC_CHECKS", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_SMOKE_FP32", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_SMOKE_AUX1", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_SMOKE_FP16_BF16_INT8", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_FULL_TESTS", - defaultValue: true, - description: "") - booleanParam( - name: "BUILD_PACKAGES", - defaultValue: true, - description: "") - booleanParam( - name: "TARGET_NOGPU", - defaultValue: true, - description: "") - booleanParam( - name: "TARGET_VEGA10", - defaultValue: false, - description: "") - booleanParam( - name: "TARGET_VEGA20", - defaultValue: false, - description: "") - booleanParam( - name: "TARGET_GFX908", - defaultValue: env.BRANCH_NAME == "develop" ? true : false, - description: "") - booleanParam( - name: "TARGET_GFX90A", - defaultValue: true, - description: "") - booleanParam( - name: "TARGET_GFX94X", - defaultValue: false, - description: "") - booleanParam( - name: "TARGET_NAVI21", - defaultValue: false, - description: "") - booleanParam( - name: "TARGET_NAVI32", - defaultValue: false, - description: "") - booleanParam( - name: "DATATYPE_NA", - defaultValue: true, - description: "") - booleanParam( - name: "DATATYPE_FP32", - defaultValue: true, - description: "") - booleanParam( - name: "DATATYPE_FP16", - defaultValue: true, - description: "") - booleanParam( - name: "DATATYPE_BF16", - defaultValue: true, - description: "") - booleanParam( - name: "DATATYPE_INT8", - defaultValue: true, - description: "") - booleanParam( - name: "PERF_TEST", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_FP16", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_FP32", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_BRANCH_OVERRIDE", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "DBSYNC_TEST", - defaultValue: true, - description: "Enable database synchronization testing stages") - string(name: "PERF_TEST_OVERRIDE", - defaultValue: '', - description: "Add extra env vars for the MIOpenDriver cmd, comma separated") - string(name: "DOCKER_IMAGE_OVERRIDE", - defaultValue: '', - description: "") - booleanParam( - name: "WORKAROUND__TARGET_GFX94X_MINIMUM_TEST_ENABLE", - defaultValue: false, - description: "") - } - - environment{ - extra_log_env = " MIOPEN_LOG_LEVEL=5 " - Fp16_flags = " -DMIOPEN_TEST_HALF=On" - Bf16_flags = " -DMIOPEN_TEST_BFLOAT16=On" - Int8_flags = " -DMIOPEN_TEST_INT8=On" - Full_test = " -DMIOPEN_TEST_ALL=On" - Smoke_targets = " check MIOpenDriver" - NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" - NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" - } - triggers{ - - cron(env.BRANCH_NAME == env.NIGHTLY_BRANCH ? env.NIGHTLY_SCHEDULE : '') - } - stages{ - stage('Build Docker'){ - when { - expression { params.BUILD_DOCKER && params.TARGET_NOGPU && params.DATATYPE_NA } - } - agent{ label rocmnode("nogpu") } - steps{ - getDockerImage() - } - } - stage("Packages") { - when { - expression { params.BUILD_PACKAGES && params.TARGET_NOGPU && params.DATATYPE_NA } - } - parallel { - stage("HIP Package") { - agent{ label rocmnode("nogpu") } - steps{ - buildHipClangJobAndReboot( package_build: "true", needs_gpu:false, needs_reboot:false) - } - } - } - } - stage("Static checks") { - when { - expression { params.BUILD_STATIC_CHECKS && params.TARGET_NOGPU && params.DATATYPE_NA } - } - parallel{ - stage('Hip Tidy') { - agent{ label rocmnode("nogpu") } - environment{ - setup_cmd = "CXX='/opt/rocm/llvm/bin/clang++' cmake -DCMAKE_PREFIX_PATH=/opt/rocm -DMIOPEN_BACKEND=HIP -DBUILD_DEV=On .. " - build_cmd = "make -j\$(nproc) -k analyze" - } - steps{ - buildHipClangJobAndReboot(setup_cmd: setup_cmd, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false) - } - } - stage('Clang Format') { - agent{ label rocmnode("nogpu") } - environment{ - execute_cmd = "find .. -iname \'*.h\' \ - -o -iname \'*.hpp\' \ - -o -iname \'*.cpp\' \ - -o -iname \'*.h.in\' \ - -o -iname \'*.hpp.in\' \ - -o -iname \'*.cpp.in\' \ - -o -iname \'*.cl\' \ - | grep -v -E '(build/)|(install/)|(fin/)' \ - | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'" - } - steps{ - buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, needs_gpu:false, needs_reboot:false) - } - } - stage('HipNoGPU Debug Build Test') { - when { - beforeAgent true - expression { params.TARGET_NOGPU } - } - agent{ label rocmnode("nogpu") } - environment{ - HipNoGPU_flags = "-DMIOPEN_BACKEND=HIPNOGPU -DMIOPEN_INSTALL_CXX_HEADERS=On" - build_cmd = "make -j\$(nproc)" - } - steps{ - buildHipClangJob( build_type: 'debug', setup_flags: HipNoGPU_flags, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false) - } - } - stage('Tuna Fin Build Test') { - agent{ label rocmnode("nogpu") } - environment{ - fin_flags = "-DMIOPEN_BACKEND=HIPNOGPU" - } - steps{ - buildHipClangJobAndReboot(setup_flags: fin_flags, make_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true") - } - } - } - } - stage("Smoke Fp32") { - when { - expression { params.BUILD_SMOKE_FP32 && params.DATATYPE_FP32 } - } - parallel{ - stage('Fp32 Hip gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp32 Hip Debug gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp32 Hip Debug gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp32 Hip Debug gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X || params.WORKAROUND__TARGET_GFX94X_MINIMUM_TEST_ENABLE } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false, build_install: "true") - } - } - } - } - stage("Smoke Aux 1") { - when { - expression { params.BUILD_SMOKE_AUX1 && params.DATATYPE_FP32 } - } - parallel{ - stage('Fp32 Hip Debug NOCOMGR gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - environment{ - // Can be removed altogether with when WORKAROUND_SWDEV_290754. - NOCOMGR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" - } - steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ', build_install: "true") - } - } - stage('Fp32 Hip Debug NOMLIR gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - environment{ - // Can be removed altogether with when WORKAROUND_SWDEV_290754. - NOMLIR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" - } - steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ', build_install: "true") - } - } - stage('Fp32 Hip Debug NOCK gfx90a Build-Only') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "", build_install: "true") - } - } - stage('Fp32 Hip Debug Embedded Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - environment{ - Embedded_flags = "-DMIOPEN_EMBED_DB='gfx906_60'" - } - steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ', build_install: "true") - } - } - stage('Fp32 Hip Static gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF', build_install: "true") - } - } - stage('Fp32 Hip Normal-Find gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - environment{ - make_targets = "test_conv2d" - execute_cmd = "bin/test_conv2d --disable-verification-cache" - } - steps{ - buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal", build_install: "true") - } - } - stage('Fp32 Hip Fast-Find gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - environment{ - make_targets = "test_conv2d" - execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 bin/test_conv2d --disable-verification-cache" - } - steps{ - buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd, build_install: "true") - } - } - stage('Fp32 Hip gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot() - } - } - stage('Fp32 Hip SqlitePerfdb gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On", build_install: "true") - } - } - } - } - stage("Smoke Fp16/Bf16/Int8") { - when { - expression { params.BUILD_SMOKE_FP16_BF16_INT8 } - } - parallel{ - stage('Fp16 Hip Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Bf16 Hip Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp16 Hip gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Bf16 Hip gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp16 Hip gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Bf16 Hip gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") - } - } - stage('Fp16 Hip gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") - } - } - stage('Bf16 Hip gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") - } - } - } - } - stage("Full Tests") { - when { - expression { params.BUILD_FULL_TESTS} - } - environment{ - // WORKAROUND_ISSUE_1148: "CTEST_PARALLEL_LEVEL=2" - // WORKAROUND_SWDEV_290754: "LLVM_PATH=/opt/rocm/llvm" - Navi21_build_cmd = "LLVM_PATH=/opt/rocm/llvm CTEST_PARALLEL_LEVEL=2 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" - } - parallel{ - stage('Dbsync gfx908') { - when { - beforeAgent true - expression { params.DBSYNC_TEST && params.TARGET_GFX908 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(lfs_pull: true, - setup_flags: "-DMIOPEN_TEST_DBSYNC=1", - make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', - needs_gpu:false, - needs_reboot:false, - build_install: "true") - } - } - stage('Dbsync gfx90a') { - when { - beforeAgent true - expression { params.DBSYNC_TEST && params.TARGET_GFX90A } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(lfs_pull: true, - setup_flags: "-DMIOPEN_TEST_DBSYNC=1", - make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', - needs_gpu:false, - needs_reboot:false, - build_install: "true") - } - } - stage('Dbsync gfx942') { - when { - beforeAgent true - expression { params.DBSYNC_TEST && (params.TARGET_GFX94X || params.WORKAROUND__TARGET_GFX94X_MINIMUM_TEST_ENABLE) } - } - options { - retry(2) - } - agent{ label rocmnode("gfx942") } - steps{ - buildHipClangJobAndReboot(lfs_pull: true, - setup_flags: "-DMIOPEN_TEST_DBSYNC=1", - make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', - needs_gpu:false, - needs_reboot:false, - build_install: "true") - } - } - stage('Int8 HIP All Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 && params.DATATYPE_INT8 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - steps{ - buildHipClangJobAndReboot( setup_flags: Int8_flags + Full_test) - } - } - stage('Bf16 Hip Install All gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true") - } - } - stage('Bf16 Hip Install All gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true") - } - } - stage('Bf16 Hip Install All gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X && params.DATATYPE_BF16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true", needs_reboot:false) - } - } - stage('Fp16 Hip All gfx1030') { - when { - beforeAgent true - expression { params.TARGET_NAVI21 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("navi21") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_cmd: Navi21_build_cmd) - } - } - stage('Fp16 Hip All gfx1101') { - when { - beforeAgent true - expression { params.TARGET_NAVI32 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("navi32") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags) - } - } - stage('Fp32 Hip All gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test) - } - } - stage('Fp32 Hip All gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test) - } - } - // stage('Fp32 Hip All gfx90a Xnack+') { - // when { - // beforeAgent true - // expression { params.TARGET_GFX90A && params.DATATYPE_FP32 } - // } - // agent{ label rocmnode("gfx90a") } - // steps{ - // buildHipClangJobAndReboot(setup_flags: Full_test, enforce_xnack_on: true) - // } - // } - stage('Fp32 Hip All gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, needs_reboot:false) - } - } - stage('Fp16 Hip Install All Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - steps{ - buildHipClangJobAndReboot( setup_flags: Full_test + Fp16_flags, build_install: "true") - } - } - stage('Fp32 Hip All Vega20') { - when { - beforeAgent true - expression { params.TARGET_VEGA20 && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("vega20") } - steps{ - buildHipClangJobAndReboot( setup_flags: Full_test) - } - } - stage('Fp32 Hip All Install gfx1030') { - when { - beforeAgent true - expression { params.TARGET_NAVI21 && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("navi21") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, build_cmd: Navi21_build_cmd, build_install: "true") - } - } - stage('Fp32 Hip All Install gfx1101') { - when { - beforeAgent true - expression { params.TARGET_NAVI32 && params.DATATYPE_FP32 } - } - options { - retry(2) - } - agent{ label rocmnode("navi32") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, build_install: "true") - } - } - stage('Fp16 Hip All Install gfx908') { - when { - beforeAgent true - expression { params.TARGET_GFX908 && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx908") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true") - } - } - stage('Fp16 Hip All Install gfx90a') { - when { - beforeAgent true - expression { params.TARGET_GFX90A && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx90a") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true") - } - } - stage('Fp16 Hip All Install gfx94X') { - when { - beforeAgent true - expression { params.TARGET_GFX94X && params.DATATYPE_FP16 } - } - options { - retry(2) - } - agent{ label rocmnode("gfx94X") } - steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true", needs_reboot:false) - } - } - } - } - stage("Performance Tests - gfx90a") { - when { - expression {params.PERF_TEST && params.TARGET_GFX90A} - } - parallel{ - stage('Fp32 BS128 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS4 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS64.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Densenet201_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Densenet201_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Densenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Densenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS512.txt" ) - } - } - stage('Fp32 BS4 Hip Performance Mobilenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance Mobilenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS64.txt" ) - } - } - stage('Fp16 BS32 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS32.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet152_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet152_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS512.txt" ) - } - } - stage('Fp16 BS32 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS32.txt" ) - } - } - stage('Fp16 BS64 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS64.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 B512 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Shufflenet_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Shufflenet_v2_FP16_BS128.txt" ) - } - } - stage('Fp16 BS128 Hip Performance SSD_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS128 Hip Performance SSD_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP32_BS128.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS4 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS64.txt" ) - } - } - stage('Fp32 BS128 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS512.txt" ) - } - } - } - } - } -} diff --git a/cmake/EnableCompilerWarnings.cmake b/cmake/EnableCompilerWarnings.cmake index b64c717020..e69de29bb2 100644 --- a/cmake/EnableCompilerWarnings.cmake +++ b/cmake/EnableCompilerWarnings.cmake @@ -1,130 +0,0 @@ -################################################################################ -# -# MIT License -# -# Copyright (c) 2017 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ -# - Enable warning all for gcc/clang or use /W4 for visual studio - -## Strict warning level -set(__msvc_cxx_compile_options /W4) - -set(__default_cxx_compile_options - -Wall - -Wextra - -Wcomment - -Wendif-labels - -Wformat - -Winit-self - -Wreturn-type - -Wsequence-point - -Wswitch - -Wtrigraphs - -Wundef - -Wuninitialized - -Wunreachable-code - -Wunused - -Wno-ignored-qualifiers - -Wno-sign-compare -) - -set(__clang_cxx_compile_options - -Weverything - -Wno-c++98-compat - -Wno-c++98-compat-pedantic - -Wno-conversion - -Wno-double-promotion - -Wno-exit-time-destructors - -Wno-extra-semi - -Wno-extra-semi-stmt - -Wno-float-conversion - -Wno-gnu-anonymous-struct - -Wno-gnu-zero-variadic-macro-arguments - -Wno-missing-prototypes - -Wno-nested-anon-types - -Wno-option-ignored - -Wno-padded - -Wno-return-std-move-in-c++11 - -Wno-shorten-64-to-32 - -Wno-sign-conversion - -Wno-unknown-warning-option - -Wno-unused-command-line-argument - -Wno-weak-vtables - -Wno-covered-switch-default - -Wno-unused-result - -Wno-unsafe-buffer-usage - -Wno-deprecated-declarations - -Wno-shadow-uncaptured-local - -Wno-global-constructors - -Wno-reserved-identifier - -Wno-zero-as-null-pointer-constant - -Wno-ignored-attributes - -Wno-deprecated - -Wno-incompatible-pointer-types - -Wno-old-style-cast - -Wno-unknown-attributes - -Wno-microsoft-cpp-macro - -Wno-microsoft-enum-value - -Wno-language-extension-token - -Wno-c++11-narrowing - -Wno-float-equal - -Wno-redundant-parens - -Wno-format-nonliteral - -Wno-unused-template - -Wno-comma - -Wno-suggest-destructor-override - -Wno-switch-enum - -Wno-shift-sign-overflow - -Wno-suggest-override - -Wno-inconsistent-missing-destructor-override - -Wno-cast-function-type - -Wno-nonportable-system-include-path - -Wno-incompatible-pointer-types - -Wno-documentation - -Wno-deprecated-builtins - -Wno-enum-constexpr-conversion - -Wno-unused-value - -Wno-unused-parameter - -Wno-missing-noreturn - -Wno-tautological-constant-out-of-range-compare - -Wno-c++20-extensions) -if(WIN32) - list(APPEND __clang_cxx_compile_options - -fdelayed-template-parsing - -fms-extensions - -fms-compatibility) -endif() - -set(__gnu_cxx_compile_options - -Wno-missing-field-initializers -) - -add_compile_options( - "$<$:${__msvc_cxx_compile_options}>" - "$<$:${__default_cxx_compile_options};${__clang_cxx_compile_options}>" - "$<$:${__default_cxx_compile_options};${__gnu_cxx_compile_options}>" -) - -unset(__msvc_cxx_compile_options) -unset(__default_cxx_compile_options) -unset(__gnu_cxx_compile_options) -unset(__clang_cxx_compile_options) diff --git a/docs/conceptual/perfdb.rst b/docs/conceptual/perfdb.rst index 3ecb3c2a5a..e69de29bb2 100644 --- a/docs/conceptual/perfdb.rst +++ b/docs/conceptual/perfdb.rst @@ -1,85 +0,0 @@ -.. meta:: - :description: Using the performance database - :keywords: MIOpen, ROCm, API, documentation, performance database - -************************************************************************************************ -Using the performance database -************************************************************************************************ - -Many MIOpen kernels have parameters that affect their performance. Setting these parameters to -optimal values allows for the best possible throughput. Optimal values depend on many factors, -including network configuration, GPU type, clock frequencies, and ROCm version. - -Due to the large number of possible configurations and settings, MIOpen provides a set of pre-tuned -values for the `most applicable` network configurations and a means for expanding the set of -optimized values. MIOpen's performance database (PerfDb) contains these pre-tuned parameter values -in addition to any user-optimized parameters. - -The PerfDb consists of two parts: - -* **System PerfDb**: A system-wide storage that holds pre-run values for the most applicable - configurations. -* **User PerfDb**: A per-user storage that holds optimized values for arbitrary configurations. - -User PerfDb `always takes precedence` over System PerfDb. - -MIOpen also has auto-tuning functionality, which is able to find optimized kernel parameter values for -a specific configuration. The auto-tune process may take a long time, but once optimized values are -found, they're stored in the User PerfDb. MIOpen then automatically reads and uses these parameter -values. - -By default, System PerfDb resides within MIOpen's install location, while User PerfDb resides in your -home directory. See :ref:`setting up locations ` for more information. - -System PerfDb is not modified during MIOpen installation. - -Auto-tuning kernels -========================================================== - -MIOpen performs auto-tuning during the these API calls: - -* ``miopenFindConvolutionForwardAlgorithm()`` -* ``miopenFindConvolutionBackwardDataAlgorithm()`` -* ``miopenFindConvolutionBackwardWeightsAlgorithm()`` - -Auto-tuning is performed for only one `problem configuration`, which is implicitly defined by the -tensor descriptors that are passed to the API function. - -In order for auto-tuning to begin, the following conditions must be met: - -* The applicable kernels have tuning parameters -* The value of the ``exhaustiveSearch`` parameter is ``true`` -* Neither System nor User PerfDb can contain values for the relevant `problem configuration`. - -You can override the latter two conditions by enforcing the search using the -``- MIOPEN_FIND_ENFORCE`` environment variable. You can also use this variable to remove values -from User PerfDb, as described in the following section. - -Using MIOPEN_FIND_ENFORCE ----------------------------------------------------------------------------------------------------------- - -``MIOPEN_FIND_ENFORCE`` supports symbolic (case-insensitive) and numeric values. Possible values -are: - -* ``NONE``/``(1)``: No change in the default behavior. -* ``DB_UPDATE/``(2)``: Do not skip auto-tune (even if PerfDb already contains optimized values). If you - request auto-tune via API, MIOpen performs it and updates PerfDb. You can use this mode for - fine-tuning the MIOpen installation on your system. However, this mode slows down processes. -* ``SEARCH``/``(3)``: Perform auto-tune even if not requested via API. In this case, the library behaves as - if the ``exhaustiveSearch`` parameter set to ``true``. If PerfDb already contains optimized values, - auto-tune is not performed. You can use this mode to tune applications that don't anticipate means - for getting the best performance from MIOpen. When in this mode, your application's first run may - take substantially longer than expected. -* ``SEARCH_DB_UPDATE``/``(4)``: A combination of ``DB_UPDATE`` and ``SEARCH``. MIOpen performs - auto-tune (and updates User PerfDb) on each ``miopenFindConvolution*()`` call. This mode is - recommended only for debugging purposes. -* ``DB_CLEAN``/``(5)``: Removes optimized values related to the `problem configuration` from User - PerfDb. Auto-tune is blocked, even if explicitly requested. System PerfDb is left intact. **Use this - option with care.** - -Updating MIOpen and User PerfDb -========================================================== - -If you install a new version of MIOpen, we strongly recommend moving or deleting your old User -PerfDb file. This prevents older database entries from affecting configurations within the newer system -database. The User PerfDb is named ``miopen.udb`` and is located at the User PerfDb path. diff --git a/docs/conceptual/porting-guide.rst b/docs/conceptual/porting-guide.rst index 2c50bba940..e69de29bb2 100644 --- a/docs/conceptual/porting-guide.rst +++ b/docs/conceptual/porting-guide.rst @@ -1,1514 +0,0 @@ -.. meta:: - :description: Porting MIOpen - :keywords: MIOpen, ROCm, API, documentation, porting - -******************************************************************** -Porting to MIOpen -******************************************************************** - -The following is a summary of the key differences between MIOpen and cuDNN. - -* Calling ``miopenFindConvolution*Algorithm()`` is `mandatory` before calling any Convolution API -* The typical calling sequence for MIOpen Convolution APIs is: - - * ``miopenConvolution*GetWorkSpaceSize()`` (returns the workspace size required by ``Find()``) - * ``miopenFindConvolution*Algorithm()`` (returns performance information for various algorithms) - * ``miopenConvolution*()`` - -MIOpen supports: - -* 4D tensors in the NCHW and NHWC storage format; the cuDNN ``__“\*Nd\*”__`` APIs don't have a - corresponding MIOpen API -* ``__`float(fp32)`__`` datatype -* ``__2D Convolutions__`` and ``__3D Convolutions__`` -* ``__2D Pooling__`` - -MIOpen doesn't support: - -* ``__Preferences__`` for convolutions -* Softmax modes (MIOpen implements the ``__SOFTMAX_MODE_CHANNEL__`` flavor) -* ``__Transform-Tensor__``, ``__Dropout__``, ``__RNNs__``, and ``__Divisive Normalization__`` - -Useful MIOpen environment variables include: - -* ``MIOPEN_ENABLE_LOGGING=1``: Logs all the MIOpen APIs called, including the parameters passed - to those APIs -* ``MIOPEN_DEBUG_GCN_ASM_KERNELS=0``: Disables hand-tuned ASM kernels (the fallback is to use - kernels written in a high-level language) -* ``MIOPEN_DEBUG_CONV_FFT=0``: Disables the FFT convolution algorithm -* ``MIOPEN_DEBUG_CONV_DIRECT=0``: Disables the direct convolution algorithm - -cuDNN versus MIOpen APIs -=================================================== - -The following sections compare cuDNN and MIOpen APIs with similar functions. - -Handle operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreate( - cudnnHandle_t *handle) - - .. code-block:: cpp - - miopenStatus_t - miopenCreate( - miopenHandle_t *handle) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroy( - cudnnHandle_t handle) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroy( - miopenHandle_t handle) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetStream( - cudnnHandle_t handle, - cudaStream_t streamId) - - .. code-block:: cpp - - miopenStatus_t - miopenSetStream( - miopenHandle_t handle, - miopenAcceleratorQueue_t streamID) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetStream( - cudnnHandle_t handle, - cudaStream_t *streamId) - - .. code-block:: cpp - - miopenStatus_t - miopenGetStream( - miopenHandle_t handle, - miopenAcceleratorQueue_t *streamID) - -Tensor operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreateTensorDescriptor( - cudnnTensorDescriptor_t *tensorDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenCreateTensorDescriptor( - miopenTensorDescriptor_t - *tensorDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroyTensorDescriptor( - cudnnTensorDescriptor_t tensorDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroyTensorDescriptor( - miopenTensorDescriptor_t tensorDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetTensor( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t yDesc, - void *y, - const void *valuePtr) - - .. code-block:: cpp - - miopenStatus_t - miopenSetTensor( - miopenHandle_t handle, - const miopenTensorDescriptor_t yDesc, - void *y, - const void *alpha) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetTensor4dDescriptor( - cudnnTensorDescriptor_t tensorDesc, - cudnnTensorFormat_t format, - cudnnDataType_t dataType, - int n, - int c, - int h, - int w) - - .. code-block:: cpp - - miopenStatus_t miopenSet4dTensorDescriptor( - miopenTensorDescriptor_t tensorDesc, - miopenDataType_t dataType, - int n, - int c, - int h, - int w) - - (Only ``NCHW`` format is supported) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetTensor4dDescriptor( - cudnnTensorDescriptor_t tensorDesc, - cudnnDataType_t *dataType, - int *n, - int *c, - int *h, - int *w, - int *nStride, - int *cStride, - int *hStride, - int *wStride) - - .. code-block:: cpp - - miopenStatus_t - miopenGet4dTensorDescriptor( - miopenTensorDescriptor_t tensorDesc, - miopenDataType_t *dataType, - int *n, - int *c, - int *h, - int *w, - int *nStride, - int *cStride, - int *hStride, - int *wStride) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnAddTensor( - cudnnHandle_t handle, - const void *alpha, - const cudnnTensorDescriptor_t aDesc, - const void *A, - const void *beta, - const cudnnTensorDescriptor_t cDesc, - void *C) - - .. code-block:: cpp - - miopenStatus_t - miopenOpTensor( - miopenHandle_t handle, - miopenTensorOp_t tensorOp, - const void *alpha1, - constmiopenTensorDescriptor_t aDesc, - const void *A, - const void *alpha2, - const miopenTensorDescriptor_t bDesc, - const void *B, - const void *beta, - const miopenTensorDescriptor_t cDesc, - void *C) - - For forward bias, use ``miopenConvolutionForwardBias``. - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnOpTensor( - cudnnHandle_t handle, - const cudnnOpTensorDescriptor_t opTensorDesc, - const void *alpha1, - const cudnnTensorDescriptor_t aDesc, - const void *A, - const void *alpha2, - const cudnnTensorDescriptor_t bDesc, - const void *B, - const void *beta, - const cudnnTensorDescriptor_t cDesc, - void *C) - - .. code-block:: cpp - - miopenStatus_t - miopenOpTensor( - miopenHandle_t handle, - miopenTensorOp_t tensorOp, - const void *alpha1, - const miopenTensorDescriptor_t aDesc, - const void *A, const void *alpha2, - const miopenTensorDescriptor_t bDesc, - const void *B, - const void *beta, - const miopenTensorDescriptor_t cDesc, - void *C) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnOpTensor( - cudnnHandle_t handle, - const cudnnOpTensorDescriptor_t opTensorDesc, - const void *alpha1, - const cudnnTensorDescriptor_t aDesc, - const void *A, - const void *alpha2, - const cudnnTensorDescriptor_t bDesc, - const void *B, - const void *beta, - const cudnnTensorDescriptor_t cDesc, - void *C) - - .. code-block:: cpp - - miopenStatus_t - miopenOpTensor( - miopenHandle_t handle, - miopenTensorOp_t tensorOp, - const void *alpha1, - const miopenTensorDescriptor_t aDesc, - const void *A, const void *alpha2, - const miopenTensorDescriptor_t bDesc, - const void *B, - const void *beta, - const miopenTensorDescriptor_t cDesc, - void *C) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnScaleTensor( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t yDesc, - void *y, - const void *alpha) - - .. code-block:: cpp - - miopenStatus_t - miopenScaleTensor( - miopenHandle_t handle, - const miopenTensorDescriptor_t yDesc, - void *y, - const void *alpha) - -Filter operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreateFilterDescriptor( - cudnnFilterDescriptor_t *filterDesc) - - All ``FilterDescriptor`` APIs are substituted by their respective ``TensorDescriptor`` API. - -Convolution operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreateConvolutionDescriptor( - cudnnConvolutionDescriptor_t *convDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenCreateConvolutionDescriptor( - miopenConvolutionDescriptor_t *convDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroyConvolutionDescriptor( - cudnnConvolutionDescriptor_t convDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroyConvolutionDescriptor( - miopenConvolutionDescriptor_t convDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetConvolution2dDescriptor( - const cudnnConvolutionDescriptor_t convDesc, - int *pad_h, - int *pad_y, - int *u, - int *v, - int *upscalex, - int *upscaley, - cudnnConvolutionMode_t *mode) - - .. code-block:: cpp - - miopenStatus_t - miopenGetConvolutionDescriptor( - miopenConvolutionDescriptor_t convDesc, - miopenConvolutionMode_t *mode, - int *pad_h, - int *pad_y, - int *u, - int *v, - int *upscalex, - int *upscaley) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetConvolution2dForwardOutputDim( - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t inputTensorDesc, - const cudnnFilterDescriptor_t filterDesc, - int *n, - int *c, - int *h, - int *w) - - .. code-block:: cpp - - miopenStatus_t - miopenGetConvolutionForwardOutputDim( - miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t inputTensorDesc, - const miopenTensorDescriptor_t filterDesc, - int *n, - int *c, - int *h, - int *w) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetConvolutionForwardWorkspaceSize( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnFilterDescriptor_t wDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t yDesc, - cudnnConvolutionFwdAlgo_t algo, - size_t *sizeInBytes) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionForwardGetWorkSpaceSize( - miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - size_t *workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetConvolutionBackwardFilterWorkspaceSize( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnFilterDescriptor_t gradDesc, - cudnnConvolutionBwdFilterAlgo_t algo, - size_t *sizeInBytes) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionBackwardWeightsGetWorkSpaceSize( - miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - size_t *workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetConvolutionBackwardDataWorkspaceSize( - cudnnHandle_t handle, - const cudnnFilterDescriptor_t wDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t dxDesc, - cudnnConvolutionBwdDataAlgo_t algo, - size_t *sizeInBytes - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionBackwardDataGetWorkSpaceSize( - miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - size_t *workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnConvolutionForward( - cudnnHandle_t handle, - const void *alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnFilterDescriptor_t wDesc, - const void *w, - const cudnnConvolutionDescriptor_t convDesc, - cudnnConvolutionFwdAlgo_t algo, - void *workSpace, - size_t workSpaceSizeInBytes, - const void *beta, - const cudnnTensorDescriptor_t yDesc, - void *y) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionForward( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenTensorDescriptor_t wDesc, - const void *w, - const miopenConvolutionDescriptor_t convDesc, - miopenConvFwdAlgorithm_t algo, - const void *beta, - const miopenTensorDescriptor_t yDesc, - void *y, - void *workSpace, - size_t workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnFindConvolutionForwardAlgorithm( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnFilterDescriptor_t wDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t yDesc, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionFwdAlgoPerf_t *perfResults) - - cudnnStatus_t - cudnnFindConvolutionForwardAlgorithmEx( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnFilterDescriptor_t wDesc, - const void *w, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t yDesc, - void *y, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionFwdAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSizeInBytes) - - cudnnStatus_t - cudnnGetConvolutionForwardAlgorithm( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnFilterDescriptor_t wDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t yDesc, - cudnnConvolutionFwdPreference_t preference, - size_t memoryLimitInBytes, - cudnnConvolutionFwdAlgo_t *algo) - - - ``FindConvolution()`` is mandatory. - Allocate workspace prior to running this API. - A table with times and memory requirements for different algorithms is returned. - You can choose the top-most algorithm if you want only the fastest algorithm. - - .. code-block:: cpp - - miopenStatus_t - miopenFindConvolutionForwardAlgorithm( - miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenTensorDescriptor_t wDesc, - const void *w, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - void *y, - const int requestAlgoCount, - int *returnedAlgoCount, - miopenConvAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnConvolutionBackwardBias( - cudnnHandle_t handle, - const void *alpha, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const void *beta, - const cudnnTensorDescriptor_t dbDesc, - void *db) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionBackwardBias( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const void *beta, - const miopenTensorDescriptor_t dbDesc, - void *db) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnFindConvolutionBackwardFilterAlgorithm( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnFilterDescriptor_t dwDesc, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionBwdFilterAlgoPerf_t *perfResults) - - cudnnStatus_t - cudnnFindConvolutionBackwardFilterAlgorithmEx( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnTensorDescriptor_t dyDesc, - const void *y, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnFilterDescriptor_t dwDesc, - void *dw, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionBwdFilterAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSizeInBytes) - - cudnnStatus_t - cudnnGetConvolutionBackwardFilterAlgorithm( - cudnnHandle_t handle, - const cudnnTensorDescriptor_t xDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnFilterDescriptor_t dwDesc, - cudnnConvolutionBwdFilterPreference_t preference, - size_t memoryLimitInBytes, - cudnnConvolutionBwdFilterAlgo_t *algo) - - - ``FindConvolution()`` is mandatory. - Allocate workspace prior to running this API. - A table with times and memory requirements for different algorithms is returned. - You can choose the top-most algorithm if you want only the fastest algorithm. - - .. code-block:: cpp - - miopenStatus_t - miopenFindConvolutionBackwardWeightsAlgorithm( - miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - void *dw, - const int requestAlgoCount, - int *returnedAlgoCount, - miopenConvAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnFindConvolutionBackwardDataAlgorithm( - cudnnHandle_t handle, - const cudnnFilterDescriptor_t wDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t dxDesc, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionBwdDataAlgoPerf_t *perfResults) - - cudnnStatus_t - cudnnFindConvolutionBackwardDataAlgorithmEx( - cudnnHandle_t handle, - const cudnnFilterDescriptor_t wDesc, - const void *w, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t dxDesc, - void *dx, - const int requestedAlgoCount, - int *returnedAlgoCount, - cudnnConvolutionBwdDataAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSizeInBytes) - - cudnnStatus_t - cudnnGetConvolutionBackwardDataAlgorithm( - cudnnHandle_t handle, - const cudnnFilterDescriptor_t wDesc, - const cudnnTensorDescriptor_t dyDesc, - const cudnnConvolutionDescriptor_t convDesc, - const cudnnTensorDescriptor_t dxDesc, - cudnnConvolutionBwdDataPreference_t preference, - size_t memoryLimitInBytes, - cudnnConvolutionBwdDataAlgo_t *algo) - - - ``FindConvolution()`` is mandatory. - Allocate workspace prior to running this API. - A table with times and memory requirements for different algorithms is returned. - You can choose the top-most algorithm if you want only the fastest algorithm. - - .. code-block:: cpp - - miopenStatus_t - miopenFindConvolutionBackwardDataAlgorithm( - miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t wDesc, - const void *w, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - const void *dx, - const int requestAlgoCount, - int *returnedAlgoCount, - miopenConvAlgoPerf_t *perfResults, - void *workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnConvolutionBackwardFilter( - cudnnHandle_t handle, - const void *alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnConvolutionDescriptor_t convDesc, - cudnnConvolutionBwdFilterAlgo_t algo, - void *workSpace, - size_t workSpaceSizeInBytes, - const void *beta, - const cudnnFilterDescriptor_t dwDesc, - void *dw) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionBackwardWeights( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenConvolutionDescriptor_t convDesc, - miopenConvBwdWeightsAlgorithm_t algo, - const void *beta, - const miopenTensorDescriptor_t dwDesc, - void *dw, - void *workSpace, - size_t workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnConvolutionBackwardData( - cudnnHandle_t handle, - const void *alpha, - const cudnnFilterDescriptor_t wDesc, - const void *w, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnConvolutionDescriptor_t convDesc, - cudnnConvolutionBwdDataAlgo_t algo, - void *workSpace, - size_t workSpaceSizeInBytes, - const void *beta, - const cudnnTensorDescriptor_t dxDesc, - void *dx) - - .. code-block:: cpp - - miopenStatus_t - miopenConvolutionBackwardData( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t wDesc, - const void *w, - const miopenConvolutionDescriptor_t convDesc, - miopenConvBwdDataAlgorithm_t algo, - const void *beta, - const miopenTensorDescriptor_t dxDesc, - void *dx, - void *workSpace, - size_t workSpaceSize) - -Softmax operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSoftmaxForward( - cudnnHandle_t handle, - cudnnSoftmaxAlgorithm_t algo, - cudnnSoftmaxMode_t mode, - const void *alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t yDesc, - void *y) - - .. code-block:: cpp - - miopenStatus_t - miopenSoftmaxForward( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t yDesc, - void *y) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSoftmaxBackward( - cudnnHandle_t handle, - cudnnSoftmaxAlgorithm_t algo, - cudnnSoftmaxMode_t mode, - const void *alpha, - const cudnnTensorDescriptor_t yDesc, - const void *y, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const void *beta, - const cudnnTensorDescriptor_t dxDesc, - void *dx) - - .. code-block:: cpp - - miopenStatus_t - miopenSoftmaxBackward( - miopenHandle_t handle, - const void *alpha, - const miopenTensorDescriptor_t yDesc, - const void *y, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const void *beta, - const miopenTensorDescriptor_t dxDesc, - void *dx) - -Pooling operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreatePoolingDescriptor( - cudnnPoolingDescriptor_t *poolingDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenCreatePoolingDescriptor( - miopenPoolingDescriptor_t *poolDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetPooling2dDescriptor( - cudnnPoolingDescriptor_t poolingDesc, - cudnnPoolingMode_t mode, - cudnnNanPropagation_t maxpoolingNanOpt, - int windowHeight, - int windowWidth, - int verticalPadding, - int horizontalPadding, - int verticalStride, - int horizontalStride) - - .. code-block:: cpp - - miopenStatus_t - miopenSet2dPoolingDescriptor( - miopenPoolingDescriptor_t poolDesc, - miopenPoolingMode_t mode, - int windowHeight, - int windowWidth, - int pad_h, - int pad_w, - int u, - int v) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetPooling2dDescriptor( - const cudnnPoolingDescriptor_t poolingDesc, - cudnnPoolingMode_t *mode, - cudnnNanPropagation_t *maxpoolingNanOpt, - int *windowHeight, - int *windowWidth, - int *verticalPadding, - int *horizontalPadding, - int *verticalStride, - int *horizontalStride) - - .. code-block:: cpp - - miopenStatus_t - miopenGet2dPoolingDescriptor( - const miopenPoolingDescriptor_t poolDesc, - miopenPoolingMode_t *mode, - int *windowHeight, - int *windowWidth, - int *pad_h, - int *pad_w, - int *u, - int *v) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetPooling2dForwardOutputDim( - const cudnnPoolingDescriptor_t poolingDesc, - const cudnnTensorDescriptor_t inputTensorDesc, - int *n, - int *c, - int *h, - int *w) - - .. code-block:: cpp - - miopenStatus_t - miopenGetPoolingForwardOutputDim( - const miopenPoolingDescriptor_t poolDesc, - const miopenTensorDescriptor_t tensorDesc, - int *n, - int *c, - int *h, - int *w) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroyPoolingDescriptor( - cudnnPoolingDescriptor_t poolingDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroyPoolingDescriptor( - miopenPoolingDescriptor_t poolDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnPoolingForward( - cudnnHandle_t handle, - const cudnnPoolingDescriptor_t poolingDesc, - const void *alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t yDesc, - void *y) - - .. code-block:: cpp - - miopenStatus_t - miopenPoolingForward( - miopenHandle_t handle, - const miopenPoolingDescriptor_t poolDesc, - const void *alpha, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t yDesc, - void *y, - bool do_backward, - void *workSpace, - size_t workSpaceSize) - - * - - NA - - .. code-block:: cpp - - miopenStatus_t - miopenPoolingGetWorkSpaceSize( - const miopenTensorDescriptor_t yDesc, - size_t *workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnPoolingBackward( - cudnnHandle_t handle, - const cudnnPoolingDescriptor_t poolingDesc, - const void *alpha, - const cudnnTensorDescriptor_t yDesc, - const void *y, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t dxDesc, - void *dx) - - .. code-block:: cpp - - miopenStatus_t - miopenPoolingBackward( - miopenHandle_t handle, - const miopenPoolingDescriptor_t poolDesc, - const void *alpha, - const miopenTensorDescriptor_t yDesc, - const void *y, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t dxDesc, - void *dx, - const void *workspace) - -Activation operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreateActivationDescriptor( - cudnnActivationDescriptor_t *activationDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenCreateActivationDescriptor( - miopenActivationDescriptor_t *activDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetActivationDescriptor( - cudnnActivationDescriptor_t activationDesc, - cudnnActivationMode_t mode, - cudnnNanPropagation_t reluNanOpt, - double reluCeiling) - - .. code-block:: cpp - - miopenStatus_t - miopenSetActivationDescriptor( - const miopenActivationDescriptor_t activDesc, - miopenActivationMode_t mode, - double activAlpha, - double activBeta, - double activPower) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetActivationDescriptor( - const cudnnActivationDescriptor_t activationDesc, - cudnnActivationMode_t *mode, - cudnnNanPropagation_t *reluNanOpt, - double *reluCeiling) - - .. code-block:: cpp - - miopenStatus_t - miopenGetActivationDescriptor( - const miopenActivationDescriptor_t activDesc, - miopenActivationMode_t *mode, - double *activAlpha, - double *activBeta, - double *activPower) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroyActivationDescriptor( - cudnnActivationDescriptor_t activationDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroyActivationDescriptor( - miopenActivationDescriptor_t activDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnActivationForward( - cudnnHandle_t handle, - cudnnActivationDescriptor_t activationDesc, - const void *alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t yDesc, - void *y) - - .. code-block:: cpp - - miopenStatus_t - miopenActivationForward( - miopenHandle_t handle, - const miopenActivationDescriptor_t activDesc, - const void *alpha, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t yDesc, - void *y) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnActivationBackward( - cudnnHandle_t handle, - cudnnActivationDescriptor_t activationDesc, - const void *alpha, - const cudnnTensorDescriptor_t yDesc, - const void *y, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t dxDesc, - void *dx) - - .. code-block:: cpp - - miopenStatus_t - miopenActivationBackward( - miopenHandle_t handle, - const miopenActivationDescriptor_t activDesc, - const void *alpha, - const miopenTensorDescriptor_t yDesc, - const void *y, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t dxDesc, - void *dx) - -LRN operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnCreateLRNDescriptor( - cudnnLRNDescriptor_t *normDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenCreateLRNDescriptor( - miopenLRNDescriptor_t - *lrnDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnSetLRNDescriptor( - cudnnLRNDescriptor_t normDesc, - unsigned lrnN, - double lrnAlpha, - double lrnBeta, - double lrnK) - - .. code-block:: cpp - - miopenStatus_t - miopenSetLRNDescriptor( - const miopenLRNDescriptor_t lrnDesc, - miopenLRNMode_t mode, - unsigned lrnN, - double lrnAlpha, - double lrnBeta, - double lrnK) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnGetLRNDescriptor( - cudnnLRNDescriptor_t normDesc, - unsigned* lrnN, - double* lrnAlpha, - double* lrnBeta, - double* lrnK) - - .. code-block:: cpp - - miopenStatus_t - miopenGetLRNDescriptor( - const miopenLRNDescriptor_t lrnDesc, - miopenLRNMode_t *mode, - unsigned *lrnN, - double *lrnAlpha, - double *lrnBeta, - double *lrnK) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDestroyLRNDescriptor( - cudnnLRNDescriptor_t lrnDesc) - - .. code-block:: cpp - - miopenStatus_t - miopenDestroyLRNDescriptor( - miopenLRNDescriptor_t lrnDesc) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnLRNCrossChannelForward( - cudnnHandle_t handle, - cudnnLRNDescriptor_t normDesc, - cudnnLRNMode_t lrnMode, - const void* alpha, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t yDesc, - void *y) - - .. code-block:: cpp - - miopenStatus_t - miopenLRNForward( - miopenHandle_t handle, - const miopenLRNDescriptor_t lrnDesc, - const void *alpha, - const miopenTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const miopenTensorDescriptor_t yDesc, - void *y, - bool do_backward, - void *workspace) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnLRNCrossChannelBackward( - cudnnHandle_t handle, - cudnnLRNDescriptor_t normDesc, - cudnnLRNMode_t lrnMode, - const void* alpha, - const cudnnTensorDescriptor_t yDesc, - const void *y, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const void *beta, - const cudnnTensorDescriptor_t dxDesc, - void *dx) - - .. code-block:: cpp - - miopenStatus_t - miopenLRNBackward( - miopenHandle_t handle, - const miopenLRNDescriptor_t lrnDesc, - const void *alpha, - const miopenTensorDescriptor_t yDesc, - const void *y, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t xDesc, - const void *x, const void *beta, - const miopenTensorDescriptor_t dxDesc, - void *dx, - const void *workspace) - - * - - NA - - .. code-block:: cpp - - miopenStatus_t - miopenLRNGetWorkSpaceSize( - const miopenTensorDescriptor_t yDesc, - size_t *workSpaceSize) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnDeriveBNTensorDescriptor( - cudnnTensorDescriptor_t derivedBnDesc, - const cudnnTensorDescriptor_t xDesc, - cudnnBatchNormMode_t mode) - - .. code-block:: cpp - - miopenStatus_t - miopenDeriveBNTensorDescriptor( - miopenTensorDescriptor_t derivedBnDesc, - const miopenTensorDescriptor_t xDesc, - miopenBatchNormMode_t bn_mode) - -Batch normalization operations -------------------------------------------------------------------------------------------- - -.. list-table:: - :header-rows: 1 - - * - - cuDNN - - MIOpen - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnBatchNormalizationForwardTraining( - cudnnHandle_t handle, - cudnnBatchNormMode_t mode, - void *alpha, - void *beta, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnTensorDescriptor_t yDesc, - void *y, - const cudnnTensorDescriptor_t - bnScaleBiasMeanVarDesc, - void *bnScale, - void *bnBias, - double exponentialAverageFactor, - void *resultRunningMean, - void *resultRunningVariance, - double epsilon, - void *resultSaveMean, - void *resultSaveInvVariance) - - .. code-block:: cpp - - miopenStatus_t - miopenBatchNormalizationForwardTraining( - miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - void *alpha, - void *beta, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenTensorDescriptor_t yDesc, - void *y, - const miopenTensorDescriptor_t - bnScaleBiasMeanVarDesc, - void *bnScale, - void *bnBias, - double expAvgFactor, - void *resultRunningMean, - void *resultRunningVariance, - double epsilon, - void *resultSaveMean, - void *resultSaveInvVariance) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnnBatchNormalizationForwardInference( - cudnnHandle_t handle, - cudnnBatchNormMode_t mode, - void *alpha, - void *beta, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnTensorDescriptor_t yDesc, - void *y, - const cudnnTensorDescriptor_t - bnScaleBiasMeanVarDesc, - const void *bnScale, - void *bnBias, - const void *estimatedMean, - const void *estimatedVariance, - double epsilon) - - .. code-block:: cpp - - miopenStatus_t - miopenBatchNormalizationForwardInference( - miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - void *alpha, - void *beta, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenTensorDescriptor_t yDesc, - void *y, - const miopenTensorDescriptor_t - bnScaleBiasMeanVarDesc, - void *bnScale, - void *bnBias, - void *estimatedMean, - void *estimatedVariance, - double epsilon) - - * - - .. code-block:: cpp - - cudnnStatus_t - cudnnBatchNormalizationBackward( - cudnnHandle_t handle, - cudnnBatchNormMode_t mode, - const void *alphaDataDiff, - const void *betaDataDiff, - const void *alphaParamDiff, - const void *betaParamDiff, - const cudnnTensorDescriptor_t xDesc, - const void *x, - const cudnnTensorDescriptor_t dyDesc, - const void *dy, - const cudnnTensorDescriptor_t dxDesc, - void *dx, - const cudnnTensorDescriptor_t - bnScaleBiasDiffDesc, - const void *bnScale, - void *resultBnScaleDiff, - void *resultBnBiasDiff, - double epsilon, - const void *savedMean, - const void *savedInvVariance) - - .. code-block:: cpp - - miopenStatus_t - miopenBatchNormalizationBackward( - miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - const void *alphaDataDiff, - const void *betaDataDiff, - const void *alphaParamDiff, - const void *betaParamDiff, - const miopenTensorDescriptor_t xDesc, - const void *x, - const miopenTensorDescriptor_t dyDesc, - const void *dy, - const miopenTensorDescriptor_t dxDesc, - void *dx, - const miopenTensorDescriptor_t - bnScaleBiasDiffDesc, - const void *bnScale, - void *resultBnScaleDiff, - void *resultBnBiasDiff, - double epsilon, - const void *savedMean, - const void *savedInvVariance) diff --git a/docs/how-to/find-and-immediate.rst b/docs/how-to/find-and-immediate.rst index d94eb01a55..e69de29bb2 100644 --- a/docs/how-to/find-and-immediate.rst +++ b/docs/how-to/find-and-immediate.rst @@ -1,240 +0,0 @@ -.. meta:: - :description: Find and immediate modes - :keywords: MIOpen, ROCm, API, documentation - -*********************************************************************************** -Using the find APIs and immediate mode -*********************************************************************************** - -MIOpen contains several convolution algorithms for each stage of training or inference. Prior to -MIOpen version 2.0, you had to call find methods in order generate a set of applicable algorithms. - -Here's a typical workflow for the find stage: - -.. code:: cpp - - miopenConvolutionForwardGetWorkSpaceSize(handle, - weightTensorDesc, - inputTensorDesc, - convDesc, - outputTensorDesc, - &maxWorkSpaceSize); - - // < allocate workspace > - - - // NOTE: - // The miopenFindConvolution*() call is expensive in terms of run time and required workspace. - // Therefore, we highly recommend reserving the required algorithm and workspace so that you can - // reuse them later (within the lifetime of the same MIOpen handle object). - // With this approach, there should be no need to invoke miopenFind*() more than once per - // application lifetime. - - miopenFindConvolutionForwardAlgorithm(handle, - inputTensorDesc, - input_device_mem, - weightTensorDesc, - weight_device_mem, - convDesc, - outputTensorDesc, - output_device_mem,, - request_algo_count, - &ret_algo_count, - perf_results, - workspace_device_mem, - maxWorkSpaceSize, - 1); - - // < select fastest algorithm > - - // < free previously allocated workspace and allocate workspace required for the selected algorithm> - - miopenConvolutionForward(handle, &alpha, - inputTensorDesc, - input_device_mem, - weightTensorDesc, - weight_device_mem, - convDesc, - perf_results[0].fwd_algo, // use the fastest algo - &beta, - outputTensorDesc, - output_device_mem, - workspace_device_mem, - perf_results[0].memory); //workspace size - -The results of `find` are returned in an array of ``miopenConvAlgoPerf_t`` structs in order of -performance, with the fastest at index 0. - -This call sequence is only run once per session, as it's inherently expensive. Within the sequence, -``miopenFindConvolution*()`` is the most expensive call. ``miopenFindConvolution*()`` caches its own -results on disk so the subsequent calls during the same MIOpen session run faster. - -Internally, MIOpen's find calls compile and benchmark a set of ``solvers`` contained in -``miopenConvAlgoPerf_t``. This is performed in parallel with ``miopenConvAlgorithm_t``. You can -control the level of parallelism using an environmental variable. Refer to the debugging section, -:ref:`controlling parallel compilation ` for more information. - -Immediate mode -===================================================== - -MIOpen v2.0 introduces immediate more, which removes the requirement for -``miopenFindConvolution*()`` calls, thereby reducing runtime costs. In this mode, you can query the -MIOpen runtime for all of the supported solutions for a given convolution configuration. The sequence -of operations for immediate mode is similar to launching regular convolutions in MIOpen (i.e., through -the use of the ``miopenFindConvolution*()`` API). However, in this case, the different APIs have a lower -runtime cost. - -A typical convolution call is similar to the following sequence: - -* You construct the MIOpen handle and relevant descriptors, such as the convolution descriptor. -* With the above data structures, you call ``miopenConvolution*GetSolutionCount`` to get the - maximum number of supported solutions for the convolution descriptor. -* The obtained count is used to allocate memory for the ``miopenConvSolution_t`` structure, - introduced in MIOpen v2.0. -* You call ``miopenConvolution*GetSolution`` to populate the ``miopenConvSolution_t`` structures - allocated above. The returned list is in the order of best performance (where the first element is the - fastest). -* While the above structure returns the amount of workspace required for an algorithm, you can - inquire the amount of a workspace required for a known solution ID using - ``miopenConvolution*GetSolutionWorkspaceSize``. However, this is not a requirement (because the - structure returned by ``miopenConvolution*GetSolution`` already has this information). -* Now you can initiate the convolution operation in ``immediate`` mode by calling - ``miopenConvolution*Immediate``. This populates the output tensor descriptor with the respective - convolution result. However, the first call to ``miopenConvolution*Immediate`` may consume more - time because if the kernel isn't present in the kernel cache, it would need to be compiled. -* Optionally, you can compile the solution of choice by calling ``miopenConvolution*CompileSolution``. - This ensures that the kernel represented by the chosen solution is populated in the kernel cache, - removing the need to compile the kernel in question. - -.. code:: cpp - - miopenConvolutionForwardGetSolutionCount(handle, - weightTensorDesc, - inputTensorDesc, - convDesc, - outputTensorDesc, - &solutionCount); - - - // < allocate an array of miopenConvSolution_t of size solutionCount > - - - miopenConvolutionForwardGetSolution(handle, - weightTensorDesc, - inputTensorDesc, - convDesc, - outputTensorDesc, - solutionCount, - &actualCount, - solutions); - - // < select a solution from solutions array > - - miopenConvolutionForwardGetSolutionWorkspaceSize(handle, - weightTensorDesc, - inputTensorDesc, - convDesc, - outputTensorDesc, - selected->solution_id, - &ws_size); - - // < allocate solution workspace of size ws_size > - - - // This stage is optional. - miopenConvolutionForwardCompileSolution(handle, - weightTensorDesc, - inputTensorDesc, - convDesc, - outputTensorDesc, - selected->solution_id); - - - - miopenConvolutionForwardImmediate(handle, - weightTensor, - weight_device_mem, - inputTensorDesc, - input_device_mem, - convDesc, - outputTensorDesc, - output_device_mem, - workspace_device_mem, - ws_size, - selected->solution_id); - -Immediate mode fallback ------------------------------------------------------------------------------------------------ - -Although immediate mode is underpinned by :doc:`FindDb <../conceptual/finddb>`, it may not contain every -configuration of interest. If FindDb encounters a database miss, it has two fallback paths it can take, -depending on whether the CMake variable ``MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK`` is set to -``ON`` or ``OFF``. - -If you require the best possible performance, run the find stage at least once. - -AI-based heuristic fallback (default) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If ``MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK`` is set to ``ON`` (default), the immediate mode -behavior upon encountering a database miss is to use an AI-based heuristic to pick the optimal -solution. - -First, the applicability of the AI-based heuristic for the given configuration is checked. If the heuristic is -applicable, it feeds various parameters of the given configuration into a neural network that has been -tuned to predict the optimal solution with 90% accuracy. - -Weighted throughput index-based fallback -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When ``MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK`` is set to ``OFF``, or the AI heuristic is not -applicable for the given convolution configuration, the immediate mode behavior upon encountering -a database miss is to use a weighted throughput index-based mechanism to estimate which solution -would be optimal (based on the convolution configuration parameters). - -Limitations of immediate mode ------------------------------------------------------------------------------------------------ - -System FindDb has only been populated for these architectures: - -* gfx906 with 64 CUs -* gfx906 with 60 CUs -* gfx900 with 64 CUs -* gfx900 with 56 CUs - -If your architecture isn't listed, you must run the find API on your system (once per application) in -order to take advantage of immediate mode's more efficient behavior. - -Backend limitations ------------------------------------------------------------------------------------------------ - -OpenCL support for immediate mode via the fallback is limited to FP32 datatypes. This is because the -current release's fallback path goes through GEMM, which is serviced through MIOpenGEMM (on -OpenCL). MIOpenGEMM only contains support for FP32. - -The HIP backend uses rocBLAS as its fallback path, which contains a more robust set of datatypes. - - -Find modes -============================================================ - -MIOpen provides a set of find modes that are used to accelerate find API calls. The different -modes are set by using the ``MIOPEN_FIND_MODE`` environment variable with one of these values: - -* ``NORMAL``/``1`` (normal find): This is the full find mode call, which benchmarks all the solvers and - returns a list. -* ``FAST``/``2`` (fast find): Checks :doc:`FindDb <../conceptual/finddb>` for an entry. If there's a FindDb - hit, it uses that entry. If there's a miss, it uses the immediate mode fallback. Offers fast start-up times - at the cost of GPU performance. -* ``HYBRID``/``3`` or unset ``MIOPEN_FIND_MODE`` (hybrid find): Checks - :doc:`FindDb <../conceptual/finddb>` for an entry. If there's a FindDb hit, it uses that entry. If there's a - miss, it uses the existing find machinery. Offers slower start-up times than fast find without the GPU - performance drop. -* ``4``: This value is reserved and should not be used. -* ``DYNAMIC_HYBRID``/``5`` (dynamic hybrid find): Checks :doc:`FindDb <../conceptual/finddb>` for an - entry. If there's a FindDb hit, it uses that entry. If there's a miss, it uses the existing find machinery - (skipping non-dynamic kernels). It offers faster start-up times than hybrid find, but GPU performance - may decrease. - -The default find mode is ``DYNAMIC_HYBRID``. To run the full ``NORMAL`` find mode, use -``export MIOPEN_FIND_MODE=NORMAL`` or ``export MIOPEN_FIND_MODE=1``. diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 2ade4839b1..e69de29bb2 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -1,40 +0,0 @@ - -.. meta:: - :description: MIOpen API reference library - :keywords: MIOpen, ROCm, API, documentation - -************************************************************* -API reference library -************************************************************* - -The MIOpen API library is structured as follows: - -* :doc:`Datatypes <./datatypes>` - -* Modules: - - * :doc:`Handle <../doxygen/html/group__handle>` - * :doc:`Tensor <../doxygen/html/group__tensor>` - * :doc:`Activation <../doxygen/html/group__activation>` - * :doc:`Convolution <../doxygen/html/group__convolutions>` - * :doc:`Recurrent neural networks (RNN) <../doxygen/html/group___r_n_n>` - * :doc:`Batch normalization <../doxygen/html/group__batchnorm>` - * :doc:`Local response normalization (LRN) <../doxygen/html/group___l_r_n>` - * :doc:`Pooling <../doxygen/html/group__pooling>` - * :doc:`Softmax <../doxygen/html/group__softmax>` - * :doc:`Fusion <../doxygen/html/group___f_u_s_i_o_n>` - * :doc:`Loss function <../doxygen/html/group___loss_function>` - * :doc:`Dropout <../doxygen/html/group__dropout>` - * :doc:`Reduction <../doxygen/html/group___tensor_reduce>` - * :doc:`Find <../doxygen/html/group__find2>` - * :doc:`Layernorm <../doxygen/html/group__layernorm>` (experimental) - * :doc:`Sum <../doxygen/html/group__sum>` (experimental) - * :doc:`GroupNorm <../doxygen/html/group__groupnorm>` (experimental) - * :doc:`Cat <../doxygen/html/group__cat>` (experimental) - * :doc:`SGD <../doxygen/html/group___s_g_d>` (experimental) - * :doc:`ReduceExtreme <../doxygen/html/group__ReduceExtreme>` (experimental) - * :doc:`Getitem <../doxygen/html/group__getitem>` (experimental) - * :doc:`ReduceCalculation <../doxygen/html/group__ReduceCalculation>` (experimental) - * :doc:`RotaryPositionalEmbeddings <../doxygen/html/group__RotaryPositionalEmbeddings>` (experimental) - * :doc:`ReLU <../doxygen/html/group___re_l_u>` (experimental) - * :doc:`GLU <../doxygen/html/group__glu>` (experimental) diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in index aa0042a51d..e69de29bb2 100644 --- a/docs/sphinx/requirements.in +++ b/docs/sphinx/requirements.in @@ -1 +0,0 @@ -rocm-docs-core[api_reference]==1.8.2 diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt index f43d2705a6..e69de29bb2 100644 --- a/docs/sphinx/requirements.txt +++ b/docs/sphinx/requirements.txt @@ -1,147 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -accessible-pygments==0.0.5 - # via pydata-sphinx-theme -alabaster==0.7.16 - # via sphinx -babel==2.15.0 - # via - # pydata-sphinx-theme - # sphinx -beautifulsoup4==4.12.3 - # via pydata-sphinx-theme -breathe==4.35.0 - # via rocm-docs-core -certifi==2024.7.4 - # via requests -cffi==1.16.0 - # via - # cryptography - # pynacl -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via sphinx-external-toc -cryptography==43.0.1 - # via pyjwt -deprecated==1.2.14 - # via pygithub -docutils==0.21.2 - # via - # breathe - # myst-parser - # pydata-sphinx-theme - # sphinx -fastjsonschema==2.19.1 - # via rocm-docs-core -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via rocm-docs-core -idna==3.7 - # via requests -imagesize==1.4.1 - # via sphinx -jinja2==3.1.4 - # via - # myst-parser - # sphinx -markdown-it-py==3.0.0 - # via - # mdit-py-plugins - # myst-parser -markupsafe==2.1.5 - # via jinja2 -mdit-py-plugins==0.4.1 - # via myst-parser -mdurl==0.1.2 - # via markdown-it-py -myst-parser==3.0.1 - # via rocm-docs-core -packaging==24.0 - # via - # pydata-sphinx-theme - # sphinx -pycparser==2.22 - # via cffi -pydata-sphinx-theme==0.15.2 - # via - # rocm-docs-core - # sphinx-book-theme -pygithub==2.3.0 - # via rocm-docs-core -pygments==2.18.0 - # via - # accessible-pygments - # pydata-sphinx-theme - # sphinx -pyjwt[crypto]==2.8.0 - # via pygithub -pynacl==1.5.0 - # via pygithub -pyyaml==6.0.1 - # via - # myst-parser - # rocm-docs-core - # sphinx-external-toc -requests==2.32.2 - # via - # pygithub - # sphinx -rocm-docs-core[api-reference]==1.8.2 - # via -r requirements.in -smmap==5.0.1 - # via gitdb -snowballstemmer==2.2.0 - # via sphinx -soupsieve==2.5 - # via beautifulsoup4 -sphinx==7.3.7 - # via - # breathe - # myst-parser - # pydata-sphinx-theme - # rocm-docs-core - # sphinx-book-theme - # sphinx-copybutton - # sphinx-design - # sphinx-external-toc - # sphinx-notfound-page -sphinx-book-theme==1.1.2 - # via rocm-docs-core -sphinx-copybutton==0.5.2 - # via rocm-docs-core -sphinx-design==0.5.0 - # via rocm-docs-core -sphinx-external-toc==1.0.1 - # via rocm-docs-core -sphinx-notfound-page==1.0.1 - # via rocm-docs-core -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -tomli==2.0.1 - # via sphinx -typing-extensions==4.11.0 - # via - # pydata-sphinx-theme - # pygithub -urllib3==2.2.2 - # via - # pygithub - # requests -wrapt==1.16.0 - # via deprecated diff --git a/driver/CMakeLists.txt b/driver/CMakeLists.txt index 256901aa94..e69de29bb2 100644 --- a/driver/CMakeLists.txt +++ b/driver/CMakeLists.txt @@ -1,84 +0,0 @@ -################################################################################ -# -# MIT License -# -# Copyright (c) 2017 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -find_package(Threads REQUIRED) - -add_executable(MIOpenDriver - InputFlags.cpp - conv_common.cpp - dm_activ.cpp - dm_adam.cpp - dm_addlayernorm.cpp - dm_bnorm.cpp - dm_cat.cpp - dm_conv.cpp - dm_convbfp16.cpp - dm_convbfp8.cpp - dm_convfp16.cpp - dm_convfp8.cpp - dm_convint8.cpp - dm_dropout.cpp - dm_fusion.cpp - dm_gemm.cpp - dm_getitem.cpp - dm_glu.cpp - dm_groupnorm.cpp - dm_layernorm.cpp - dm_lrn.cpp - dm_pool.cpp - dm_prelu.cpp - dm_reduce.cpp - dm_reduceextreme.cpp - dm_reducecalculation.cpp - dm_rnn.cpp - dm_rope.cpp - dm_softmax.cpp - dm_t5layernorm.cpp - dm_tensorop.cpp - dm_transformers_adam_w.cpp - main.cpp - registry_driver_maker.cpp - rocrand_wrapper.cpp) -if(WIN32) - # Refer to https://en.cppreference.com/w/cpp/language/types for details. - target_compile_options(MIOpenDriver PRIVATE $:-U__LP64__>>) -endif() -add_dependencies(MIOpenDriver generate_kernels) -target_include_directories(MIOpenDriver PRIVATE ../src/kernels) -target_link_libraries(MIOpenDriver MIOpen Threads::Threads roc::rocrand) -if(NOT MIOPEN_EMBED_DB STREQUAL "") -target_link_libraries(MIOpenDriver $ ) -endif() -# Cmake does not add flags correctly for gcc -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set_target_properties(MIOpenDriver PROPERTIES COMPILE_FLAGS -pthread LINK_FLAGS -pthread) -endif() - -if( NOT ENABLE_ASAN_PACKAGING ) - install(TARGETS MIOpenDriver - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE - DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif() diff --git a/driver/conv_driver.hpp b/driver/conv_driver.hpp index 900f52f683..e69de29bb2 100644 --- a/driver/conv_driver.hpp +++ b/driver/conv_driver.hpp @@ -1,3760 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_CONV_DRIVER_HPP -#define GUARD_MIOPEN_CONV_DRIVER_HPP - -#include "InputFlags.hpp" -#include "conv_verify.hpp" -#include "conv_common.hpp" -#include "driver.hpp" -#include "mloConvHost.hpp" -#include "random.hpp" -#include "rocrand_wrapper.hpp" -#include "tensor_driver.hpp" -#include "timer.hpp" -#include "util_driver.hpp" -#include "util_file.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include <../test/cpu_bias.hpp> -#include <../test/cpu_conv.hpp> -#include <../test/serialize.hpp> -#include <../test/tensor_holder.hpp> -#include <../test/verify.hpp> - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Declare hidden function for MIGraphX to smoke test it. -extern "C" MIOPEN_EXPORT miopenStatus_t -miopenHiddenSetConvolutionFindMode(miopenConvolutionDescriptor_t convDesc, int findMode); - -#define WORKAROUND_ISSUE_2176 1 // https://github.com/AMDComputeLibraries/MLOpen/issues/2176 - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DRIVER_PAD_BUFFERS_2M) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DRIVER_USE_GPU_REFERENCE) -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DRIVER_SUBNORM_PERCENTAGE) - -// 0 - Allocate WS size as reported by the library (default) -// 1 - Do not allocate workspace. -// 2...16 - Allocate smaller WS. Size = default/value. -// Other - The driver allocates workspace size equal to the value of the variable (in bytes). -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DRIVER_CONV_WORKSPACE_SIZE_ADJUST) - -// Support in the library discontinued, but left in the driver -// for reference in the future. -#define miopenInt8x4 (static_cast(4)) - -struct AutoMiopenWarmupMode -{ - AutoMiopenWarmupMode() - { - debug_logging_quiet_prev = miopen::debug::LoggingQuiet; - debug_find_enforce_disable_prev = miopen::debug::FindEnforceDisable; - debug_is_warmup_ongoing_prev = miopen::debug::IsWarmupOngoing; - miopen::debug::LoggingQuiet = true; - miopen::debug::FindEnforceDisable = true; - miopen::debug::IsWarmupOngoing = true; - } - AutoMiopenWarmupMode(const AutoMiopenWarmupMode&) = delete; - AutoMiopenWarmupMode(AutoMiopenWarmupMode&&) = delete; - AutoMiopenWarmupMode& operator=(const AutoMiopenWarmupMode&) = delete; - AutoMiopenWarmupMode& operator=(AutoMiopenWarmupMode&&) = delete; - ~AutoMiopenWarmupMode() - { - miopen::debug::LoggingQuiet = debug_logging_quiet_prev; - miopen::debug::FindEnforceDisable = debug_find_enforce_disable_prev; - miopen::debug::IsWarmupOngoing = debug_is_warmup_ongoing_prev; - } - -private: - bool debug_logging_quiet_prev; - bool debug_find_enforce_disable_prev; - bool debug_is_warmup_ongoing_prev; -}; - -struct AutoPrepareForGpuReference -{ - AutoPrepareForGpuReference() - { - quiet_prev = miopen::debug::LoggingQuiet; - naive_prev = miopen::debug::AlwaysEnableConvDirectNaive; - miopen::debug::AlwaysEnableConvDirectNaive = true; - miopen::debug::LoggingQuiet = true; - } - AutoPrepareForGpuReference(const AutoPrepareForGpuReference&) = delete; - AutoPrepareForGpuReference(AutoPrepareForGpuReference&&) = delete; - AutoPrepareForGpuReference& operator=(const AutoPrepareForGpuReference&) = delete; - AutoPrepareForGpuReference& operator=(AutoPrepareForGpuReference&&) = delete; - ~AutoPrepareForGpuReference() - { - miopen::debug::LoggingQuiet = quiet_prev; - miopen::debug::AlwaysEnableConvDirectNaive = naive_prev; - } - -private: - bool naive_prev; - bool quiet_prev; -}; - -static inline void AdjustWorkspacesizeVariableFromEnv(std::size_t& sz) -{ - auto adj = env::value(MIOPEN_DRIVER_CONV_WORKSPACE_SIZE_ADJUST); - if(adj == 0ULL) - return; // nop - auto sz_save = sz; - if(adj == 1ULL) - sz = 0ULL; - else if(1 <= adj && adj <= 16) - sz /= adj; - else - sz = adj; - MIOPEN_LOG_CUSTOM( - miopen::LoggingLevel::Info2, "MIOpenDriver", "From " << sz_save << " to " << sz); - return; -} - -static inline miopenDataType_t DataTypeFromShortString(const std::string& type) -{ - static const std::unordered_map conv_map = { - {"fp32", miopenFloat}, - {"fp16", miopenHalf}, - {"bf16", miopenBFloat16}, - {"fp8", miopenFloat8}, - {"bf8", miopenBFloat8}}; - - const auto res = conv_map.find(type); - if(res != conv_map.end()) - { - return res->second; - } - else - { - MIOPEN_THROW("Invalid compute/cast type short hand supplied"); - } -} - -template -class GpumemTensor -{ - std::unique_ptr dev; - tensor host; - bool is_gpualloc = false; - -public: - void SetGpuallocMode(bool v) { is_gpualloc = v; } - tensor& GetTensor() { return host; } - - void AllocOnHost(miopenTensorDescriptor_t t) - { - host = tensor(miopen::deref(t)); - if(is_gpualloc) // We do not need host data. - { - host.data.clear(); - host.data.shrink_to_fit(); // To free host memory. - } - } - - std::vector& GetVector() - { - if(is_gpualloc) - MIOPEN_THROW("[MIOpenDriver] GpumemTensor::GetVector should not be called in " - "'--gpualloc 1' mode"); - return host.data; - } - - Tgpu* GetVectorData() { return is_gpualloc ? nullptr : host.data.data(); } - std::size_t GetVectorSize() const { return is_gpualloc ? 0 : host.data.size(); } - - void - InitHostData(const size_t sz, // - const bool do_write, // If set to false, then only generate random data. This is - // necessary to reproduce values in input buffers even if some - // directions are skipped. For example, inputs for Backward - // will be the same for both "-F 0" and "-F 2". - std::function generator) - { - if(is_gpualloc) - { - /// In gpualloc mode, we do not care about reproducibility of results, because - /// validation is not used. Therefore, we do not have to always generate random value - /// (\ref move_rand) - return; - } - - for(size_t i = 0; i < sz; ++i) - { - /// \anchor move_rand - /// Generate random value, even if buffer is unused. This provides the same - /// initialization of input buffers regardless of which kinds of - /// convolutions are currently selectedfor testing (see the "-F" option). - /// Verification cache would be broken otherwise. - auto val = generator(); - if(do_write) - GetVector()[i] = val; - } - } - - status_t AllocOnDevice(stream, context_t ctx, const size_t sz) - { - dev = std::make_unique(ctx, sz, sizeof(Tgpu)); - return STATUS_SUCCESS; - } - - status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz) - { - AllocOnDevice(q, ctx, sz); - if(is_gpualloc) - { - /// \anchor gpualloc_random_init - /// In gpualloc mode, we do not want to leave input buffers uninitialized, because - /// there could be NaNs and Infs, which may affect the performance (which we are - /// interested to evaluate in this mode). Initialization with all 0's is not the - /// best choice as well, because GPU HW may optimize out computations with 0's and - /// that could affect performance of kernels too. That is why we are using - /// rocrand to initialize input buffers. - /// - /// However we do not care about precision in gpualloc mode, because validation - /// is not used. Therefore, range (0,1] is fine. - return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); - } - return dev->ToGPU(q, GetVectorData()); - } - - template - status_t AllocOnDevice(stream, context_t ctx, const size_t sz, std::vector&) - { - static_assert(std::is_same::value // - || std::is_same::value, // - "Before enabling more types, check thoroughly."); - dev = std::make_unique(ctx, sz, sizeof(T)); - return STATUS_SUCCESS; - } - - template - status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz, std::vector& init) - { - AllocOnDevice(q, ctx, sz, init); - if(is_gpualloc) - { - /// \ref gpualloc_random_init - return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); - } - return dev->ToGPU(q, init.data()); - } - - status_t CopyFromDeviceToHost(stream q) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, GetVectorData()); - } - - template - status_t CopyFromDeviceToHost(stream q, tensor& t) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, t.data.data()); - } - - template - status_t CopyFromDeviceToHost(stream q, std::vector& v) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, v.data()); - } - - auto GetDevicePtr() -> auto { return dev->GetMem(); } -}; - -template -class GpumemVector -{ - std::unique_ptr dev; - std::vector host; - bool is_gpualloc = false; - -public: - void SetGpuallocMode(bool v) { is_gpualloc = v; } - void AllocOnHost(std::size_t sz) - { - if(is_gpualloc) // We do not need host data. - return; - host = std::vector(sz, static_cast(0)); - } - std::vector& GetVector() - { - if(is_gpualloc) - MIOPEN_THROW("[MIOpenDriver] GpumemVector::GetVector should not be called in " - "'--gpualloc 1' mode"); - return host; - } - - Tgpu* GetVectorData() { return is_gpualloc ? nullptr : host.data(); } - std::size_t GetVectorSize() const { return is_gpualloc ? 0 : host.size(); } - - status_t AllocOnDevice(stream, context_t ctx, const size_t sz) - { - dev = std::make_unique(ctx, sz, sizeof(Tgpu)); - return STATUS_SUCCESS; - } - - status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz) - { - AllocOnDevice(q, ctx, sz); - if(is_gpualloc) - { - /// \ref gpumem_random_init - return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); - } - return dev->ToGPU(q, GetVectorData()); - } - - status_t CopyFromDeviceToHost(stream q) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, GetVectorData()); - } - - template - status_t CopyFromDeviceToHost(stream q, tensor& t) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, t.data.data()); - } - - auto GetDevicePtr() -> auto { return dev->GetMem(); } -}; - -// Tgpu and Tref are the data-type in GPU memory and CPU memory respectively. -// They are not necessarily the same as the computation type on GPU or CPU -template -class ConvDriver : public Driver -{ -public: - ConvDriver() : Driver() - { - miopenCreateTensorDescriptor(&inputTensor); - miopenCreateTensorDescriptor(&weightTensor); - miopenCreateTensorDescriptor(&outputTensor); - miopenCreateTensorDescriptor(&biasTensor); - miopenCreateTensorDescriptor(&inputTensor_vect4); - miopenCreateTensorDescriptor(&weightTensor_vect4); - miopenCreateConvolutionDescriptor(&convDesc); - - { - AutoMiopenWarmupMode warmupMode; - miopenCreateTensorDescriptor(&warmupInputTensor); - miopenCreateTensorDescriptor(&warmupWeightTensor); - miopenCreateTensorDescriptor(&warmupOutputTensor); - miopenCreateConvolutionDescriptor(&warmupConvDesc); - } - - workspace_dev = nullptr; - // the variable name is implementation dependent, checking size instead - InitDataType(); - } - - int AddCmdLineArgs() override; - int ParseCmdLineArgs(int argc, char* argv[]) override; - InputFlags& GetInputFlags() override { return inflags; } - - // function to validate the Layout type parameters. - // Layout types are -In,Out,Fil etc.This function validates the - // layout parameter value to std (NCHW/NHWC/NCDHW/NDHWC) values, - // defined in MIOpen lib. - // layout_type - input value supplied with MIOpen driver command. - void ValidateLayoutInputParameters(std::string layout_type); - void ValidateVectorizedParameters(int vector_dim, int vector_length); - - // Helper function to check the Layout type short names - // Short names are defined as I,O,f. W.r.t In/Out/fil layout - // types. - int ChkLayout_ShortName(); - - int GetandSetData() override; - bool TensorsCasted() const; - std::vector GetInputTensorLengthsFromCmdLine(); - std::vector GetWeightTensorLengthsFromCmdLine(); - std::vector GetBiasTensorLengthsFromCmdLine(); - - int SetConvDescriptorFromCmdLineArgs(); - - std::vector GetOutputTensorLengths(); - - int AllocateBuffersAndCopy() override; - - bool UseGPUReference(); - - int FindForward(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx); - int RunForwardGPU() override; - int RunForwardCPU(); - int RunForwardGPUReference(); - int RunWarmupFindForwardGPU(); - - int FindBackwardData(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx); - int FindBackwardWeights(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx); - int RunBackwardGPU() override; - int RunBackwardDataCPU(); - int RunBackwardWeightsCPU(); - int RunBackwardBiasCPU(); - int RunBackwardDataGPUReference(); - int RunBackwardWeightsGPUReference(); - // int RunBackwardBiasGPUReference(); - - int VerifyBackward() override; - int VerifyForward() override; - ~ConvDriver() override - { - miopenDestroyTensorDescriptor(biasTensor); - miopenDestroyTensorDescriptor(outputTensor); - miopenDestroyTensorDescriptor(weightTensor); - miopenDestroyTensorDescriptor(inputTensor); - miopenDestroyTensorDescriptor(inputTensor_vect4); - miopenDestroyTensorDescriptor(weightTensor_vect4); - miopenDestroyConvolutionDescriptor(convDesc); - - miopenDestroyTensorDescriptor(warmupInputTensor); - miopenDestroyTensorDescriptor(warmupWeightTensor); - miopenDestroyTensorDescriptor(warmupOutputTensor); - miopenDestroyConvolutionDescriptor(warmupConvDesc); - } - -private: - const miopenDataType_t warmup_data_type = miopenFloat; - typedef float warmup_Tgpu; - - InputFlags inflags; - - boost::optional immediate_solution; - - GpumemTensor in; - GpumemVector din; - GpumemTensor wei; - GpumemVector dwei; - GpumemTensor out; - GpumemTensor dout; - GpumemTensor b; - GpumemVector db; - GpumemTensor warmup_in; - GpumemTensor warmup_wei; - GpumemTensor warmup_out; - - miopenTensorDescriptor_t inputTensor; - miopenTensorDescriptor_t weightTensor; - miopenTensorDescriptor_t outputTensor; - miopenTensorDescriptor_t biasTensor; - miopenTensorDescriptor_t inputTensor_vect4; - miopenTensorDescriptor_t weightTensor_vect4; - miopenTensorDescriptor_t warmupInputTensor; - miopenTensorDescriptor_t warmupWeightTensor; - miopenTensorDescriptor_t warmupOutputTensor; - - std::unique_ptr in_vect4_dev; - std::unique_ptr wei_vect4_dev; - - std::unique_ptr workspace_dev; - std::size_t ws_sizeof_find_fwd; - std::size_t ws_sizeof_find_bwd; - std::size_t ws_sizeof_find_wrw; - std::size_t warmup_ws_sizeof_find; - - tensor outhost; - tensor dwei_host; - tensor din_host; - tensor db_host; - - std::vector out_int8; - std::vector b_int8; - - miopenConvolutionDescriptor_t convDesc; - miopenConvolutionDescriptor_t warmupConvDesc; - miopenConvolutionMode_t mode; - - bool is_wrw = true, is_bwd = true, is_fwd = true; - bool is_wrw_winograd = false; - bool is_wrw_igemm = false; - bool is_fwd_igemm = false; - bool is_bwd_igemm = false; - bool time_enabled = false; - bool wall_enabled = false; - bool warmup_enabled = false; - bool is_gpualloc = false; - int num_iterations = 1; - - // Used to avoid wasting time for verification after failure of Run*GPU(). - // We can't properly control this from the main() level. - // RunBackwardGPU() and VerifyBackward() do the job for both Bwd and WrW. - // If RunBackwardGPU() fails, then main() doesn't know if Bwd or WrW has failed. - // Also main() has no ways to for controlling how Verify works except skipping the whole call. - bool is_fwd_run_failed = false, is_bwd_run_failed = false, is_wrw_run_failed = false; - - Timer wall; - Timer2 fwd_auxiliary; - Timer2 bwd_auxiliary; - Timer2 wrw_auxiliary; - Timer2 fwd_auxiliary_gwss; - Timer2 bwd_auxiliary_gwss; - Timer2 wrw_auxiliary_gwss; - Timer2 warmup_wall_total; // Counts also auxiliary time. - - float ComputeAverageTime(const float total_time, const float first_time) const - { - if(num_iterations > 1) - return (total_time - first_time) / (num_iterations - 1); - return total_time; - } - - void PrintForwardTime(float kernel_total_time, float kernel_first_time) const; - int RunForwardGpuImmed(bool is_transform); - int RunForwardGpuFind(bool is_transform); - void PrintBackwardDataTime(float kernel_total_time, float kernel_first_time); - int RunBackwardDataGpuImmed(); - int RunBackwardDataGpuFind(); - void PrintBackwardWrwTime(float kernel_total_time, float kernel_first_time); - int RunBackwardWrwGpuImmed(); - int RunBackwardWrwGpuFind(); - - double GetDefaultTolerance() const - { - // Computation error of fp16 is ~2^13 (=8192) bigger than - // the one of fp32 because mantissa is shorter by 13 bits. - auto tolerance = (sizeof(Tgpu) == 4 || sizeof(Tgpu) == 1) ? 1.5e-6 : 8.2e-3; - - // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. - if(std::is_same::value) - tolerance *= 8.0; - constexpr bool is_fp8 = std::is_same::value; - constexpr bool is_bfp8 = std::is_same::value; - if(is_bfp8 || is_fp8 || TensorsCasted()) - tolerance *= 37.0; - return tolerance; - } - - enum class Direction - { - Fwd, - Bwd, - WrW, - BwdBias - }; - - std::string GetVerificationCacheFileName(const Direction& direction) const; - bool IsInputTensorTransform() const; - - bool TryReadVerificationCache(const Direction& direction, - miopenTensorDescriptor_t& tensorDesc, - Tref* data) const; - void TrySaveVerificationCache(const Direction& direction, std::vector& data) const; - - void DebugPrintWorkspaceDev() const - { - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Info2, - "MIOpenDriver", - "ptr=" << (workspace_dev != nullptr ? workspace_dev->GetMem() : nullptr) - << " size=" - << (workspace_dev != nullptr ? workspace_dev->GetSize() : 0ULL)); - } - - void ResizeWorkspaceDev(context_t ctx, std::size_t size) - { - workspace_dev.reset(); - if(size > 0) - workspace_dev = std::unique_ptr(new GPUMem(ctx, size, 1)); - DebugPrintWorkspaceDev(); - } - - // Helper functions, can be moved out of class. - void PrintImmedSolutionInfo(const miopenConvSolution_t& s) const - { - std::cout << "- id: " << s.solution_id << " algo: " << s.algorithm << ", time: " << s.time - << " ms, ws: " << s.workspace_size - << ", name: " << miopen::solver::Id(s.solution_id).ToString() << std::endl; - } - - std::string AlgorithmSolutionToString(const miopenConvSolution_t& s) const - { - std::ostringstream oss; - oss << "Algorithm: " << s.algorithm << ", Solution: " << s.solution_id << '/' - << ((s.solution_id != 0) ? miopen::solver::Id(s.solution_id).ToString() - : std::string("UNKNOWN")); - return oss.str(); - } - - /// Find() updates find-db with the most recent information (unless find-db is disabled). - /// Therefore, after Find(), Immediate mode returns the "best" found solution - /// as the 1st solution in the list, and we can use Immediate mode to find out - /// the name of the Solver selected during Find() and then used in Run(). - void GetSolutionAfterFind(const miopenConvAlgoPerf_t& found, - const Direction& direction, - const miopenTensorDescriptor_t& inTensor, - const miopenTensorDescriptor_t& weiTensor, - const miopenTensorDescriptor_t& outTensor, - miopenConvSolution_t& solution); -}; - -// Check if int8 type tensor x and w need to be transformed to a pack of 4 elements along channel -// (NCHW_VECT_C format) -template -bool ConvDriver::IsInputTensorTransform() const -{ - return (inflags.GetValueInt("tensor_vect") == 1 && data_type == miopenInt8 && - inflags.GetValueInt("in_channels") % 4 != 0) || - data_type == miopenInt8x4; -} - -template -int ConvDriver::ParseCmdLineArgs(int argc, char* argv[]) -{ - - inflags.Parse(argc, argv); - - // try to set a default layout value for 3d conv if not specified from cmd line - int spatial_dim = inflags.GetValueInt("spatial_dim"); - - const std::string default_layout = (spatial_dim == 2) ? "NCHW" : "NCDHW"; - - // inflags value is empty, default value is used - // if it is supplied via cmd line, check the value. - if(inflags.GetValueStr("in_layout").empty()) - { - inflags.SetValue("in_layout", default_layout); - } - else - { - std::string in_layoutValue = inflags.GetValueStr("in_layout"); - ValidateLayoutInputParameters(in_layoutValue); - inflags.SetValue("in_layout", in_layoutValue); - } - // fil layout argument value check - if(inflags.GetValueStr("fil_layout").empty()) - { - inflags.SetValue("fil_layout", default_layout); - } - else - { - std::string fil_layoutValue = inflags.GetValueStr("fil_layout"); - ValidateLayoutInputParameters(fil_layoutValue); - inflags.SetValue("fil_layout", fil_layoutValue); - } - // out layout argument check - if(inflags.GetValueStr("out_layout").empty()) - { - inflags.SetValue("out_layout", default_layout); - } - else - { - std::string out_layoutValue = inflags.GetValueStr("out_layout"); - ValidateLayoutInputParameters(out_layoutValue); - inflags.SetValue("out_layout", out_layoutValue); - } - - // vectorized tensor Dimension & Length check - int vector_dim = inflags.GetValueInt("tensor_vect"); - int vector_length = inflags.GetValueInt("vector_length"); - - ValidateVectorizedParameters(vector_dim, vector_length); - if(vector_length != 1 && vector_dim == 1) - { - inflags.SetValue("in_layout", - inflags.GetValueStr("in_layout") + "c" + std::to_string(vector_length)); - inflags.SetValue("fil_layout", - inflags.GetValueStr("fil_layout") + "c" + std::to_string(vector_length)); - inflags.SetValue("out_layout", - inflags.GetValueStr("out_layout") + "c" + std::to_string(vector_length)); - } - - if(inflags.GetValueStr("mode") == "conv") - { - mode = miopenConvolution; - } - else if(inflags.GetValueStr("mode") == "trans") - { - mode = miopenTranspose; - } - else - { - std::cout << "Incorrect Convolution Mode: '" << inflags.GetValueStr("mode") << '\'' - << std::endl; - return 1; - } - - num_iterations = inflags.GetValueInt("iter"); - if(num_iterations < 1) - { - std::cout << "Fatal: Number of iterations must be > 0: " << num_iterations << std::endl; - return 1; - } - time_enabled = (inflags.GetValueInt("time") != 0); - { - const int val = inflags.GetValueInt("wall"); - if(val >= 1) - { - if(!time_enabled) - { - std::cout << "Info: '--wall " << val << "' is ignored because '--time' is not set" - << std::endl; - } - else - { - wall_enabled = (val >= 1); - warmup_enabled = (val >= 2); - } - } - } - - if(time_enabled) - { - miopenEnableProfiling(GetHandle(), true); - } - - is_fwd = (inflags.GetValueInt("forw") == 0 || inflags.GetValueInt("forw") & 1); - is_bwd = (inflags.GetValueInt("forw") == 0 || inflags.GetValueInt("forw") & 2); - is_wrw = (inflags.GetValueInt("forw") == 0 || inflags.GetValueInt("forw") & 4); - - const auto solution_str = inflags.GetValueStr("solution"); - auto solution_value = static_cast(miopen::solver::Id(solution_str.c_str()).Value()); - if(solution_value == 0) // Assume number on input - { - solution_value = std::strtol(solution_str.c_str(), nullptr, 10); - if(errno == ERANGE) - { - errno = 0; - solution_value = 0; - } - } - if(solution_value >= 0) - immediate_solution = solution_value; - - const std::set valid_cast_types = {"fp32", "fp16", "bf16", "fp8", "bf8"}; - if(inflags.GetValueStr("in_cast_type") != "-1") - { - const auto in_cast_type = inflags.GetValueStr("in_cast_type"); - if(valid_cast_types.find(in_cast_type) == valid_cast_types.end()) - { - std::cout << "Invalid value for in_cast_type argument:" << in_cast_type << std::endl; - return 1; - } - } - if(inflags.GetValueStr("wei_cast_type") != "-1") - { - const auto wei_cast_type = inflags.GetValueStr("wei_cast_type"); - if(valid_cast_types.find(wei_cast_type) == valid_cast_types.end()) - { - std::cout << "Invalid value for wei_cast_type argument:" << wei_cast_type << std::endl; - return 1; - } - } - if(inflags.GetValueStr("out_cast_type") != "-1") - { - const auto out_cast_type = inflags.GetValueStr("out_cast_type"); - if(valid_cast_types.find(out_cast_type) == valid_cast_types.end()) - { - std::cout << "Invalid value for out_cast_type argument:" << out_cast_type << std::endl; - return 1; - } - } - - is_gpualloc = (inflags.GetValueInt("gpualloc") == 1); - - if(is_gpualloc && inflags.GetValueInt("verify") == 1) - { - std::cerr << "Error: '--gpualloc 1' should not be used with enabled verification. Add " - "'--verify 0' to options." - << std::endl; - exit(EXIT_FAILURE); - } - - in.SetGpuallocMode(is_gpualloc); - din.SetGpuallocMode(is_gpualloc); - wei.SetGpuallocMode(is_gpualloc); - dwei.SetGpuallocMode(is_gpualloc); - out.SetGpuallocMode(is_gpualloc); - dout.SetGpuallocMode(is_gpualloc); - b.SetGpuallocMode(is_gpualloc); - db.SetGpuallocMode(is_gpualloc); - warmup_in.SetGpuallocMode(is_gpualloc); - warmup_wei.SetGpuallocMode(is_gpualloc); - warmup_out.SetGpuallocMode(is_gpualloc); - - return 0; -} - -template -void ConvDriver::ValidateLayoutInputParameters(std::string layout_value) -{ - if((ChkLayout_ShortName())) - { - std::cerr << " Invalid Layout Short Name = " << ChkLayout_ShortName() << std::endl; - exit(EXIT_FAILURE); - } - else - { - if((layout_value.compare("NCHW") == 0) || (layout_value.compare("NHWC") == 0) || - (layout_value.compare("CHWN") == 0) || (layout_value.compare("NCDHW") == 0) || - (layout_value.compare("NDHWC") == 0)) - { - // do nothing,Values are matching as defined in Lib. - } - else - { - std::cerr << "Invalid Layout Parameter Value - " << layout_value << std::endl; - exit(EXIT_FAILURE); - } - } -} - -template -void ConvDriver::ValidateVectorizedParameters(int vector_dim, int vector_length) -{ - if(((vector_length == 4 || vector_length == 8) && vector_dim == 1) || - (vector_length == 1 && vector_dim == 0)) - { - // do nothing,Values are matching as defined in Lib. - } - else - { - std::cerr << "Invalid Tensor Vectorization Parameter Value - " - << "vector_dim:" << vector_dim << ", vector_length:" << vector_length - << std::endl; - exit(EXIT_FAILURE); - } -} - -template -int ConvDriver::ChkLayout_ShortName() -{ - // check for short name of layout type - if((inflags.FindShortName("in_layout") == 'I') && - (inflags.FindShortName("out_layout") == 'O') && (inflags.FindShortName("fil_layout") == 'f')) - { - // do noting - // found valid short names - return 0; - } - else - { - std::cerr << "Error:Invalid Short Name!" << std::endl; - exit(EXIT_FAILURE); - } -} - -template -bool ConvDriver::TensorsCasted() const -{ - return inflags.GetValueStr("in_cast_type") != "-1" || - inflags.GetValueStr("wei_cast_type") != "-1" || - inflags.GetValueStr("out_cast_type") != "-1"; -} - -template -int ConvDriver::GetandSetData() -{ - std::vector in_len = GetInputTensorLengthsFromCmdLine(); - std::vector wei_len = GetWeightTensorLengthsFromCmdLine(); - - SetTensorNd(inputTensor, in_len, inflags.GetValueStr("in_layout"), data_type); - if(inflags.GetValueStr("in_cast_type") != "-1") - { - const auto in_cast_type = DataTypeFromShortString(inflags.GetValueStr("in_cast_type")); - miopenSetTensorCastType(inputTensor, in_cast_type); - } - SetTensorNd(weightTensor, wei_len, inflags.GetValueStr("fil_layout"), data_type); - if(inflags.GetValueStr("wei_cast_type") != "-1") - { - const auto in_cast_type = DataTypeFromShortString(inflags.GetValueStr("wei_cast_type")); - miopenSetTensorCastType(weightTensor, in_cast_type); - } - - if(inflags.GetValueInt("tensor_vect") == 1 && data_type == miopenInt8) - { - data_type = miopenInt8x4; - } - - if(IsInputTensorTransform()) - { - std::vector in_len_vect4(in_len.begin(), in_len.end()), - wei_len_vect4(wei_len.begin(), wei_len.end()); - in_len_vect4[1] = ((in_len[1] + 3) / 4) * 4; - SetTensorNd(inputTensor_vect4, in_len_vect4, data_type); - wei_len_vect4[1] = ((wei_len[1] + 3) / 4) * 4; - SetTensorNd(weightTensor_vect4, wei_len_vect4, data_type); - } - SetConvDescriptorFromCmdLineArgs(); - - std::vector out_len = GetOutputTensorLengths(); - if(miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorNCHWc4 || - miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorNCHWc8) - { - out_len[1] *= miopen::deref(inputTensor).GetVectorLength(); - } - if(miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorCHWNc4 || - miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorCHWNc8) - { - out_len[0] *= miopen::deref(inputTensor).GetVectorLength(); - } - miopenDataType_t y_type = - (data_type == miopenInt8 || data_type == miopenInt8x4) ? miopenInt32 : data_type; - SetTensorNd(outputTensor, out_len, inflags.GetValueStr("out_layout"), y_type); - if(inflags.GetValueStr("out_cast_type") != "-1") - { - const auto out_cast_type = DataTypeFromShortString(inflags.GetValueStr("out_cast_type")); - miopenSetTensorCastType(outputTensor, out_cast_type); - } - - if(inflags.GetValueInt("bias") != 0) - { - std::vector bias_len = GetBiasTensorLengthsFromCmdLine(); - SetTensorNd(biasTensor, bias_len, data_type); - } - - if(warmup_enabled) - { - AutoMiopenWarmupMode warmupMode; - std::vector warmup_in_len = {1, 1, 16, 16}; // NCHW - std::vector warmup_wei_len = {1, 1, 1, 1}; // KCYX - SetTensorNd(warmupInputTensor, warmup_in_len, warmup_data_type); - SetTensorNd(warmupWeightTensor, warmup_wei_len, warmup_data_type); - - const int spatial_dim = 2; - const int group_count = 1; - std::vector pads = {0, 0}; - std::vector conv_strides = {1, 1}; - std::vector conv_dilations = {1, 1}; - miopenInitConvolutionNdDescriptor(warmupConvDesc, - spatial_dim, - pads.data(), - conv_strides.data(), - conv_dilations.data(), - miopenConvolution); - miopenSetConvolutionFindMode(warmupConvDesc, miopenConvolutionFindModeNormal); - miopenHiddenSetConvolutionFindMode( - warmupConvDesc, - static_cast(miopenConvolutionFindModeNormal)); // Repeat via hidden API. - miopenSetConvolutionGroupCount(warmupConvDesc, group_count); - - int warmup_out_len_size = miopen::deref(warmupInputTensor).GetNumDims(); - std::vector warmup_out_len(warmup_out_len_size); - miopenGetConvolutionNdForwardOutputDim(warmupConvDesc, - warmupInputTensor, - warmupWeightTensor, - &warmup_out_len_size, - warmup_out_len.data()); - SetTensorNd(warmupOutputTensor, warmup_out_len, warmup_data_type); - } - return (0); -} - -template -int ConvDriver::AddCmdLineArgs() -{ - - inflags.AddInputFlag("in_layout", - 'I', - "", - "Input Layout (Default=NCHW for 2d conv, NCDHW for 3d conv)", - "string", - true); - inflags.AddInputFlag("out_layout", - 'O', - "", - "Output Layout (Default=NCHW for 2d conv, NCDHW for 3d conv)", - "string", - true); - inflags.AddInputFlag("fil_layout", - 'f', - "", - "Filter Layout (Default=NCHW for 2d conv, NCDHW for 3d conv)", - "string", - true); - inflags.AddInputFlag( - "spatial_dim", '_', "2", "convolution spatial dimension (Default-2)", "int"); - inflags.AddInputFlag("forw", - 'F', - "0", - "Flag enables fwd, bwd, wrw convolutions" - "\n0 fwd+bwd+wrw (default)" - "\n1 fwd only" - "\n2 bwd only" - "\n4 wrw only" - "\n3 fwd+bwd" - "\n5 fwd+wrw" - "\n6 bwd+wrw", - "int"); - inflags.AddInputFlag("batchsize", 'n', "100", "Mini-batch size (Default=100)", "int"); - inflags.AddInputFlag("in_channels", 'c', "3", "Number of Input Channels (Default=3)", "int"); - inflags.AddInputFlag("in_d", '!', "32", "Input Depth (Default=32)", "int"); - inflags.AddInputFlag("in_h", 'H', "32", "Input Height (Default=32)", "int"); - inflags.AddInputFlag("in_w", 'W', "32", "Input Width (Default=32)", "int"); - inflags.AddInputFlag( - "out_channels", 'k', "32", "Number of Output Channels (Default=32)", "int"); - inflags.AddInputFlag("fil_d", '@', "3", "Filter Depth (Default=3)", "int"); - inflags.AddInputFlag("fil_h", 'y', "3", "Filter Height (Default=3)", "int"); - inflags.AddInputFlag("fil_w", 'x', "3", "Filter Width (Default=3)", "int"); - inflags.AddInputFlag( - "conv_stride_d", '#', "1", "Convolution Stride for Depth (Default=1)", "int"); - inflags.AddInputFlag( - "conv_stride_h", 'u', "1", "Convolution Stride for Height (Default=1)", "int"); - inflags.AddInputFlag( - "conv_stride_w", 'v', "1", "Convolution Stride for Width (Default=1)", "int"); - inflags.AddInputFlag("pad_d", '$', "0", "Zero Padding for Depth (Default=0)", "int"); - inflags.AddInputFlag("pad_h", 'p', "0", "Zero Padding for Height (Default=0)", "int"); - inflags.AddInputFlag("pad_w", 'q', "0", "Zero Padding for Width (Default=0)", "int"); - inflags.AddInputFlag("pad_val", 'r', "0", "Padding Value (Default=0)", "int"); - inflags.AddInputFlag( - "trans_output_pad_d", '%', "0", "Zero Padding Output for Depth (Default=0)", "int"); - inflags.AddInputFlag( - "trans_output_pad_h", 'Y', "0", "Zero Padding Output for Height (Default=0)", "int"); - inflags.AddInputFlag( - "trans_output_pad_w", 'X', "0", "Zero Padding Output for Width (Default=0)", "int"); - inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); - inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); - inflags.AddInputFlag("verification_cache", - 'C', - "", - "Use specified directory to cache verification data. Off by default.", - "string"); - inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); - inflags.AddInputFlag("wall", - 'w', - "0", - "Wall-clock Time Each Layer" - "\n0 Off (Default)" - "\n1 On, requires '--time 1')" - "\n2 On, warm-up the library (prefetch db caches), requires '--time 1'", - "int"); - inflags.AddInputFlag("search", 's', "0", "Search Kernel Config (Default=0)", "int"); - inflags.AddInputFlag("printconv", 'P', "1", "Print Convolution Dimensions (Default=1)", "int"); - inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); - inflags.AddInputFlag("in_data", 'd', "", "Input data filename (Default=)", "string"); - inflags.AddInputFlag("weights", 'e', "", "Input weights filename (Default=)", "string"); - inflags.AddInputFlag("bias", 'b', "", "Use Bias (Default=0)", "int"); - inflags.AddInputFlag( - "mode", 'm', "conv", "Convolution Mode (conv, trans) (Default=conv)", "str"); - - inflags.AddInputFlag( - "pad_mode", 'z', "default", "Padding Mode (same, valid, default) (Default=default)", "str"); - inflags.AddInputFlag("tensor_vect", - 'Z', - "0", - "tensor vectorization type (none, vect_c, vect_n) (Default=0)", - "int"); - inflags.AddInputFlag( - "vector_length", 'L', "1", "tensor vectorization length (Default=1)", "int"); - inflags.AddInputFlag("dilation_d", '^', "1", "Dilation of Filter Depth (Default=1)", "int"); - inflags.AddInputFlag("dilation_h", 'l', "1", "Dilation of Filter Height (Default=1)", "int"); - inflags.AddInputFlag("dilation_w", 'j', "1", "Dilation of Filter Width (Default=1)", "int"); - inflags.AddInputFlag("in_bias", 'a', "", "Input bias filename (Default=)", "string"); - inflags.AddInputFlag("group_count", 'g', "1", "Number of Groups (Default=1)", "int"); - inflags.AddInputFlag("dout_data", - 'D', - "", - "dy data filename for backward weight computation (Default=)", - "string"); - inflags.AddInputFlag("solution", - 'S', - "-1", - "Use immediate mode, run solution with specified id." - "\nAccepts integer argument N:" - "\n=0 Immediate mode, build and run fastest solution" - "\n>0 Immediate mode, build and run solution_id = N" - "\n<0 Use Find() API (Default=-1)" - "\nAlso accepts symbolic name of solution:" - "\n Immediate mode, build and run specified solution" - "\n Use Find() API", - "string"); - inflags.AddInputFlag("gpualloc", - 'G', - "0", - "Controls allocation and initialization buffers on GPU and CPU." - "\n0 Init input buffers on CPU and copy them to GPU. After convolution" - "\n is executed, copy output buffer to CPU (Default)." - "\n1 No copying. Use hipMalloc to allocate and rocrand to init buffers" - "\n directly on GPU. Verification (-V 1) won't succeed in this mode.", - "int"); - inflags.AddInputFlag( - "in_cast_type", 'U', "-1", "Cast type for input tensor, default to not set", "string"); - inflags.AddInputFlag( - "out_cast_type", 'T', "-1", "Cast type for output tensor, default to not set", "string"); - inflags.AddInputFlag( - "wei_cast_type", 'R', "-1", "Cast type for weight tensor, default to not set", "string"); - - return 0; -} - -template -std::vector ConvDriver::GetInputTensorLengthsFromCmdLine() -{ - std::vector in_lens; - - int spatial_dim = inflags.GetValueInt("spatial_dim"); - in_lens.resize(2 + spatial_dim); - - in_lens[0] = inflags.GetValueInt("batchsize"); - in_lens[1] = inflags.GetValueInt("in_channels"); - - auto in_spatial_lens = boost::adaptors::slice(in_lens, 2, 2 + spatial_dim); - - if(spatial_dim == 2) - { - in_spatial_lens[0] = inflags.GetValueInt("in_h"); - in_spatial_lens[1] = inflags.GetValueInt("in_w"); - } - else if(spatial_dim == 3) - { - in_spatial_lens[0] = inflags.GetValueInt("in_d"); - in_spatial_lens[1] = inflags.GetValueInt("in_h"); - in_spatial_lens[2] = inflags.GetValueInt("in_w"); - } - else - { - MIOPEN_THROW("unsupported convolution dimension"); - } - - return in_lens; -} - -template -std::vector ConvDriver::GetWeightTensorLengthsFromCmdLine() -{ - std::vector wei_lens; - - int spatial_dim = inflags.GetValueInt("spatial_dim"); - wei_lens.resize(2 + spatial_dim); - - auto wei_spatial_lens = boost::adaptors::slice(wei_lens, 2, 2 + spatial_dim); - - int group_count = std::max(inflags.GetValueInt("group_count"), 1); - - int wei_k_len = inflags.GetValueInt("out_channels"); - int wei_c_len = inflags.GetValueInt("in_channels"); - - if(spatial_dim == 2) - { - wei_spatial_lens[0] = inflags.GetValueInt("fil_h"); - wei_spatial_lens[1] = inflags.GetValueInt("fil_w"); - } - else if(spatial_dim == 3) - { - wei_spatial_lens[0] = inflags.GetValueInt("fil_d"); - wei_spatial_lens[1] = inflags.GetValueInt("fil_h"); - wei_spatial_lens[2] = inflags.GetValueInt("fil_w"); - } - else - { - MIOPEN_THROW("unsupported convolution dimension"); - } - - if(group_count > 1) - { - if(wei_c_len % group_count != 0 || wei_k_len % group_count != 0 || - group_count > wei_c_len || group_count > wei_k_len) - { - MIOPEN_THROW("Invalid group number\n"); - } - } - - if(mode == miopenTranspose) - { - wei_lens[0] = wei_c_len; - wei_lens[1] = wei_k_len / group_count; - } - else - { - wei_lens[0] = wei_k_len; - wei_lens[1] = wei_c_len / group_count; - } - - return wei_lens; -} - -template -std::vector ConvDriver::GetBiasTensorLengthsFromCmdLine() -{ - int spatial_dim = inflags.GetValueInt("spatial_dim"); - - std::vector bias_lens(2 + spatial_dim, 1); - - bias_lens[1] = inflags.GetValueInt("out_channels"); - - return bias_lens; -} - -template -int ConvDriver::SetConvDescriptorFromCmdLineArgs() -{ - int spatial_dim = inflags.GetValueInt("spatial_dim"); - - std::vector in_spatial_lens(spatial_dim); - std::vector wei_spatial_lens(spatial_dim); - std::vector pads(spatial_dim); - std::vector conv_strides(spatial_dim); - std::vector conv_dilations(spatial_dim); - std::vector trans_output_pads(spatial_dim); - - if(spatial_dim == 2) - { - in_spatial_lens[0] = inflags.GetValueInt("in_h"); - in_spatial_lens[1] = inflags.GetValueInt("in_w"); - wei_spatial_lens[0] = inflags.GetValueInt("fil_h"); - wei_spatial_lens[1] = inflags.GetValueInt("fil_w"); - pads[0] = inflags.GetValueInt("pad_h"); - pads[1] = inflags.GetValueInt("pad_w"); - conv_strides[0] = inflags.GetValueInt("conv_stride_h"); - conv_strides[1] = inflags.GetValueInt("conv_stride_w"); - conv_dilations[0] = inflags.GetValueInt("dilation_h"); - conv_dilations[1] = inflags.GetValueInt("dilation_w"); - trans_output_pads[0] = inflags.GetValueInt("trans_output_pad_h"); - trans_output_pads[1] = inflags.GetValueInt("trans_output_pad_w"); - } - else if(spatial_dim == 3) - { - in_spatial_lens[0] = inflags.GetValueInt("in_d"); - in_spatial_lens[1] = inflags.GetValueInt("in_h"); - in_spatial_lens[2] = inflags.GetValueInt("in_w"); - wei_spatial_lens[0] = inflags.GetValueInt("fil_d"); - wei_spatial_lens[1] = inflags.GetValueInt("fil_h"); - wei_spatial_lens[2] = inflags.GetValueInt("fil_w"); - pads[0] = inflags.GetValueInt("pad_d"); - pads[1] = inflags.GetValueInt("pad_h"); - pads[2] = inflags.GetValueInt("pad_w"); - conv_strides[0] = inflags.GetValueInt("conv_stride_d"); - conv_strides[1] = inflags.GetValueInt("conv_stride_h"); - conv_strides[2] = inflags.GetValueInt("conv_stride_w"); - conv_dilations[0] = inflags.GetValueInt("dilation_d"); - conv_dilations[1] = inflags.GetValueInt("dilation_h"); - conv_dilations[2] = inflags.GetValueInt("dilation_w"); - trans_output_pads[0] = inflags.GetValueInt("trans_output_pad_d"); - trans_output_pads[1] = inflags.GetValueInt("trans_output_pad_h"); - trans_output_pads[2] = inflags.GetValueInt("trans_output_pad_w"); - } - else - { - MIOPEN_THROW("unsupported convolution dimension"); - } - - int out_c = inflags.GetValueInt("out_channels"); - int in_c = inflags.GetValueInt("in_channels"); - int group_count = std::max(inflags.GetValueInt("group_count"), 1); - - if(group_count > 1) - { - if(in_c % group_count != 0 || out_c % group_count != 0 || group_count > in_c || - group_count > out_c) - { - printf("Invalid group number\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - } - - // adjust padding based on user-defined padding mode - if(mode == miopenConvolution && - (miopen::all_of(conv_dilations, [](auto v) { return v == 1; }) || - miopen::all_of(wei_spatial_lens, [](auto v) { return v == 1; }))) - { - if((inflags.GetValueStr("pad_mode")) == "same") - { - for(int i = 0; i < spatial_dim; ++i) - { - pads[i] = - (in_spatial_lens[i] % conv_strides[i] == 0) - ? (std::max((wei_spatial_lens[i] - conv_strides[i]), 0)) - : (std::max((wei_spatial_lens[i] - (in_spatial_lens[i] % conv_strides[i])), - 0)); - pads[i] /= 2; - } - } - else if((inflags.GetValueStr("pad_mode")) == "valid") - { - for(int i = 0; i < spatial_dim; ++i) - { - pads[i] = 0; - } - } - } - - miopenInitConvolutionNdDescriptor( - convDesc, spatial_dim, pads.data(), conv_strides.data(), conv_dilations.data(), mode); - - miopenSetConvolutionGroupCount(convDesc, group_count); - if(mode == miopenTranspose) - { - miopenSetTransposeConvNdOutputPadding(convDesc, spatial_dim, trans_output_pads.data()); - } - - return miopenStatusSuccess; -} - -template -std::vector ConvDriver::GetOutputTensorLengths() -{ - int ndim = miopen::deref(inputTensor).GetNumDims(); - - std::vector out_lens(ndim); - - miopenGetConvolutionNdForwardOutputDim( - convDesc, inputTensor, weightTensor, &ndim, out_lens.data()); - - return out_lens; -} - -namespace { - -template -void RanGenSubnormBuffer(T* buf, size_t size, int percentage) -{ - if(percentage == 0) - return; - float perc = static_cast(percentage) / 100; - size_t size_need_subnorm = static_cast(static_cast(size) * perc); - std::vector need_subnorm(size, false); - std::fill_n(need_subnorm.begin(), std::min(size_need_subnorm, size), true); - std::shuffle(need_subnorm.begin(), need_subnorm.end(), prng::details::get_prng()); - std::transform(need_subnorm.begin(), need_subnorm.end(), buf, buf, [](bool need, auto val) { - return need ? prng::gen_subnorm() : val; - }); -} - -} // namespace - -template -int ConvDriver::AllocateBuffersAndCopy() -{ - if(wall_enabled) - { - fwd_auxiliary.start(); - fwd_auxiliary.pause(); - bwd_auxiliary.start(); - bwd_auxiliary.pause(); - wrw_auxiliary.start(); - wrw_auxiliary.pause(); - fwd_auxiliary_gwss.start(); - fwd_auxiliary_gwss.pause(); - bwd_auxiliary_gwss.start(); - bwd_auxiliary_gwss.pause(); - wrw_auxiliary_gwss.start(); - wrw_auxiliary_gwss.pause(); - if(warmup_enabled) - { - warmup_wall_total.start(); - warmup_wall_total.pause(); - } - } - - bool is_transform = IsInputTensorTransform(); - bool is_int8 = data_type == miopenInt8 || data_type == miopenInt8x4; - // Data generated for very low precision types follows the same constraints whether its fp8, - // bfp8 or even if the interim tensors are being casted - bool is_fp8 = data_type == miopenFloat8 || data_type == miopenBFloat8 || TensorsCasted(); - size_t in_sz = GetTensorSize(inputTensor); - size_t wei_sz = GetTensorSize(weightTensor); - size_t out_sz = GetTensorSize(outputTensor); - auto subnorm_percentage = env::value(MIOPEN_DRIVER_SUBNORM_PERCENTAGE); - if(subnorm_percentage != 0) - std::cout << "MIOPEN_DRIVER_SUBNORM_PERCENTAGE = " << subnorm_percentage << std::endl; - - // Workaround: Pad buffers allocations to be a multiple of 2M - if(env::enabled(MIOPEN_DRIVER_PAD_BUFFERS_2M)) - { - // PadBufferSize(in_sz, sizeof(Tgpu)); - PadBufferSize(wei_sz, sizeof(Tgpu)); - PadBufferSize(out_sz, sizeof(Tgpu)); - } - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - ws_sizeof_find_fwd = 0; - ws_sizeof_find_wrw = 0; - ws_sizeof_find_bwd = 0; - - if(warmup_enabled) - { - do - { - AutoMiopenWarmupMode warmupMode; - size_t warmup_in_sz = GetTensorSize(warmupInputTensor); - size_t warmup_wei_sz = GetTensorSize(warmupWeightTensor); - size_t warmup_out_sz = GetTensorSize(warmupOutputTensor); - if(env::enabled(MIOPEN_DRIVER_PAD_BUFFERS_2M)) - { - PadBufferSize(warmup_wei_sz, sizeof(warmup_Tgpu)); - PadBufferSize(warmup_out_sz, sizeof(warmup_Tgpu)); - } - - warmup_ws_sizeof_find = 0; - warmup_wall_total.resume(wall_enabled); - miopenStatus_t rc = miopenConvolutionForwardGetWorkSpaceSize(GetHandle(), - warmupWeightTensor, - warmupInputTensor, - warmupConvDesc, - warmupOutputTensor, - &warmup_ws_sizeof_find); - warmup_wall_total.pause(wall_enabled); - if(rc != miopenStatusSuccess) - { - std::cout << "Warm-up: Error getting workspace size, status = " << rc - << ". Warm-up disabled." << std::endl; - warmup_enabled = false; - break; - } - if(warmup_ws_sizeof_find != 0) - { - std::cout << "Warm-up: This step should not require workspace, but asks for " - << warmup_ws_sizeof_find << ". Warm-up disabled." << std::endl; - warmup_enabled = false; - break; - } - - warmup_in.AllocOnHost(warmupInputTensor); - warmup_wei.AllocOnHost(warmupWeightTensor); - warmup_out.AllocOnHost(warmupOutputTensor); - - status_t status = STATUS_SUCCESS; - status |= warmup_in.AllocOnDeviceAndInit(q, ctx, warmup_in_sz); - status |= warmup_wei.AllocOnDeviceAndInit(q, ctx, warmup_wei_sz); - status |= warmup_out.AllocOnDeviceAndInit(q, ctx, warmup_out_sz); - - if(status != STATUS_SUCCESS) - { - std::cout << "Warm-up: Error copying data to GPU, status = " << status - << ". Warm-up disabled." << std::endl; - warmup_enabled = false; - break; - } - - const int rcf = RunWarmupFindForwardGPU(); - if(rcf != 0) - { - std::cout << "Warm-up: RunWarmupFindForwardGPU() FAILED, rcf = " << rcf - << ". Warm-up disabled." << std::endl; - warmup_enabled = false; - break; - } - } while(false); - } - - if(!immediate_solution) - { - miopenStatus_t rc = miopenStatusSuccess; - if(is_wrw && rc == miopenStatusSuccess) - { - wrw_auxiliary.resume(wall_enabled); - wrw_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionBackwardWeightsGetWorkSpaceSize(GetHandle(), - outputTensor, - inputTensor, - convDesc, - weightTensor, - &ws_sizeof_find_wrw); - wrw_auxiliary_gwss.pause(wall_enabled); - wrw_auxiliary.pause(wall_enabled); - AdjustWorkspacesizeVariableFromEnv(ws_sizeof_find_wrw); - } - if(is_bwd && rc == miopenStatusSuccess) - { - bwd_auxiliary.resume(wall_enabled); - bwd_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionBackwardDataGetWorkSpaceSize(GetHandle(), - outputTensor, - weightTensor, - convDesc, - inputTensor, - &ws_sizeof_find_bwd); - bwd_auxiliary_gwss.pause(wall_enabled); - bwd_auxiliary.pause(wall_enabled); - AdjustWorkspacesizeVariableFromEnv(ws_sizeof_find_bwd); - } - if(is_fwd && rc == miopenStatusSuccess) - { - fwd_auxiliary.resume(wall_enabled); - fwd_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionForwardGetWorkSpaceSize( - GetHandle(), - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? inputTensor_vect4 : inputTensor), - convDesc, - outputTensor, - &ws_sizeof_find_fwd); - fwd_auxiliary_gwss.pause(wall_enabled); - fwd_auxiliary.pause(wall_enabled); - AdjustWorkspacesizeVariableFromEnv(ws_sizeof_find_fwd); - } - if(rc != miopenStatusSuccess) - { - std::cout << "Error getting workspace size, status = " << rc << std::endl; - return rc; - } - } - - if(is_fwd || is_wrw) - in.AllocOnHost(inputTensor); - if(is_fwd || is_bwd) - wei.AllocOnHost(weightTensor); - if(is_fwd) - out.AllocOnHost(outputTensor); - if(is_bwd || is_wrw) - dout.AllocOnHost(outputTensor); - - if(is_bwd) - din.AllocOnHost(in_sz); - if(is_wrw) - dwei.AllocOnHost(wei_sz); - if(is_int8) - out_int8 = std::vector(out_sz, 0); - if(is_transform) - { - in_vect4_dev = std::unique_ptr( - new GPUMem(ctx, GetTensorSize(inputTensor_vect4), sizeof(Tgpu))); - wei_vect4_dev = std::unique_ptr( - new GPUMem(ctx, GetTensorSize(weightTensor_vect4), sizeof(Tgpu))); - } - - outhost = tensor(miopen::deref(outputTensor).GetLayout_t(), - miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides()); - din_host = tensor(miopen::deref(inputTensor).GetLayout_t(), - miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides()); - dwei_host = tensor(miopen::deref(weightTensor).GetLayout_t(), - miopen::deref(weightTensor).GetLengths(), - miopen::deref(weightTensor).GetStrides()); - - std::string inFileName = inflags.GetValueStr("in_data"); - std::string weiFileName = inflags.GetValueStr("weights"); - std::string biasFileName = inflags.GetValueStr("in_bias"); - std::string doutFileName = inflags.GetValueStr("dout_data"); - - bool dataRead = false; - if(is_fwd || is_wrw) - if(!inFileName.empty()) - dataRead = readBufferFromFile(in.GetVectorData(), in_sz, inFileName.c_str()); - - bool weiRead = false; - if(is_fwd || is_bwd) - if(!weiFileName.empty()) - weiRead = readBufferFromFile(wei.GetVectorData(), wei_sz, weiFileName.c_str()); - - const Tgpu Data_scale = is_int8 ? static_cast(127) - : (is_fp8 ? static_cast(1.0) : static_cast(0.01)); - const Tgpu Data_min = (is_fp8 ? static_cast(-1.0) : static_cast(0.0)); - const Tgpu Data_max = (is_fp8 ? static_cast(1.0) : static_cast(1.0)); - if(is_int8) - { - if(inflags.GetValueInt("bias") != 0) - { - size_t b_sz = GetTensorSize(biasTensor); - b_int8 = std::vector(b_sz, 0.f); - - if(!is_gpualloc) - { - bool read = false; - if(!biasFileName.empty()) - read = readBufferFromFile(b_int8.data(), b_sz, biasFileName.c_str()); - if(!read) - for(size_t i = 0; i < b_sz; ++i) - b_int8[i] = static_cast(i % 8) + prng::gen_canonical(); - } - std::ignore = b.AllocOnDeviceAndInit(q, ctx, b_sz, b_int8); - } - } - else - { - - bool doutRead = false; - if(is_bwd || is_wrw) - if(!doutFileName.empty()) - doutRead = - readBufferFromFile(dout.GetVectorData(), out_sz, doutFileName.c_str()); - - if(!doutRead) - { - auto gen = [&]() -> auto - { - return is_fp8 ? prng::gen_A_to_B(Data_min, Data_max) : prng::gen_0_to_B(Data_scale); - }; - dout.InitHostData(out_sz, is_bwd || is_wrw, gen); - } - - if(is_wrw) - if(!is_gpualloc) - RanGenSubnormBuffer(dout.GetVectorData(), out_sz, subnorm_percentage); - - if(inflags.GetValueInt("bias") != 0) - { - size_t b_sz = GetTensorSize(biasTensor); - - b.AllocOnHost(biasTensor); - db.AllocOnHost(b_sz); - db_host = tensor(miopen::deref(biasTensor)); - - // Init tensor on host - bool b_read = false; - if(!biasFileName.empty()) - b_read = readBufferFromFile(b.GetVectorData(), b_sz, biasFileName.c_str()); - - if(!is_gpualloc) - { - for(size_t i = 0; i < b_sz; ++i) - { - if(!b_read) - { - /// (i % 8) can't be converted to F8 type as there is no suitable - /// conversion, but we have conversions from int and from uint8_t. - /// int is not good as it would produce negative results - /// after truncation of size_t, while we want positive values. - /// uint8_t is fine because (i % 8) fits into 3 bits. - b.GetVector()[i] = static_cast(static_cast(i) % 8) // - + (is_fp8 ? prng::gen_A_to_B(Data_min, Data_max) // - : prng::gen_canonical()); - } - db.GetVector()[i] = static_cast(static_cast(i) % 8) // - + (is_fp8 ? prng::gen_A_to_B(Data_min, Data_max) // - : prng::gen_canonical()); - } - } - - b.AllocOnDeviceAndInit(q, ctx, b_sz); - db.AllocOnDeviceAndInit(q, ctx, b_sz); - } - } - - if(!dataRead) - { - auto gen = [&]() -> Tgpu { - return is_fp8 ? prng::gen_A_to_B(Data_min, Data_max) : prng::gen_0_to_B(Data_scale); - }; - in.InitHostData(in_sz, is_fwd || is_wrw, gen); - } - - if(!weiRead) - { - auto gen = [&]() -> auto { return Data_scale * conv::RanGenWeights(); }; - wei.InitHostData(wei_sz, is_fwd || is_bwd, gen); - } - - if(is_fwd || is_bwd) - if(!is_gpualloc) - RanGenSubnormBuffer(wei.GetVectorData(), wei_sz, subnorm_percentage); - - if(inflags.GetValueInt("dump_output")) - { - if(is_fwd || is_wrw) - dumpBufferToFile("dump_in.bin", in.GetVectorData(), in_sz); - if(is_fwd || is_bwd) - dumpBufferToFile("dump_wei.bin", wei.GetVectorData(), wei_sz); - if(inflags.GetValueInt("bias") != 0) - dumpBufferToFile("dump_bias.bin", b.GetVectorData(), b.GetVectorSize()); - if(is_bwd || is_wrw) - dumpBufferToFile("dump_dout.bin", dout.GetVectorData(), out_sz); - } - - status_t status = STATUS_SUCCESS; - - if(is_fwd || is_wrw) - { - status |= in.AllocOnDeviceAndInit(q, ctx, in_sz); - } - if(is_bwd) - { - status |= din.AllocOnDevice(q, ctx, in_sz); - } - if(is_fwd || is_bwd) - { - status |= wei.AllocOnDeviceAndInit(q, ctx, wei_sz); - } - if(is_wrw) - { - status |= dwei.AllocOnDevice(q, ctx, wei_sz); - } - if(is_bwd || is_wrw) - { - status |= dout.AllocOnDeviceAndInit(q, ctx, out_sz); - } - if(is_fwd) - { - /// \todo: For the temporary conversion to half, this is required, however, that would also - /// need change elsewhere which has not yet been implemented: - /// - /// out_dev = ... (is_fp8 ? sizeof(half) : sizeof(Tgpu)) - /// - /// \note The above todo is necessary only when tensor casting is used. --atamazov Feb 2024 - std::ignore = is_fp8; - - status |= is_int8 ? out.AllocOnDevice(q, ctx, out_sz, out_int8) // - : out.AllocOnDevice(q, ctx, out_sz); - } - - if(status != STATUS_SUCCESS) - { - std::cout << "Error copying data to GPU, status = " << status << std::endl; - return miopenStatusNotInitialized; - } - return miopenStatusSuccess; -} - -template -bool ConvDriver::UseGPUReference() -{ - if(!env::disabled(MIOPEN_DRIVER_USE_GPU_REFERENCE)) - { - if((miopen_type{} == miopenFloat && - (miopen_type{} == miopenFloat || miopen_type{} == miopenHalf || - miopen_type{} == miopenBFloat16 || miopen_type{} == miopenFloat8 || - miopen_type{} == miopenBFloat8)) || - (miopen_type{} == miopenInt32 && miopen_type{} == miopenInt8)) - return true; - else - return false; - } - else - return false; -} - -template -int ConvDriver::FindForward(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx) -{ - bool is_transform = IsInputTensorTransform(); - fwd_auxiliary.resume(wall_enabled); - ResizeWorkspaceDev(ctx, ws_sizeof_find_fwd); - const auto rc = miopenFindConvolutionForwardAlgorithm( - GetHandle(), - (is_transform ? inputTensor_vect4 : inputTensor), - (is_transform ? in_vect4_dev->GetMem() : in.GetDevicePtr()), - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? wei_vect4_dev->GetMem() : wei.GetDevicePtr()), - convDesc, - outputTensor, - out.GetDevicePtr(), - request_algo_count, - &ret_algo_count, - perf_results.data(), - workspace_dev != nullptr ? workspace_dev->GetMem() : nullptr, - ws_sizeof_find_fwd, - (inflags.GetValueInt("search") == 1) ? true : false); - fwd_auxiliary.pause(wall_enabled); - return rc; -} - -template -void ConvDriver::PrintForwardTime(const float kernel_total_time, - const float kernel_first_time) const -{ - float kernel_average_time = ComputeAverageTime(kernel_total_time, kernel_first_time); - printf("GPU Kernel Time Forward Conv. Elapsed: %f ms (average)\n", kernel_average_time); - - const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; - if(num_dim != 2 && num_dim != 3) - { - printf("stats: for conv%ud\n", num_dim); - return; - } - - int group_count = std::max(inflags.GetValueInt("group_count"), 1); - - if(num_dim == 2) - { - int in_n, in_c, in_h, in_w; - std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_h, wei_w) = - miopen::tien<4>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_h, out_w; - std::tie(out_n, out_c, out_h, out_w) = - miopen::tien<4>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * wei_h * wei_w * out_c * out_h * - out_w / group_count; - size_t inputBytes = - in_n * in_c * in_h * in_w * miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); - size_t weightBytes = wei_n * wei_c * wei_h * wei_w * - miopen::GetTypeSize(miopen::deref(weightTensor).GetType()); - size_t readBytes = inputBytes + weightBytes; - - size_t outputBytes = 1.0 * out_n * out_c * out_h * out_w * - miopen::GetTypeSize(miopen::deref(outputTensor).GetType()); - - printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", - "fwd-conv", - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - out_h, - out_w, - wei_h, - wei_w, - out_c, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } - else - { // 3d - int in_n, in_c, in_d, in_h, in_w; - std::tie(in_n, in_c, in_d, in_h, in_w) = - miopen::tien<5>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_d, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_d, wei_h, wei_w) = - miopen::tien<5>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_d, out_h, out_w; - std::tie(out_n, out_c, out_d, out_h, out_w) = - miopen::tien<5>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * in_d * wei_h * wei_w * wei_d * - out_c * out_d * out_h * out_w / group_count; - size_t inputBytes = in_n * in_c * in_d * in_h * in_w * - miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); - size_t weightBytes = wei_n * wei_c * wei_d * wei_h * wei_w * - miopen::GetTypeSize(miopen::deref(weightTensor).GetType()); - size_t readBytes = inputBytes + weightBytes; - - size_t outputBytes = 1.0 * out_n * out_c * out_d * out_h * out_w * - miopen::GetTypeSize(miopen::deref(outputTensor).GetType()); - - printf("stats: name , n, c, do, ho, wo, z, y, x, k, flopCnt, bytesRead, bytesWritten, " - "GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, " - "%.0f, %.0f, %f\n", - "fwd-conv", - wei_d, - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - out_d, - out_h, - out_w, - wei_d, - wei_h, - wei_w, - out_c, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } -} - -/// Always warm-ups Find API. Why: this is definitely required for Find mode. -/// For Immediate mode, this guarantees that we won't hit fallback. -/// Immediate mode API is warmed-up only when driver is used in Immediate mode. -template -int ConvDriver::RunWarmupFindForwardGPU() -{ - if(!warmup_enabled) - return 0; - - AutoMiopenWarmupMode warmupMode; - - int find_count; - miopenConvAlgoPerf_t find_result; - warmup_wall_total.resume(wall_enabled); - auto rc = miopenFindConvolutionForwardAlgorithm(GetHandle(), - warmupInputTensor, - warmup_in.GetDevicePtr(), - warmupWeightTensor, - warmup_wei.GetDevicePtr(), - warmupConvDesc, - warmupOutputTensor, - warmup_out.GetDevicePtr(), - 1, - &find_count, - &find_result, - nullptr, - 0, - false); - warmup_wall_total.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return 10; - if(find_count == 0) - return 20; - - miopenConvSolution_t solution; - if(immediate_solution) - { - std::size_t immed_count; - warmup_wall_total.resume(wall_enabled); - rc = miopenConvolutionForwardGetSolutionCount(handle, - warmupWeightTensor, - warmupInputTensor, - warmupConvDesc, - warmupOutputTensor, - &immed_count); - warmup_wall_total.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return 30; - if(immed_count < 1) - return 40; - - warmup_wall_total.resume(wall_enabled); - rc = miopenConvolutionForwardGetSolution(handle, - warmupWeightTensor, - warmupInputTensor, - warmupConvDesc, - warmupOutputTensor, - 1, - &immed_count, - &solution); - warmup_wall_total.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return 50; - if(immed_count < 1) - return 60; - if(solution.workspace_size != 0) - return 70; - - warmup_wall_total.resume(wall_enabled); - rc = miopenConvolutionForwardImmediate(handle, - warmupWeightTensor, - warmup_wei.GetDevicePtr(), - warmupInputTensor, - warmup_in.GetDevicePtr(), - warmupConvDesc, - warmupOutputTensor, - warmup_out.GetDevicePtr(), - nullptr, - 0, - solution.solution_id); - warmup_wall_total.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return 80; - } - - warmup_wall_total.stop(wall_enabled); - std::ostringstream ss; - ss << "Warm-up: "; - if(wall_enabled) - ss << "Wall-clock Total Time: " << warmup_wall_total.gettime_ms() << " ms, "; - ss << "Find Algorithm: " << find_result.fwd_algo; - if(immediate_solution) - ss << ", Immediate Algorithm: " << miopen::ConvolutionAlgoToString(solution.algorithm) - << '[' << solution.solution_id << ']'; - ss << std::endl; - std::cout << ss.str(); - return rc; -} - -template -int ConvDriver::RunForwardGPU() -{ - if(!is_fwd) - return 0; - - int rc; - bool is_transform = IsInputTensorTransform(); - - if(is_transform) - { - float aph = 1.0; - float bta = 0.0; - miopenTransformTensor(GetHandle(), - &aph, - inputTensor, - in.GetDevicePtr(), - &bta, - inputTensor_vect4, - in_vect4_dev->GetMem()); - - miopenTransformTensor(GetHandle(), - &aph, - weightTensor, - wei.GetDevicePtr(), - &bta, - weightTensor_vect4, - wei_vect4_dev->GetMem()); - } - - if(immediate_solution) - rc = RunForwardGpuImmed(is_transform); - else - rc = RunForwardGpuFind(is_transform); - - is_fwd_run_failed = (rc != 0); - - if(rc != miopenStatusSuccess) - return rc; - - if(inflags.GetValueInt("bias") != 0) - { - float alpha = static_cast(1), beta = static_cast(0); - - miopenConvolutionForwardBias(GetHandle(), - &alpha, - biasTensor, - b.GetDevicePtr(), - &beta, - outputTensor, - out.GetDevicePtr()); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - - printf("GPU Kernel Time Forward Conv. Bias Elapsed: %f ms\n", time); - } - } - - bool is_int8 = data_type == miopenInt8 || data_type == miopenInt8x4; - if(is_int8) - out.CopyFromDeviceToHost(GetStream(), out_int8); - else - out.CopyFromDeviceToHost(GetStream()); - - if(inflags.GetValueInt("dump_output")) - { - if(is_int8) - dumpBufferToFile("dump_fwd_out_gpu.bin", out_int8.data(), out_int8.size()); - else - dumpBufferToFile( - "dump_fwd_out_gpu.bin", out.GetVectorData(), out.GetVectorSize()); - } - - return rc; -} - -template -void ConvDriver::GetSolutionAfterFind( - const miopenConvAlgoPerf_t& found, - const ConvDriver::Direction& direction, - const miopenTensorDescriptor_t& in_tensor, - const miopenTensorDescriptor_t& wei_tensor, - const miopenTensorDescriptor_t& out_tensor, - miopenConvSolution_t& solution) -{ - AutoMiopenWarmupMode warmupMode; // Shut logging. - miopenConvAlgorithm_t found_algo; - switch(direction) - { - case Direction::Fwd: found_algo = static_cast(found.fwd_algo); break; - case Direction::Bwd: - found_algo = static_cast(found.bwd_data_algo); - break; - case Direction::WrW: - found_algo = static_cast(found.bwd_weights_algo); - break; - case Direction::BwdBias: // nop - MIOPEN_THROW("BwdBias is not supported"); - } - std::size_t immed_count = 0; - miopenStatus_t rc = miopenStatusUnknownError; - switch(direction) - { - case Direction::Fwd: - rc = miopenConvolutionForwardGetSolution( - handle, wei_tensor, in_tensor, convDesc, out_tensor, 1, &immed_count, &solution); - break; - case Direction::Bwd: - rc = miopenConvolutionBackwardDataGetSolution( - handle, out_tensor, wei_tensor, convDesc, in_tensor, 1, &immed_count, &solution); - break; - case Direction::WrW: - rc = miopenConvolutionBackwardWeightsGetSolution( - handle, out_tensor, in_tensor, convDesc, wei_tensor, 1, &immed_count, &solution); - break; - case Direction::BwdBias: // nop - break; - } - if(rc != miopenStatusSuccess // (formatting) - || immed_count < 1 // It should not be so if Find succeeded. - || found_algo != solution.algorithm // These must match. - || solution.time < 0) // Fallback mode (no entry in find-db -- disabled?) - { - // Ignore errors, just skip printing the solver information. - solution.solution_id = 0; - } -} - -template -int ConvDriver::RunForwardGpuFind(const bool is_transform) -{ - int ret_algo_count; - int request_algo_count = 2; - // The library returns `request_algo_count` algorithms to the caller. However this does - // not affect auto-tuning and find-db updates. Internally, the library searches for - // *all* available algorithms during Find(), -- regardless of how many algorithms - // requested, -- so perf-db and find-db are fully updated. - std::vector perf_results(request_algo_count); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto rc = FindForward(ret_algo_count, request_algo_count, perf_results, ctx); - if(rc != miopenStatusSuccess) - return rc; - - if(ret_algo_count == 0) - throw std::runtime_error("Find Forward Conv. ret_algo_count == 0"); - - float alpha = static_cast(1), beta = static_cast(0); - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - - const auto algo = perf_results[0].fwd_algo; // use the fastest algo - const auto ws_size = perf_results[0].memory; - is_fwd_igemm = (algo == miopenConvolutionFwdAlgoImplicitGEMM); - - auto in_tens = (is_transform ? inputTensor_vect4 : inputTensor); - auto in_buff = (is_transform ? in_vect4_dev->GetMem() : in.GetDevicePtr()); - auto wei_tens = (is_transform ? weightTensor_vect4 : weightTensor); - auto wei_buff = (is_transform ? wei_vect4_dev->GetMem() : wei.GetDevicePtr()); - - if(ws_size > ws_sizeof_find_fwd) - { - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Error, - "MIOpenDriver", - "Find returns bigger workspace than provided " << ws_sizeof_find_fwd - << " < " << ws_size); - return miopenStatusInternalError; - } - ResizeWorkspaceDev(ctx, ws_size); - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionForward(GetHandle(), - &alpha, - in_tens, - in_buff, - wei_tens, - wei_buff, - convDesc, - algo, - &beta, - outputTensor, - out.GetDevicePtr(), - workspace_dev != nullptr ? workspace_dev->GetMem() : nullptr, - ws_size); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - fwd_auxiliary.stop(); - fwd_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Forward Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << fwd_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << fwd_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - miopenConvSolution_t solution; - GetSolutionAfterFind( - perf_results[0], Direction::Fwd, in_tens, wei_tens, outputTensor, solution); - std::cout << "MIOpen Forward Conv. " << AlgorithmSolutionToString(solution) << std::endl; - PrintForwardTime(kernel_total_time, kernel_first_time); - } - - return rc; -} - -template -int ConvDriver::RunForwardGpuImmed(const bool is_transform) -{ - std::size_t count; - fwd_auxiliary.resume(wall_enabled); - auto rc = - miopenConvolutionForwardGetSolutionCount(handle, - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? inputTensor_vect4 : inputTensor), - convDesc, - outputTensor, - &count); - fwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - if(count < 1) - return miopenStatusNotImplemented; - - auto solutions = std::vector(count); - fwd_auxiliary.resume(wall_enabled); - rc = miopenConvolutionForwardGetSolution(handle, - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? inputTensor_vect4 : inputTensor), - convDesc, - outputTensor, - count, - &count, - solutions.data()); - fwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - - std::cout << "Forward Conv solutions available: " << count << std::endl; - if(count < 1) - return miopenStatusNotImplemented; - - solutions.resize(count); - const miopenConvSolution_t* selected = nullptr; - - for(const auto& s : solutions) - PrintImmedSolutionInfo(s); - - if(*immediate_solution == 0) - selected = &solutions.front(); - else - for(const auto& s : solutions) - if(*immediate_solution == s.solution_id) - { - selected = &s; - break; - } - - miopenConvSolution_t voluntary = { - -1.0, 0, *immediate_solution, static_cast(-1)}; - if(selected == nullptr) - { - std::cout << "Warning: Solution id (" << *immediate_solution - << ") is not reported by the library. Trying it anyway..." << std::endl; - selected = &voluntary; - } - - std::size_t ws_size; - - fwd_auxiliary.resume(wall_enabled); - fwd_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionForwardGetSolutionWorkspaceSize( - handle, - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? inputTensor_vect4 : inputTensor), - convDesc, - outputTensor, - selected->solution_id, - &ws_size); - fwd_auxiliary_gwss.pause(wall_enabled); - fwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto ws = std::unique_ptr{ws_size > 0 ? new GPUMem{ctx, ws_size, 1} : nullptr}; - - fwd_auxiliary.resume(wall_enabled); - rc = miopenConvolutionForwardCompileSolution(handle, - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? inputTensor_vect4 : inputTensor), - convDesc, - outputTensor, - selected->solution_id); - fwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionForwardImmediate( - handle, - (is_transform ? weightTensor_vect4 : weightTensor), - (is_transform ? wei_vect4_dev->GetMem() : wei.GetDevicePtr()), - (is_transform ? inputTensor_vect4 : inputTensor), - (is_transform ? in_vect4_dev->GetMem() : in.GetDevicePtr()), - convDesc, - outputTensor, - out.GetDevicePtr(), - ws ? ws->GetMem() : nullptr, - ws_size, - selected->solution_id); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - fwd_auxiliary.stop(); - fwd_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Forward Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << fwd_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << fwd_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - std::cout << "MIOpen Forward Conv. " << AlgorithmSolutionToString(*selected) << std::endl; - PrintForwardTime(kernel_total_time, kernel_first_time); - } - - is_fwd_igemm = (selected->algorithm == miopenConvolutionAlgoImplicitGEMM); - return miopenStatusSuccess; -} - -template -int ConvDriver::RunForwardCPU() -{ - if(mode == miopenTranspose) - { - cpu_convolution_backward_data(miopen::deref(convDesc).GetSpatialDimension(), - outhost, - wei.GetTensor(), - in.GetTensor(), - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - - if(inflags.GetValueInt("bias") != 0) - { - cpu_bias_forward(outhost, b.GetTensor()); - } - } - else - { - cpu_convolution_forward(miopen::deref(convDesc).GetSpatialDimension(), - in.GetTensor(), - wei.GetTensor(), - outhost, - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - - if(inflags.GetValueInt("bias") != 0) - { - outhost.par_for_each([&](auto out_n_id, auto out_k_id, auto... out_spatial_id_pack) { - outhost(out_n_id, out_k_id, out_spatial_id_pack...) = - double(outhost(out_n_id, out_k_id, out_spatial_id_pack...)) + - double(b.GetVector()[out_k_id]); - }); - } - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_fwd_out_cpu.bin", outhost.data.data(), outhost.data.size()); - } - - TrySaveVerificationCache(Direction::Fwd, outhost.data); - return 0; -} - -template -int ConvDriver::RunForwardGPUReference() -{ - AutoPrepareForGpuReference naive_conv_enable; - - if(inflags.GetValueInt("bias") != 0) - { - std::cout << "gpu reference convolution does not support bias yet" << std::endl; - return -1; - } - auto ref_solution_id = mode == miopenTranspose // - ? miopen::solver::Id("ConvDirectNaiveConvBwd").Value() - : miopen::solver::Id("ConvDirectNaiveConvFwd").Value(); - auto rc = miopenConvolutionForwardImmediate(handle, - weightTensor, - wei.GetDevicePtr(), - inputTensor, - in.GetDevicePtr(), - convDesc, - outputTensor, - out.GetDevicePtr(), - nullptr, - 0, - ref_solution_id); - if(rc != miopenStatusSuccess) - { - std::cout << "reference kernel fail to run " - << miopen::solver::Id(ref_solution_id).ToString() << std::endl; - return rc; - } - - if(miopen_type{} == miopen_type{} || miopen_type{} == miopenInt8 || - miopen_type{} == miopenInt8x4) - out.CopyFromDeviceToHost(GetStream(), outhost); - else - { - if(!is_gpualloc) - { - auto out_tmp = tensor(miopen::deref(outputTensor)); - out.CopyFromDeviceToHost(GetStream(), out_tmp); - for(size_t i = 0; i < out_tmp.data.size(); ++i) - { - outhost.data[i] = static_cast(out_tmp.data[i]); - } - } - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile( - "dump_fwd_out_gpu_ref.bin", outhost.data.data(), outhost.data.size()); - } - - // TrySaveVerificationCache(Direction::Fwd, outhost.data); - return 0; -} - -template -int ConvDriver::FindBackwardData(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx) -{ - bwd_auxiliary.resume(wall_enabled); - ResizeWorkspaceDev(ctx, ws_sizeof_find_bwd); - const auto rc = miopenFindConvolutionBackwardDataAlgorithm( - GetHandle(), - outputTensor, - dout.GetDevicePtr(), - weightTensor, - wei.GetDevicePtr(), - convDesc, - inputTensor, - din.GetDevicePtr(), - request_algo_count, - &ret_algo_count, - perf_results.data(), - workspace_dev != nullptr ? workspace_dev->GetMem() : nullptr, - ws_sizeof_find_bwd, - (inflags.GetValueInt("search") == 1) ? true : false); - bwd_auxiliary.pause(wall_enabled); - return rc; -} - -template -int ConvDriver::FindBackwardWeights(int& ret_algo_count, - int request_algo_count, - std::vector& perf_results, - context_t ctx) -{ - wrw_auxiliary.resume(wall_enabled); - ResizeWorkspaceDev(ctx, ws_sizeof_find_wrw); - const auto rc = miopenFindConvolutionBackwardWeightsAlgorithm( - GetHandle(), - outputTensor, - dout.GetDevicePtr(), - inputTensor, - in.GetDevicePtr(), - convDesc, - weightTensor, - dwei.GetDevicePtr(), - request_algo_count, - &ret_algo_count, - perf_results.data(), - workspace_dev != nullptr ? workspace_dev->GetMem() : nullptr, - ws_sizeof_find_wrw, - (inflags.GetValueInt("search") == 1) ? true : false); - wrw_auxiliary.pause(wall_enabled); - return rc; -} - -template -int ConvDriver::RunBackwardGPU() -{ - if(data_type == miopenInt8 || data_type == miopenInt8x4) - { - std::cout << "Int8 Backward Convolution is not supported" << std::endl; - return 0; - } - - if(!(is_bwd || is_wrw)) - return 0; - - int ret = 0; - - if(is_bwd) - { - auto rc = immediate_solution ? RunBackwardDataGpuImmed() : RunBackwardDataGpuFind(); - is_bwd_run_failed = (rc != 0); - ret |= rc; - } - - if(is_wrw) - { - auto rc = immediate_solution ? RunBackwardWrwGpuImmed() : RunBackwardWrwGpuFind(); - is_wrw_run_failed = (rc != 0); - ret |= (rc << 16); // Differentiate WrW and Bwd error codes. - } - - if(inflags.GetValueInt("dump_output")) - { - if(is_bwd) - dumpBufferToFile( - "dump_bwd_din_gpu.bin", din.GetVectorData(), din.GetVectorSize()); - if(is_wrw) - dumpBufferToFile( - "dump_bwd_dwei_gpu.bin", dwei.GetVectorData(), dwei.GetVectorSize()); - } - - if(inflags.GetValueInt("bias") != 0) - { - float alpha = static_cast(1), beta = static_cast(0); - - ret |= miopenConvolutionBackwardBias(GetHandle(), - &alpha, - outputTensor, - dout.GetDevicePtr(), - &beta, - biasTensor, - db.GetDevicePtr()); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - printf("GPU Kernel Time Backward Bias Conv. Elapsed: %f ms\n", time); - } - - db.CopyFromDeviceToHost(GetStream()); - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_db_gpu.bin", db.GetVectorData(), db.GetVectorSize()); - } - } - return ret; -} - -template -int ConvDriver::RunBackwardDataGpuFind() -{ - int ret_algo_count; - int request_algo_count = 2; - std::vector perf_results_data(request_algo_count); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto rc = FindBackwardData(ret_algo_count, request_algo_count, perf_results_data, ctx); - if(rc != miopenStatusSuccess) - return rc; - - if(ret_algo_count == 0) - throw std::runtime_error("Find Backward Data Conv. ret_algo_count == 0"); - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - float alpha = static_cast(1), beta = static_cast(0); - - const auto algo = perf_results_data[0].bwd_data_algo; - const auto ws_size = perf_results_data[0].memory; - is_bwd_igemm = (algo == miopenConvolutionBwdDataAlgoImplicitGEMM); - - if(ws_size > ws_sizeof_find_bwd) - { - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Error, - "MIOpenDriver", - "Find returns bigger workspace than provided " << ws_sizeof_find_bwd - << " < " << ws_size); - return miopenStatusInternalError; - } - ResizeWorkspaceDev(ctx, ws_size); - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionBackwardData(GetHandle(), - &alpha, - outputTensor, - dout.GetDevicePtr(), - weightTensor, - wei.GetDevicePtr(), - convDesc, - algo, - &beta, - inputTensor, - din.GetDevicePtr(), - workspace_dev != nullptr ? workspace_dev->GetMem() - : nullptr, - ws_size); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - bwd_auxiliary.stop(); - bwd_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Backward Data Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << bwd_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << bwd_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - miopenConvSolution_t solution; - GetSolutionAfterFind(perf_results_data[0], - Direction::Bwd, - inputTensor, - weightTensor, - outputTensor, - solution); - std::cout << "MIOpen Backward Data Conv. " << AlgorithmSolutionToString(solution) - << std::endl; - PrintBackwardDataTime(kernel_total_time, kernel_first_time); - } - - din.CopyFromDeviceToHost(GetStream()); - return rc; -} - -template -void ConvDriver::PrintBackwardDataTime(float kernel_total_time, float kernel_first_time) -{ - float kernel_average_time = ComputeAverageTime(kernel_total_time, kernel_first_time); - printf("GPU Kernel Time Backward Data Conv. Elapsed: %f ms (average)\n", kernel_average_time); - - const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; - if(num_dim != 2 && num_dim != 3) - { - printf("stats: for conv%ud\n", num_dim); - return; - } - - int group_count = std::max(inflags.GetValueInt("group_count"), 1); - - if(num_dim == 2) - { - int in_n, in_c, in_h, in_w; - std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_h, wei_w) = - miopen::tien<4>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_h, out_w; - std::tie(out_n, out_c, out_h, out_w) = - miopen::tien<4>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * wei_h * wei_w * out_c * out_h * - out_w / group_count; - size_t weightBytes = wei_n * wei_c * wei_h * wei_w * - miopen::GetTypeSize(miopen::deref(weightTensor).GetType()); - size_t inputBytes = - in_n * in_c * out_c * miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); - size_t readBytes = inputBytes + weightBytes; - - size_t outputBytes = 1.0 * out_n * out_c * out_h * out_w * - miopen::GetTypeSize(miopen::deref(outputTensor).GetType()); - - printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", - "bwdd-conv", - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - wei_h, - wei_w, - out_c, - out_h, - out_w, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } - else - { // 3d - int in_n, in_c, in_d, in_h, in_w; - std::tie(in_n, in_c, in_d, in_h, in_w) = - miopen::tien<5>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_d, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_d, wei_h, wei_w) = - miopen::tien<5>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_d, out_h, out_w; - std::tie(out_n, out_c, out_d, out_h, out_w) = - miopen::tien<5>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * wei_d * wei_h * wei_w * out_c * - out_d * out_h * out_w / group_count; - size_t weightBytes = wei_n * wei_c * wei_d * wei_h * wei_w * - miopen::GetTypeSize(miopen::deref(weightTensor).GetType()); - size_t inputBytes = - in_n * in_c * out_c * miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); - size_t readBytes = inputBytes + weightBytes; - - size_t outputBytes = 1.0 * out_n * out_c * out_d * out_h * out_w * - miopen::GetTypeSize(miopen::deref(outputTensor).GetType()); - - printf( - "stats: name, n, c, do, ho, wo, z, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d %zu, %zu, %zu, %.0f, " - "%.0f, %f\n", - "bwdd-conv", - wei_d, - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - wei_d, - wei_h, - wei_w, - out_c, - out_d, - out_h, - out_w, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } -} - -template -int ConvDriver::RunBackwardWrwGpuFind() -{ - int ret_algo_count; - int request_algo_count = 2; - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - - float alpha = static_cast(1), beta = static_cast(0); - std::vector perf_results_weights(request_algo_count); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto rc = FindBackwardWeights(ret_algo_count, request_algo_count, perf_results_weights, ctx); - if(rc != miopenStatusSuccess) - return rc; - - if(ret_algo_count == 0) - throw std::runtime_error("Find Backward Weights Conv. ret_algo_count == 0"); - - const auto algo = perf_results_weights[0].bwd_weights_algo; - const auto ws_size = perf_results_weights[0].memory; - is_wrw_winograd = (algo == miopenConvolutionBwdWeightsAlgoWinograd); - is_wrw_igemm = (algo == miopenConvolutionBwdWeightsAlgoImplicitGEMM); - - if(ws_size > ws_sizeof_find_wrw) - { - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Error, - "MIOpenDriver", - "Find returns bigger workspace than provided " << ws_sizeof_find_wrw - << " < " << ws_size); - return miopenStatusInternalError; - } - ResizeWorkspaceDev(ctx, ws_size); - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionBackwardWeights(GetHandle(), - &alpha, - outputTensor, - dout.GetDevicePtr(), - inputTensor, - in.GetDevicePtr(), - convDesc, - algo, - &beta, - weightTensor, - dwei.GetDevicePtr(), - workspace_dev != nullptr ? workspace_dev->GetMem() - : nullptr, - ws_size); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - wrw_auxiliary.stop(); - wrw_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Backward Weights Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << wrw_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << wrw_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - miopenConvSolution_t solution; - GetSolutionAfterFind(perf_results_weights[0], - Direction::WrW, - inputTensor, - weightTensor, - outputTensor, - solution); - std::cout << "MIOpen Backward Weights Conv. " << AlgorithmSolutionToString(solution) - << std::endl; - PrintBackwardWrwTime(kernel_total_time, kernel_first_time); - } - - dwei.CopyFromDeviceToHost(GetStream()); - return rc; -} - -template -void ConvDriver::PrintBackwardWrwTime(float kernel_total_time, float kernel_first_time) -{ - float kernel_average_time = ComputeAverageTime(kernel_total_time, kernel_first_time); - printf("GPU Kernel Time Backward Weights Conv. Elapsed: %f ms (average)\n", - kernel_average_time); - - const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; - if(num_dim != 2 && num_dim != 3) - { - printf("stats: for conv%ud\n", num_dim); - return; - } - - int group_count = std::max(inflags.GetValueInt("group_count"), 1); - - if(num_dim == 2) - { - int in_n, in_c, in_h, in_w; - std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_h, wei_w) = - miopen::tien<4>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_h, out_w; - std::tie(out_n, out_c, out_h, out_w) = - miopen::tien<4>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * wei_h * wei_w * out_c * out_h * - out_w / group_count; - size_t readBytes = 0; - size_t outputBytes = 0; - - printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", - "bwdw-conv", - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - out_h, - out_w, - wei_h, - wei_w, - out_c, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } - else - { // 3d - int in_n, in_c, in_d, in_h, in_w; - std::tie(in_n, in_c, in_d, in_h, in_w) = - miopen::tien<5>(miopen::deref(inputTensor).GetLengths()); - int wei_c, wei_n, wei_d, wei_h, wei_w; - std::tie(wei_c, wei_n, wei_d, wei_h, wei_w) = - miopen::tien<5>(miopen::deref(weightTensor).GetLengths()); - int out_n, out_c, out_d, out_h, out_w; - std::tie(out_n, out_c, out_d, out_h, out_w) = - miopen::tien<5>(miopen::deref(outputTensor).GetLengths()); - - size_t flopCnt = static_cast(2) * in_n * in_c * wei_d * wei_h * wei_w * out_c * - out_d * out_h * out_w / group_count; - size_t readBytes = 0; - size_t outputBytes = 0; - - printf( - "stats: name, n, c, do, ho, wo, z, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " - "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, " - "%.0f, %f\n", - "bwdw-conv", - wei_d, - wei_h, - wei_w, - miopen::deref(convDesc).GetConvStrides()[0], - in_n, - in_c, - out_d, - out_h, - out_w, - wei_d, - wei_h, - wei_w, - out_c, - flopCnt, - readBytes, - outputBytes, - flopCnt / kernel_average_time / 1e6, - (readBytes + outputBytes) / kernel_average_time / 1e6, - kernel_average_time); - } -} - -template -int ConvDriver::RunBackwardDataGpuImmed() -{ - std::size_t count; - - bwd_auxiliary.resume(wall_enabled); - auto rc = miopenConvolutionBackwardDataGetSolutionCount( - handle, outputTensor, weightTensor, convDesc, inputTensor, &count); - bwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - if(count < 1) - return miopenStatusNotImplemented; - - auto solutions = std::vector(count); - bwd_auxiliary.resume(wall_enabled); - rc = miopenConvolutionBackwardDataGetSolution( - handle, outputTensor, weightTensor, convDesc, inputTensor, count, &count, solutions.data()); - bwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - std::cout << "Backward Data Conv solutions available: " << count << std::endl; - if(count < 1) - return miopenStatusNotImplemented; - - solutions.resize(count); - const miopenConvSolution_t* selected = nullptr; - - for(const auto& s : solutions) - PrintImmedSolutionInfo(s); - - if(*immediate_solution == 0) - selected = &solutions.front(); - else - for(const auto& s : solutions) - if(*immediate_solution == s.solution_id) - { - selected = &s; - break; - } - - miopenConvSolution_t voluntary = { - -1.0, 0, *immediate_solution, static_cast(-1)}; - if(selected == nullptr) - { - std::cout << "Warning: Solution id (" << *immediate_solution - << ") is not reported by the library. Trying it anyway..." << std::endl; - selected = &voluntary; - } - - std::size_t ws_size; - - bwd_auxiliary.resume(wall_enabled); - bwd_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionBackwardDataGetSolutionWorkspaceSize( - handle, outputTensor, weightTensor, convDesc, inputTensor, selected->solution_id, &ws_size); - bwd_auxiliary_gwss.pause(wall_enabled); - bwd_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto ws = std::unique_ptr{ws_size > 0 ? new GPUMem{ctx, ws_size, 1} : nullptr}; - - bwd_auxiliary.resume(wall_enabled); - rc = miopenConvolutionBackwardDataCompileSolution( - handle, outputTensor, weightTensor, convDesc, inputTensor, selected->solution_id); - bwd_auxiliary.pause(wall_enabled); - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionBackwardDataImmediate(handle, - outputTensor, - dout.GetDevicePtr(), - weightTensor, - wei.GetDevicePtr(), - convDesc, - inputTensor, - din.GetDevicePtr(), - ws ? ws->GetMem() : nullptr, - ws_size, - selected->solution_id); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - bwd_auxiliary.stop(); - bwd_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Backward Data Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << bwd_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << bwd_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - std::cout << "MIOpen Backward Data Conv. " << AlgorithmSolutionToString(*selected) - << std::endl; - PrintBackwardDataTime(kernel_total_time, kernel_first_time); - } - - is_bwd_igemm = (selected->algorithm == miopenConvolutionAlgoImplicitGEMM); - din.CopyFromDeviceToHost(GetStream()); - return rc; -} - -template -int ConvDriver::RunBackwardWrwGpuImmed() -{ - std::size_t count; - wrw_auxiliary.resume(wall_enabled); - auto rc = miopenConvolutionBackwardWeightsGetSolutionCount( - handle, outputTensor, inputTensor, convDesc, weightTensor, &count); - wrw_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - if(count < 1) - return miopenStatusNotImplemented; - - auto solutions = std::vector(count); - wrw_auxiliary.resume(wall_enabled); - rc = miopenConvolutionBackwardWeightsGetSolution( - handle, outputTensor, inputTensor, convDesc, weightTensor, count, &count, solutions.data()); - wrw_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - std::cout << "Backward Weights Conv solutions available: " << count << std::endl; - if(count < 1) - return miopenStatusNotImplemented; - - solutions.resize(count); - const miopenConvSolution_t* selected = nullptr; - - for(const auto& s : solutions) - PrintImmedSolutionInfo(s); - - if(*immediate_solution == 0) - selected = &solutions.front(); - else - for(const auto& s : solutions) - if(*immediate_solution == s.solution_id) - { - selected = &s; - break; - } - - miopenConvSolution_t voluntary = { - -1.0, 0, *immediate_solution, static_cast(-1)}; - if(selected == nullptr) - { - std::cout << "Warning: Solution id (" << *immediate_solution - << ") is not reported by the library. Trying it anyway..." << std::endl; - selected = &voluntary; - } - - std::size_t ws_size; - - wrw_auxiliary.resume(wall_enabled); - wrw_auxiliary_gwss.resume(wall_enabled); - rc = miopenConvolutionBackwardWeightsGetSolutionWorkspaceSize( - handle, outputTensor, inputTensor, convDesc, weightTensor, selected->solution_id, &ws_size); - wrw_auxiliary_gwss.pause(wall_enabled); - wrw_auxiliary.pause(wall_enabled); - if(rc != miopenStatusSuccess) - return rc; - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - auto ws = std::unique_ptr{ws_size > 0 ? new GPUMem{ctx, ws_size, 1} : nullptr}; - - wrw_auxiliary.resume(wall_enabled); - rc = miopenConvolutionBackwardWeightsCompileSolution( - handle, outputTensor, inputTensor, convDesc, weightTensor, selected->solution_id); - wrw_auxiliary.pause(wall_enabled); - - float kernel_total_time = 0.f; - float kernel_first_time = 0.f; - float wall_first_time = 0.f; - - wall.start(wall_enabled); - - for(int i = 0; i < num_iterations; i++) - { - rc = miopenConvolutionBackwardWeightsImmediate(handle, - outputTensor, - dout.GetDevicePtr(), - inputTensor, - in.GetDevicePtr(), - convDesc, - weightTensor, - dwei.GetDevicePtr(), - ws ? ws->GetMem() : nullptr, - ws_size, - selected->solution_id); - if(rc != miopenStatusSuccess) - return rc; - - if(wall_enabled && i == 0) - wall_first_time = wall.interim_time_ms(); - - if(time_enabled) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - } - - if(wall_enabled) - { - wall.stop(); - wrw_auxiliary.stop(); - wrw_auxiliary_gwss.stop(); - std::cout << "Wall-clock Time Backward Weights Conv. Elapsed: " - << ComputeAverageTime(wall.gettime_ms(), wall_first_time) << " ms" - << ", Auxiliary API calls: " << wrw_auxiliary.gettime_ms() << " ms" - << " (GWSS: " << wrw_auxiliary_gwss.gettime_ms() << ')' << std::endl; - } - if(time_enabled) - { - std::cout << "MIOpen Backward Weights Conv. " << AlgorithmSolutionToString(*selected) - << std::endl; - PrintBackwardWrwTime(kernel_total_time, kernel_first_time); - } - - is_wrw_winograd = (selected->algorithm == miopenConvolutionAlgoWinograd); - is_wrw_igemm = (selected->algorithm == miopenConvolutionAlgoImplicitGEMM); - dwei.CopyFromDeviceToHost(GetStream()); - return rc; -} - -template -int ConvDriver::RunBackwardWeightsCPU() -{ - if(mode == miopenTranspose) - { - cpu_convolution_backward_weight(miopen::deref(convDesc).GetSpatialDimension(), - dout.GetTensor(), - dwei_host, - in.GetTensor(), - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - } - else - { - cpu_convolution_backward_weight(miopen::deref(convDesc).GetSpatialDimension(), - in.GetTensor(), - dwei_host, - dout.GetTensor(), - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile( - "dump_bwd_dwei_cpu.bin", dwei_host.data.data(), dwei_host.data.size()); - } - - TrySaveVerificationCache(Direction::WrW, dwei_host.data); - return 0; -} - -template -int ConvDriver::RunBackwardDataCPU() -{ - if(mode == miopenTranspose) - { - cpu_convolution_forward(miopen::deref(convDesc).GetSpatialDimension(), - dout.GetTensor(), - wei.GetTensor(), - din_host, - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - } - else - { - cpu_convolution_backward_data(miopen::deref(convDesc).GetSpatialDimension(), - din_host, - wei.GetTensor(), - dout.GetTensor(), - miopen::deref(convDesc).GetConvPads(), - miopen::deref(convDesc).GetConvStrides(), - miopen::deref(convDesc).GetConvDilations(), - miopen::deref(convDesc).GetGroupCount()); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_din_cpu.bin", din_host.data.data(), din_host.data.size()); - } - - TrySaveVerificationCache(Direction::Bwd, din_host.data); - return 0; -} - -template -int ConvDriver::RunBackwardBiasCPU() -{ - cpu_bias_backward_data(dout.GetTensor(), db_host); - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_db_cpu.bin", db_host.data.data(), db_host.data.size()); - } - - TrySaveVerificationCache(Direction::BwdBias, db_host.data); - return 0; -} - -template -int ConvDriver::RunBackwardWeightsGPUReference() -{ - AutoPrepareForGpuReference naive_conv_enable; - - auto ref_solution_id = miopen::solver::Id("ConvDirectNaiveConvWrw").Value(); - auto rc = miopenConvolutionBackwardWeightsImmediate(handle, - outputTensor, - dout.GetDevicePtr(), - inputTensor, - in.GetDevicePtr(), - convDesc, - weightTensor, - dwei.GetDevicePtr(), - nullptr, - 0, - ref_solution_id); - if(rc != miopenStatusSuccess) - { - std::cout << "reference kernel fail to run " - << miopen::solver::Id(ref_solution_id).ToString() << std::endl; - return rc; - } - - if(miopen_type{} == miopen_type{}) - dwei.CopyFromDeviceToHost(GetStream(), dwei_host); - else - { - if(!is_gpualloc) - { - auto dwei_tmp = tensor(miopen::deref(weightTensor)); - dwei.CopyFromDeviceToHost(GetStream(), dwei_tmp); - for(size_t i = 0; i < dwei_tmp.data.size(); ++i) - { - dwei_host.data[i] = static_cast(dwei_tmp.data[i]); - } - } - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile( - "dump_bwd_dwei_gpu_ref.bin", dwei_host.data.data(), dwei_host.data.size()); - } - - // TrySaveVerificationCache(Direction::WrW, dwei_host.data); - return 0; -} - -template -int ConvDriver::RunBackwardDataGPUReference() -{ - AutoPrepareForGpuReference naive_conv_enable; - - auto ref_solution_id = mode == miopenTranspose // - ? miopen::solver::Id("ConvDirectNaiveConvFwd").Value() - : miopen::solver::Id("ConvDirectNaiveConvBwd").Value(); - auto rc = miopenConvolutionBackwardDataImmediate(handle, - outputTensor, - dout.GetDevicePtr(), - weightTensor, - wei.GetDevicePtr(), - convDesc, - inputTensor, - din.GetDevicePtr(), - nullptr, - 0, - ref_solution_id); - if(rc != miopenStatusSuccess) - { - std::cout << "reference kernel fail to run " - << miopen::solver::Id(ref_solution_id).ToString() << std::endl; - return rc; - } - - if(miopen_type{} == miopen_type{}) - din.CopyFromDeviceToHost(GetStream(), din_host); - else - { - if(!is_gpualloc) - { - auto din_tmp = tensor(miopen::deref(inputTensor)); - din.CopyFromDeviceToHost(GetStream(), din_tmp); - for(size_t i = 0; i < din_tmp.data.size(); ++i) - { - din_host.data[i] = static_cast(din_tmp.data[i]); - } - } - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile( - "dump_bwd_din_gpu_ref.bin", din_host.data.data(), din_host.data.size()); - } - - // TrySaveVerificationCache(Direction::Bwd, din_host.data); - return 0; -} - -template -std::string ConvDriver::GetVerificationCacheFileName( - const ConvDriver::Direction& direction) const -{ - std::ostringstream ss; - - miopenConvolutionMode_t unused; - - int spatial_dim = inflags.GetValueInt("spatial_dim"); - - std::vector pads(spatial_dim); - std::vector conv_strides(spatial_dim); - std::vector conv_dilations(spatial_dim); - std::vector trans_output_pads(spatial_dim); - - miopenGetConvolutionNdDescriptor(convDesc, - spatial_dim, - &spatial_dim, - pads.data(), - conv_strides.data(), - conv_dilations.data(), - &unused); - - auto get_basename_string = [&]() { - switch(direction) - { - case Direction::Fwd: return "conv_fwd_out"; - case Direction::Bwd: return "conv_bwd_dat"; - case Direction::WrW: return "conv_bwd_wei"; - case Direction::BwdBias: return "bias_bwd_dat"; - } - return ""; // For gcc. - }; - - auto get_datatype_string = [](auto type) { - if(std::is_same::value) - { - return "int8"; - } - if(std::is_same::value) - { - return "int32"; - } - else if(std::is_same::value) - { - return "float16"; - } - else if(std::is_same::value) - { - return "float"; - } - else if(std::is_same::value) - { - return "double"; - } - else if(std::is_same::value) - { - return "bfloat16"; - } - else - { - MIOPEN_THROW("unknown data type"); - } - }; - - ss << get_basename_string(); - ss << "_" << mode; - ss << "_" << spatial_dim; - ss << "_" << miopen::deref(convDesc).paddingMode; - ss << "_" << miopen::deref(convDesc).GetGroupCount(); - miopen::LogRange(ss << "_", miopen::deref(inputTensor).GetLengths(), "x"); - miopen::LogRange(ss << "_", miopen::deref(weightTensor).GetLengths(), "x"); - miopen::LogRange(ss << "_", pads, "x"); - miopen::LogRange(ss << "_", conv_strides, "x"); - miopen::LogRange(ss << "_", conv_dilations, "x"); - miopen::LogRange(ss << "_", trans_output_pads, "x"); - ss << "_" << inflags.GetValueInt("pad_val"); - ss << "_" << inflags.GetValueInt("bias"); - ss << "_" - << "GPU" << get_datatype_string(Tgpu{}); - ss << "_" - << "REF" << get_datatype_string(Tref{}); - - return ss.str(); -} - -template -bool ConvDriver::TryReadVerificationCache( - const ConvDriver::Direction& direction, - miopenTensorDescriptor_t& tensorDesc, - Tref* data) const -{ - const auto verification_cache_path = inflags.GetValueStr("verification_cache"); - - if(!verification_cache_path.empty()) - { - const auto file_path = - verification_cache_path + "/" + GetVerificationCacheFileName(direction); - - if(std::ifstream(file_path).good()) - { - if(readBufferFromFile(data, GetTensorSize(tensorDesc), file_path.c_str())) - { - return true; - } - } - } - - return false; -} - -template -void ConvDriver::TrySaveVerificationCache( - const ConvDriver::Direction& direction, std::vector& data) const -{ - const auto verification_cache_path = inflags.GetValueStr("verification_cache"); - if(!verification_cache_path.empty()) - { - const auto file_path = - verification_cache_path + "/" + GetVerificationCacheFileName(direction); - dumpBufferToFile(file_path.c_str(), data.data(), data.size()); - } -} - -template -int ConvDriver::VerifyForward() -{ - if(!is_fwd) - return 0; - - MIOPEN_THROW_IF(is_gpualloc, "'-G 1' and '-V 1' are incompatible"); - - if(!is_fwd_run_failed) - if(!TryReadVerificationCache(Direction::Fwd, outputTensor, outhost.data.data())) - { - if(UseGPUReference()) - RunForwardGPUReference(); - else - RunForwardCPU(); - } - - const auto isInt8 = (data_type == miopenInt8 || data_type == miopenInt8x4); - auto error = is_fwd_run_failed ? std::numeric_limits::max() - : (isInt8 ? miopen::rms_range(outhost.data, out_int8) - : miopen::rms_range(outhost.data, out.GetVector())); - - auto tolerance = GetDefaultTolerance(); - // iGemm's deviation is higher than other algorithms. - // The reason is most likely different order of computations. - if(is_fwd_igemm) - tolerance = tolerance * 10; - - if(!std::isfinite(error) || error > tolerance) - { - std::cout << "Forward Convolution FAILED: " << error << " > " << tolerance << std::endl; - return EC_VerifyFwd; - } - - std::cout << "Forward Convolution Verifies OK on " << (UseGPUReference() ? "GPU" : "CPU") - << " reference (" << error << " < " << tolerance << ')' << std::endl; - - return 0; -} - -template -int ConvDriver::VerifyBackward() -{ - if(data_type == miopenInt8 || data_type == miopenInt8x4) - { - std::cout << "Int8 Backward Convolution is not supported" << std::endl; - return 0; - } - - if(!(is_bwd || is_wrw)) - return 0; - - MIOPEN_THROW_IF(is_gpualloc, "'-G 1' and '-V 1' are incompatible"); - - int cumulative_rc = 0; - if(is_bwd) - { - if(!is_bwd_run_failed) - if(!TryReadVerificationCache(Direction::Bwd, inputTensor, din_host.data.data())) - { - if(UseGPUReference()) - RunBackwardDataGPUReference(); - else - RunBackwardDataCPU(); - } - - auto error_data = is_bwd_run_failed ? std::numeric_limits::max() - : miopen::rms_range(din_host.data, din.GetVector()); - - auto tolerance = GetDefaultTolerance(); - // iGemm's deviation is higher than other algorithms. - // The reason is most likely different order of computations. - if(is_bwd_igemm) - tolerance = tolerance * 10; - - if(!std::isfinite(error_data) || error_data > tolerance) - { - std::cout << "Backward Convolution Data FAILED: " << error_data << " > " << tolerance - << std::endl; - cumulative_rc |= EC_VerifyBwd; - } - else - { - std::cout << "Backward Convolution Data Verifies OK on " - << (UseGPUReference() ? "GPU" : "CPU") << " reference (" << error_data - << " < " << tolerance << ')' << std::endl; - } - } - - if(is_wrw) - { - if(!is_wrw_run_failed) - if(!TryReadVerificationCache(Direction::WrW, weightTensor, dwei_host.data.data())) - { - if(UseGPUReference()) - RunBackwardWeightsGPUReference(); - else - RunBackwardWeightsCPU(); - } - - // WrW deviation is ~twice worse than Bwd due to more FP computations involved, - // which means more roundings, so GPU amd CPU computations diverge more. - auto tolerance = 2 * GetDefaultTolerance(); - - // fp32 transposed convolutions show worse precision. - if(mode == miopenTranspose && std::is_same::value) - tolerance *= 2; - - // Winograd and iGemm WrW algorithms reveal bigger deviation than other algos. - if(is_wrw_winograd && std::is_same::value) - { - tolerance *= 10; - } - else if(is_wrw_igemm) - { - if(std::is_same::value) -#if WORKAROUND_ISSUE_2176 - tolerance = 0.01; -#else - tolerance *= 10; -#endif - else if(std::is_same::value) - tolerance *= 5; - } - // bfloat8 has very poor accuracy in wrw direction - if(std::is_same::value) - tolerance = tolerance * 2; - - auto error_weights = is_wrw_run_failed - ? std::numeric_limits::max() - : miopen::rms_range(dwei_host.data, dwei.GetVector()); - - if(!std::isfinite(error_weights) || error_weights > tolerance) - { - std::cout << "Backward Convolution Weights FAILED: " << error_weights << " > " - << tolerance << std::endl; - cumulative_rc |= EC_VerifyWrw; - } - else - { - std::cout << "Backward Convolution Weights Verifies OK on " - << (UseGPUReference() ? "GPU" : "CPU") << " reference (" << error_weights - << " < " << tolerance << ')' << std::endl; - } - } - - if(inflags.GetValueInt("bias") != 0) - { - if(!TryReadVerificationCache(Direction::BwdBias, biasTensor, db_host.data.data())) - { - RunBackwardBiasCPU(); - } - - auto error_bias = miopen::rms_range(db_host.data, db.GetVector()); - const auto tolerance = GetDefaultTolerance(); - if(!std::isfinite(error_bias) || error_bias > tolerance) - { - std::cout << "Backward Convolution Bias FAILED: " << error_bias << " > " << tolerance - << std::endl; - cumulative_rc |= EC_VerifyBwdBias; - } - else - { - std::cout << "Backward Convolution Bias Verifies OK on " - << (UseGPUReference() ? "GPU" : "CPU") << " reference (" << error_bias << ')' - << std::endl; - } - } - - return cumulative_rc; -} - -#endif // GUARD_MIOPEN_CONV_DRIVER_HPP diff --git a/driver/cumulative_reduction_driver.hpp b/driver/cumulative_reduction_driver.hpp new file mode 100644 index 0000000000..5e9cee571c --- /dev/null +++ b/driver/cumulative_reduction_driver.hpp @@ -0,0 +1,457 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "driver.hpp" +#include "mloCumulativeReductionHost.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" + +#include <../test/ford.hpp> +#include <../test/verify.hpp> + +#include + +inline std::vector GetStrides(std::vector lengths, bool contiguous) +{ + if(!contiguous) + std::swap(lengths.front(), lengths.back()); + std::vector strides(lengths.size()); + strides.back() = 1; + for(int i = lengths.size() - 2; i >= 0; --i) + strides[i] = strides[i + 1] * lengths[i + 1]; + if(!contiguous) + std::swap(strides.front(), strides.back()); + return strides; +} + +template +class CumulativeReductionDriver : public Driver +{ +public: + CumulativeReductionDriver() : Driver() + { + miopenCreateTensorDescriptor(&inputDesc); + miopenCreateTensorDescriptor(&outputDesc); + miopenCreateTensorDescriptor(&indicesDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + Tref GetTolerance(); + int VerifyBackward() override; + int VerifyForward() override; + ~CumulativeReductionDriver() override + { + miopenDestroyTensorDescriptor(inputDesc); + miopenDestroyTensorDescriptor(outputDesc); + miopenDestroyTensorDescriptor(indicesDesc); + } + +private: + InputFlags inflags; + + int forw; + + miopenTensorDescriptor_t inputDesc; + miopenTensorDescriptor_t outputDesc; + miopenTensorDescriptor_t indicesDesc; + + std::unique_ptr input_dev; + std::unique_ptr output_dev; + std::unique_ptr indices_dev; + + std::vector input; + std::vector output; + std::vector indices; + + std::vector output_host; + std::vector indices_host; + + int dim; + bool exclusive; + bool reverse; + + miopenCumOp_t cumOp; +}; + +template +int CumulativeReductionDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + + auto inTensorParam = inflags.GetValueTensor("input"); + auto input_length = inTensorParam.lengths; + if(input_length.empty()) + { + std::cout << "Tensor must not be empty"; + return miopenStatusBadParm; + } + + int contiguous = inflags.GetValueInt("Contiguous"); + if(contiguous != 0 && contiguous != 1) + { + std::cerr << "Error Tensor Contiguous should be 0 or 1" << std::endl; + return miopenStatusBadParm; + } + + std::vector cumOpList = { + MIOPEN_CUM_MAX, MIOPEN_CUM_MIN, MIOPEN_CUM_SUM, MIOPEN_CUM_PROD}; + int cumOpInt = inflags.GetValueInt("CumulativeOperation"); + bool valid = true; + for(auto op : cumOpList) + if(cumOpInt != static_cast(op)) + { + valid = false; + break; + } + if(valid) + { + std::cerr << "Error CumulativeOperation value should be in set {" << cumOpList << "}" + << std::endl; + return miopenStatusBadParm; + } + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::GetandSetData() +{ + dim = inflags.GetValueInt("dim"); + exclusive = (inflags.GetValueInt("exclusive") != 0); + reverse = (inflags.GetValueInt("reverse") != 0); + cumOp = (miopenCumOp_t)inflags.GetValueInt("CumulativeOperation"); + + auto lengths = inflags.GetValueTensor("input").lengths; + auto strides = GetStrides(lengths, inflags.GetValueInt("Contiguous") != 0); + + if(SetTensorNd(inputDesc, lengths, strides, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing input tensor: " + inflags.GetValueStr("input") + "."); + + if(SetTensorNd(outputDesc, lengths, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing output tensor"); + + if(SetTensorNd(indicesDesc, lengths, miopen_type{}) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing indices tensor"); + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag( + "forw", 'F', "1", "Run only Forward CumulativeReduction (Default=1)", "int"); + inflags.AddTensorFlag("input", 'D', "256x4x256", "input tensor descriptor"); + inflags.AddInputFlag( + "dim", 'd', "0", "The dimension to do the operation over (Default=0)", "int"); + inflags.AddInputFlag("exclusive", + 'e', + "0", + "Enable exclusive calculation. 0 for False, 1 for True (Default=0)", + "int"); + inflags.AddInputFlag( + "reverse", + 'r', + "0", + "Reverse the calculation order to back to front. 0 for False, 1 for True (Default=0)", + "int"); + inflags.AddInputFlag("CumulativeOperation", + 'O', + "1", + "Operator used. 1 for Max, 2 for Min, 3 for Sum, 4 for Prod (Default=1)", + "int"); + inflags.AddInputFlag("Contiguous", + 'C', + "1", + "Is input tensor contiguous? (Default=1 for contiguous tensor)", + "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "0", "Verify Each Layer (Default=0)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::AllocateBuffersAndCopy() +{ + size_t input_sz = miopen::deref(inputDesc).GetElementSpace(); + size_t output_sz = miopen::deref(outputDesc).GetElementSpace(); + size_t indices_sz = miopen::deref(indicesDesc).GetElementSpace(); + + uint32_t ctx = 0; + + input_dev = std::unique_ptr(new GPUMem(ctx, input_sz, sizeof(Tgpu))); + output_dev = std::unique_ptr(new GPUMem(ctx, output_sz, sizeof(Tgpu))); + indices_dev = std::unique_ptr(new GPUMem(ctx, indices_sz, sizeof(int))); + + input = std::vector(input_sz); + output = std::vector(output_sz, static_cast(0.0f)); + indices = std::vector(indices_sz, static_cast(-1)); + + output_host = std::vector(output_sz, static_cast(0.0f)); + indices_host = std::vector(indices_sz, static_cast(-1)); + + for(int i = 0; i < input_sz; i++) + input[i] = prng::gen_A_to_B(static_cast(-100), static_cast(100)); + + if(input_dev->ToGPU(GetStream(), input.data()) != 0) + { + std::cerr << "Error copying (input) to GPU, size: " << input_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(output_dev->ToGPU(GetStream(), output.data()) != 0) + { + std::cerr << "Error copying (output) to GPU, size: " << output_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(indices_dev->ToGPU(GetStream(), indices.data()) != 0) + { + std::cerr << "Error copying (indices) to GPU, size: " << indices_dev->GetSize() + << std::endl; + return miopenStatusAllocFailed; + } + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::RunForwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenCumulativeReductionForward( + GetHandle(), + inputDesc, + input_dev->GetMem(), + outputDesc, + output_dev->GetMem(), + indicesDesc, + (cumOp == MIOPEN_CUM_MAX || cumOp == MIOPEN_CUM_MIN ? indices_dev->GetMem() : nullptr), + dim, + exclusive, + reverse, + cumOp); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Forward Cumulative Reduction Elapsed: " + << t.gettime_ms() / iter << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Forward Cumulative Reduction Elapsed: " << kernel_average_time + << " ms" << std::endl; + } + + if(output_dev->FromGPU(GetStream(), output.data()) != 0) + { + std::cerr << "Error copying (output_dev) from GPU, size: " << output_dev->GetSize() + << std::endl; + return miopenStatusInternalError; + } + if(indices_dev->FromGPU(GetStream(), indices.data()) != 0) + { + std::cerr << "Error copying (indices_dev) from GPU, size: " << indices_dev->GetSize() + << std::endl; + return miopenStatusInternalError; + } + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::RunForwardCPU() +{ + int32_t mloStatus = miopenStatusSuccess; + + switch(cumOp) + { + case MIOPEN_CUM_MAX: + mloStatus = + mloCumulativeReductionForwardRunHost(inputDesc, + outputDesc, + indicesDesc, + input.data(), + output_host.data(), + indices_host.data(), + dim, + exclusive, + reverse); + break; + case MIOPEN_CUM_MIN: + mloStatus = + mloCumulativeReductionForwardRunHost(inputDesc, + outputDesc, + indicesDesc, + input.data(), + output_host.data(), + indices_host.data(), + dim, + exclusive, + reverse); + break; + case MIOPEN_CUM_SUM: + mloStatus = + mloCumulativeReductionForwardRunHost(inputDesc, + outputDesc, + indicesDesc, + input.data(), + output_host.data(), + nullptr, + dim, + exclusive, + reverse); + break; + case MIOPEN_CUM_PROD: + mloStatus = + mloCumulativeReductionForwardRunHost(inputDesc, + outputDesc, + indicesDesc, + input.data(), + output_host.data(), + nullptr, + dim, + exclusive, + reverse); + break; + default: + std::cout << "The CPU version of Cumulative Reduction with Operation code " << cumOp + << " has not been implemented" << std::endl; + mloStatus = miopenStatusNotImplemented; + break; + } + + return mloStatus; +} + +template +int CumulativeReductionDriver::RunBackwardGPU() +{ + return miopenStatusNotImplemented; +} + +template +int CumulativeReductionDriver::RunBackwardCPU() +{ + return miopenStatusNotImplemented; +} + +template +Tref CumulativeReductionDriver::GetTolerance() +{ + // Computation error of fp16 is ~2^13 (=8192) bigger than + // the one of fp32 because mantissa is shorter by 13 bits. + auto tolerance = std::is_same::value ? 1.5e-6 : 8.2e-3; + + // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. + if(std::is_same::value) + tolerance *= 8.0; + return tolerance; +} + +template +int CumulativeReductionDriver::VerifyForward() +{ + RunForwardCPU(); + const Tref tolerance = GetTolerance(); + auto error_output = miopen::rms_range(output_host, output); + auto error_indices = miopen::rms_range(indices_host, indices); + + if(!std::isfinite(error_output) || error_output > tolerance) + { + std::cout << "Forward Cumulative Reduction Output FAILED: " << error_output << " > " + << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward Cumulative Reduction Output Verifies OK on CPU reference (" + << error_output << " < " << tolerance << ')' << std::endl; + } + + if(!std::isfinite(error_indices) || error_indices > tolerance) + { + std::cout << "Forward Cumulative Reduction Indices FAILED: " << error_indices << " > " + << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward Cumulative Reduction Indices Verifies OK on CPU reference (" + << error_indices << " < " << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int CumulativeReductionDriver::VerifyBackward() +{ + return miopenStatusNotImplemented; +} diff --git a/driver/dm_cumulative_reduction.cpp b/driver/dm_cumulative_reduction.cpp new file mode 100644 index 0000000000..09da7b45b5 --- /dev/null +++ b/driver/dm_cumulative_reduction.cpp @@ -0,0 +1,41 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "registry_driver_maker.hpp" +#include "cumulative_reduction_driver.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "cum") + return new CumulativeReductionDriver(); + if(base_arg == "cumfp16") + return new CumulativeReductionDriver(); + if(base_arg == "cumbfp16") + return new CumulativeReductionDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_glu.cpp b/driver/dm_glu.cpp index 666ccf03da..e69de29bb2 100644 --- a/driver/dm_glu.cpp +++ b/driver/dm_glu.cpp @@ -1,40 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include "registry_driver_maker.hpp" -#include "glu_driver.hpp" - -static Driver* makeDriver(const std::string& base_arg) -{ - if(base_arg == "glu") - return new GLUDriver(); - if(base_arg == "glufp16") - return new GLUDriver(); - if(base_arg == "glubfp16") - return new GLUDriver(); - return nullptr; -} - -REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_kthvalue.cpp b/driver/dm_kthvalue.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/driver/driver.hpp b/driver/driver.hpp index aa0b89f10a..e69de29bb2 100644 --- a/driver/driver.hpp +++ b/driver/driver.hpp @@ -1,323 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_DRIVER_HPP -#define GUARD_MIOPEN_DRIVER_HPP - -#include -#include "random.hpp" - -#include "InputFlags.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -using half = half_float::half; -using hip_bfloat16 = bfloat16; -#include -using float16 = half_float::half; -using float8 = miopen_f8::hip_f8; -using bfloat8 = miopen_f8::hip_f8; -#include -#include - -#if MIOPEN_BACKEND_OPENCL -#if defined(__APPLE__) || defined(__MACOSX) -#include -#else -#include -#endif -#elif MIOPEN_BACKEND_HIP -#include -#endif - -#define UNPACK_VEC4(v) (v[0]), (v[1]), (v[2]), (v[3]) - -// Use values which are distinctively greater then miopenStatus_t, -// so that these can be ORed with any miopen status code -// without loss of information. -typedef enum -{ - // These four codes could be returned together, ORed: - EC_VerifyFwd = 0x100, - EC_VerifyBwd = 0x200, - EC_VerifyWrw = 0x400, - EC_VerifyBwdBias = 0x800, -} errorCode_t; - -struct GPUMem -{ - -#if MIOPEN_BACKEND_OPENCL - GPUMem(){}; - GPUMem(cl_context& ctx, size_t psz, size_t pdata_sz) : sz(psz), data_sz(pdata_sz) - { - buf = clCreateBuffer(ctx, CL_MEM_READ_WRITE, data_sz * sz, nullptr, nullptr); - } - - int ToGPU(cl_command_queue& q, void* p) const - { - return clEnqueueWriteBuffer(q, buf, CL_TRUE, 0, data_sz * sz, p, 0, nullptr, nullptr); - } - int FromGPU(cl_command_queue& q, void* p) const - { - return clEnqueueReadBuffer(q, buf, CL_TRUE, 0, data_sz * sz, p, 0, nullptr, nullptr); - } - - cl_mem GetMem() const { return buf; } - size_t GetSize() const { return sz * data_sz; } - - ~GPUMem() { clReleaseMemObject(buf); } - - cl_mem buf; - size_t sz; - size_t data_sz; - -#elif MIOPEN_BACKEND_HIP - - GPUMem(){}; - GPUMem(uint32_t ctx, size_t psz, size_t pdata_sz) : _ctx(ctx), sz(psz), data_sz(pdata_sz) - { - auto status = hipMalloc(static_cast(&buf), GetSize()); - if(status != hipSuccess) - MIOPEN_THROW_HIP_STATUS(status, - "[MIOpenDriver] hipMalloc " + std::to_string(GetSize())); - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Info2, - "MIOpenDriver", - "hipMalloc " << GetSize() << " at " << buf << " Ok"); - } - - int ToGPU(hipStream_t q, void* p) - { - _q = q; - return static_cast(hipMemcpy(buf, p, GetSize(), hipMemcpyHostToDevice)); - } - int FromGPU(hipStream_t q, void* p) - { - hipDeviceSynchronize(); - _q = q; - return static_cast(hipMemcpy(p, buf, GetSize(), hipMemcpyDeviceToHost)); - } - - void* GetMem() { return buf; } - size_t GetSize() { return sz * data_sz; } - - ~GPUMem() - { - size_t size = 0; - auto status = hipMemPtrGetInfo(buf, &size); - if(status != hipSuccess) - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Warning, - "MIOpenDriver", - "hipMemPtrGetInfo at " << buf << ' ' - << miopen::HIPErrorMessage(status, "")); - status = hipFree(buf); - if(status != hipSuccess) - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Error, - "MIOpenDriver", - "hipFree " << size << " at " << buf << ' ' - << miopen::HIPErrorMessage(status, "")); - else - MIOPEN_LOG_CUSTOM(miopen::LoggingLevel::Info2, - "MIOpenDriver", - "hipFree " << size << " at " << buf << " Ok"); - } - - hipStream_t _q; // Place holder for opencl context - uint32_t _ctx; - void* buf; - size_t sz; - size_t data_sz; -#endif -}; - -inline void PadBufferSize(size_t& sz, int datatype_sz) -{ - size_t page_sz = (2 * 1024 * 1024) / datatype_sz; - if(sz % page_sz != 0) - { - sz = ((sz + page_sz) / page_sz) * page_sz; - } -} - -[[noreturn]] inline void Usage() -{ - printf("Usage: ./driver *base_arg* *other_args*\n"); - printf("Supported Base Arguments: conv[fp16|int8|bfp16], pool[fp16], lrn[fp16], " - "activ[fp16], softmax[fp16], bnorm[fp16], rnn[fp16], gemm[fp16], ctc, dropout[fp16], " - "tensorop, reduce[fp16|fp64], layernorm[bfp16|fp16], sum[bfp16|fp16], " - "groupnorm[bfp16|fp16], cat[bfp16|fp16], addlayernorm[bfp16|fp16], " - "t5layernorm[bfp16|fp16], adam[fp16], ampadam, reduceextreme[bfp16|fp16], " - "adamw[fp16], ampadamw, transformersadamw[fp16], transformersampadamw, " - "getitem[bfp16|fp16], reducecalculation[bfp16|fp16], rope[bfp16|fp16], " - "prelu[bfp16|fp16], glu[bfp16|fp16]\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) -} - -inline std::string ParseBaseArg(int argc, char* argv[]) -{ - if(argc < 2) - { - printf("FAILED: Invalid Number of Input Arguments\n"); - Usage(); - } - - std::string arg = argv[1]; - - if(arg != "conv" && arg != "convfp16" && arg != "convint8" && arg != "convbfp16" && - arg != "pool" && arg != "poolfp16" && arg != "lrn" && arg != "lrnfp16" && arg != "activ" && - arg != "activfp16" && arg != "softmax" && arg != "softmaxfp16" && arg != "bnorm" && - arg != "bnormfp16" && arg != "rnn" && arg != "rnnfp16" && arg != "rnn_seq" && - arg != "rnn_seqfp16" && arg != "gemm" && arg != "gemmfp16" && arg != "ctc" && - arg != "dropout" && arg != "dropoutfp16" && arg != "tensorop" && arg != "reduce" && - arg != "reducefp16" && arg != "reducefp64" && arg != "layernorm" && arg != "layernormfp16" && - arg != "layernormbfp16" && arg != "sum" && arg != "sumfp16" && arg != "sumbfp16" && - arg != "groupnorm" && arg != "groupnormfp16" && arg != "groupnormbfp16" && arg != "cat" && - arg != "catfp16" && arg != "catbfp16" && arg != "addlayernorm" && - arg != "addlayernormfp16" && arg != "addlayernormbfp16" && arg != "t5layernorm" && - arg != "t5layernormfp16" && arg != "t5layernormbfp16" && arg != "adam" && - arg != "adamfp16" && arg != "ampadam" && arg != "reduceextreme" && - arg != "reduceextremefp16" && arg != "reduceextremebfp16" && arg != "adamw" && - arg != "adamwfp16" && arg != "ampadamw" && arg != "transformersadamw" && - arg != "transformersadamwfp16" && arg != "transformersampadamw" && arg != "getitem" && - arg != "getitemfp16" && arg != "getitembfp16" && arg != "reducecalculation" && - arg != "reducecalculationfp16" && arg != "reducecalculationbfp16" && arg != "rope" && - arg != "ropefp16" && arg != "ropebfp16" && arg != "prelu" && arg != "prelufp16" && - arg != "prelubfp16" && arg != "glu" && arg != "glufp16" && arg != "glubfp16" && - arg != "--version") - { - printf("FAILED: Invalid Base Input Argument\n"); - Usage(); - } - else if(arg == "-h" || arg == "--help" || arg == "-?") - Usage(); - else - return arg; -} - -class Driver -{ -public: - Driver() - { - data_type = miopenFloat; -#if MIOPEN_BACKEND_OPENCL - miopenCreate(&handle); -#elif MIOPEN_BACKEND_HIP - hipStream_t s; - hipStreamCreate(&s); - miopenCreateWithStream(&handle, s); -#endif - - miopenGetStream(handle, &q); - } - - miopenHandle_t GetHandle() { return handle; } - miopenDataType_t GetDataType() { return data_type; } - -#if MIOPEN_BACKEND_OPENCL - cl_command_queue& GetStream() { return q; } -#elif MIOPEN_BACKEND_HIP - hipStream_t& GetStream() { return q; } -#endif - virtual ~Driver() { miopenDestroy(handle); } - - // TODO: add timing APIs - virtual int AddCmdLineArgs() = 0; - virtual int ParseCmdLineArgs(int argc, char* argv[]) = 0; - virtual InputFlags& GetInputFlags() = 0; - virtual int GetandSetData() = 0; - virtual int AllocateBuffersAndCopy() = 0; - virtual int RunForwardGPU() = 0; - virtual int VerifyForward() = 0; - virtual int RunBackwardGPU() = 0; - virtual int VerifyBackward() = 0; - -protected: - template - void InitDataType(); - miopenHandle_t handle; - miopenDataType_t data_type; - -#if MIOPEN_BACKEND_OPENCL - cl_command_queue q; -#elif MIOPEN_BACKEND_HIP - hipStream_t q; -#endif -}; - -template <> -inline void Driver::InitDataType() -{ - data_type = miopenInt8; -} -template <> -inline void Driver::InitDataType() -{ - data_type = miopenFloat; -} -template <> -inline void Driver::InitDataType() -{ - data_type = miopenHalf; -} -template <> -inline void Driver::InitDataType() -{ - data_type = miopenBFloat16; -} -template <> -inline void Driver::InitDataType() -{ - data_type = miopenFloat8; -} -template <> -inline void Driver::InitDataType() -{ - data_type = miopenBFloat8; -} -// "std::is_same{}" used to avoid "static_assert" compilation error, -// which occurs when the condition does not depend in any way on the template parameters. -template -inline void Driver::InitDataType() -{ - static_assert(std::is_same{}, "unsupported Tgpu"); -} - -template -inline std::ostream& operator<<(std::ostream& os, const std::vector& vs) -{ - os << "{ size: " << vs.size() << ", entries: "; - for(auto& v : vs) - os << v << " "; - os << "}"; - return os; -} - -#endif // GUARD_MIOPEN_DRIVER_HPP diff --git a/driver/dropout_driver.hpp b/driver/dropout_driver.hpp index 0215c8c64c..e69de29bb2 100644 --- a/driver/dropout_driver.hpp +++ b/driver/dropout_driver.hpp @@ -1,482 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_DROPOUT_DRIVER_HPP -#define GUARD_MIOPEN_DROPOUT_DRIVER_HPP - -#include "InputFlags.hpp" -#include "driver.hpp" -#include "dropout_gpu_emulator.hpp" -#include "tensor_driver.hpp" -#include "timer.hpp" -#include "util_driver.hpp" -#include "util_file.hpp" - -#include <../test/verify.hpp> - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -template -class DropoutDriver : public Driver -{ -public: - DropoutDriver() : Driver() - { - miopenCreateTensorDescriptor(&inputTensor); - miopenCreateTensorDescriptor(&outputTensor); - - miopenCreateDropoutDescriptor(&DropoutDesc); - reservespace_dev = nullptr; - data_type = std::is_same{} ? miopenHalf : miopenFloat; - } - - int AddCmdLineArgs() override; - int ParseCmdLineArgs(int argc, char* argv[]) override; - InputFlags& GetInputFlags() override { return inflags; } - - int GetandSetData() override; - std::vector GetInputTensorLengthsFromCmdLine(std::string input_str); - - int AllocateBuffersAndCopy() override; - - int RunForwardGPU() override; - int RunForwardCPU(); - int RunBackwardGPU() override; - int RunBackwardCPU(); - int VerifyForward() override; - int VerifyBackward() override; - - ~DropoutDriver() override - { - miopenDestroyTensorDescriptor(inputTensor); - miopenDestroyTensorDescriptor(outputTensor); - - miopenDestroyDropoutDescriptor(DropoutDesc); - } - -private: - InputFlags inflags; - - miopenTensorDescriptor_t inputTensor; - miopenTensorDescriptor_t outputTensor; - - std::unique_ptr in_dev; - std::unique_ptr out_dev; - std::unique_ptr dout_dev; - std::unique_ptr din_dev; - std::unique_ptr reservespace_dev; - std::unique_ptr states_dev; - - tensor in; - tensor out; - tensor dout; - tensor din; - tensor outhost; - tensor din_host; - - std::vector states_host; - std::vector reservespace; - std::vector reservespace_host; - - miopenDropoutDescriptor_t DropoutDesc; - - float dropout; - unsigned long long seed; - bool use_mask; -}; - -template -int DropoutDriver::ParseCmdLineArgs(int argc, char* argv[]) -{ - inflags.Parse(argc, argv); - - if(inflags.GetValueInt("time") == 1) - { - miopenEnableProfiling(GetHandle(), true); - } - return miopenStatusSuccess; -} - -template -int DropoutDriver::GetandSetData() -{ - std::vector in_len = GetInputTensorLengthsFromCmdLine(inflags.GetValueStr("input_dim")); - SetTensorNd(inputTensor, in_len, data_type); - SetTensorNd(outputTensor, in_len, data_type); - - dropout = static_cast(inflags.GetValueDouble("dropout")); - use_mask = static_cast(inflags.GetValueInt("use_mask")); - - auto seed_low = static_cast(std::max(inflags.GetValueInt("seed_low"), 0)); - auto seed_high = static_cast(std::max(inflags.GetValueInt("seed_high"), 0)); - seed = seed_high << 32 | seed_low; - - return miopenStatusSuccess; -} - -template -int DropoutDriver::AddCmdLineArgs() -{ - inflags.AddInputFlag( - "forw", 'F', "0", "Direction, Forward = 1, Backward = 2 , Both = 0 (Default=0)", "int"); - inflags.AddInputFlag( - "input_dim", 'd', "4", "Input dimension (Default=4, support up to 5D)", "vector"); - inflags.AddInputFlag("dropout", 'p', "0.5", "Dropout rate (Default=0.5)", "float"); - inflags.AddInputFlag( - "seed_low", 'l', "0", "Least significant 32 bits of seed (Default=0)", "int"); - inflags.AddInputFlag( - "seed_high", 'm', "0", "Most significant 32 bits of seed (Default=0)", "int"); - inflags.AddInputFlag("use_mask", - 'e', - "0", - "Use existing mask in reservespace: Use 1, Not use 0 (Default=0)", - "int"); - inflags.AddInputFlag( - "gen_file", - 'f', - "0", - "Generate and write/overwrite PRNG skipahead files (1), No operation (0) (Default=0)", - "int"); - inflags.AddInputFlag("iter", 'i', "1", "Number of Iterations (Default=1)", "int"); - inflags.AddInputFlag("verify", 'V', "1", "Verify Dropout (Default=1)", "int"); - inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); - inflags.AddInputFlag( - "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); - inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); - - return 0; -} - -template -std::vector DropoutDriver::GetInputTensorLengthsFromCmdLine(std::string input_str) -{ - std::vector in_lens; - std::stringstream ss(input_str); - - int cont = 0; - int element; - - while(ss >> element) - { - if(cont++ >= 5) - { - std::cout << "Only support up to 5D-tensor dropout" << std::endl; - break; - } - - if(ss.peek() == ',' || ss.peek() == ' ') - { - ss.ignore(); - } - - in_lens.push_back(element); - } - - return in_lens; -} - -template -int DropoutDriver::AllocateBuffersAndCopy() -{ - size_t in_sz = GetTensorSize(inputTensor); - size_t out_sz = GetTensorSize(outputTensor); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(inputTensor, &reserveSpaceSizeInBytes); - size_t reserveSpaceSize = reserveSpaceSizeInBytes / sizeof(unsigned char); - - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - states_dev = - std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); - - // if(inflags.GetValueInt("gen_file")) - // generate_skipahead_file(); - - miopenSetDropoutDescriptor(DropoutDesc, - GetHandle(), - dropout, - states_dev->GetMem(), - states_dev->GetSize(), - seed, - use_mask, - false, - MIOPEN_RNG_PSEUDO_XORWOW); - - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - din_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - dout_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - - reservespace_dev = - std::unique_ptr(new GPUMem(ctx, reserveSpaceSize, sizeof(unsigned char))); - - in = tensor(miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides()); - din = tensor(miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides()); - out = tensor(miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides()); - dout = tensor(miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides()); - - outhost = tensor(miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides()); - din_host = tensor(miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides()); - - reservespace = std::vector(reserveSpaceSize, static_cast(1)); - reservespace_host = std::vector(reserveSpaceSize, static_cast(1)); - - states_host = std::vector(states_size); - - Tgpu Data_scale = static_cast(0.01); - - for(int i = 0; i < in_sz; i++) - { - in.data[i] = prng::gen_0_to_B(Data_scale); - } - - for(int i = 0; i < out_sz; i++) - { - dout.data[i] = prng::gen_0_to_B(Data_scale); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_in.bin", in.data.data(), in_sz); - dumpBufferToFile("dump_dout.bin", dout.data.data(), out_sz); - } - - status_t status; - status = in_dev->ToGPU(q, in.data.data()); - status |= din_dev->ToGPU(q, din.data.data()); - status |= out_dev->ToGPU(q, out.data.data()); - status |= dout_dev->ToGPU(q, dout.data.data()); - - if(inflags.GetValueInt("use_mask") == 1) - { - for(int i = 0; i < reserveSpaceSize; i++) - { - reservespace[i] = static_cast(prng::gen_canonical() > dropout); - reservespace_host[i] = reservespace[i]; - } - status |= reservespace_dev->ToGPU(q, reservespace.data()); - } - - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - - return miopenStatusSuccess; -} - -template -int DropoutDriver::RunForwardGPU() -{ - float kernel_total_time = 0.0; - float kernel_first_time = 0.0; - - Timer t; - START_TIME - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - miopenDropoutForward(GetHandle(), - DropoutDesc, - inputTensor, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - - if(inflags.GetValueInt("time") == 1) - { - STOP_TIME - if(WALL_CLOCK) - printf("Wall-clock Time Dropout Elapsed: %f ms\n", - t.gettime_ms() / inflags.GetValueInt("iter")); - - int iter = inflags.GetValueInt("iter"); - float kernel_average_time = - iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; - printf("GPU Kernel Time Forward Dropout. Elapsed: %f ms (average)\n", kernel_average_time); - } - - out_dev->FromGPU(GetStream(), out.data.data()); - reservespace_dev->FromGPU(GetStream(), reservespace.data()); - - return miopenStatusSuccess; -} - -template -int DropoutDriver::RunForwardCPU() -{ - InitKernelStateEmulator(states_host, DropoutDesc); - RunDropoutForwardEmulator(GetHandle(), - DropoutDesc, - inputTensor, - inputTensor, - in.data, - outputTensor, - outhost.data, - reservespace_host, - states_host); - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_fwd_out_cpu.bin", outhost.data.data(), outhost.data.size()); - } - - return miopenStatusSuccess; -} - -template -int DropoutDriver::RunBackwardGPU() -{ - float kernel_total_time = 0.0; - float kernel_first_time = 0.0; - - Timer t; - START_TIME - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - miopenDropoutBackward(GetHandle(), - DropoutDesc, - inputTensor, - outputTensor, - dout_dev->GetMem(), - inputTensor, - din_dev->GetMem(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - - if(inflags.GetValueInt("time") == 1) - { - STOP_TIME - if(WALL_CLOCK) - printf("Wall-clock Time Backward Dropout Elapsed: %f ms\n", - t.gettime_ms() / inflags.GetValueInt("iter")); - - int iter = inflags.GetValueInt("iter"); - float kernel_average_time = - iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; - printf("GPU Kernel Time Backward Dropout. Elapsed: %f ms (average)\n", kernel_average_time); - } - - din_dev->FromGPU(GetStream(), din.data.data()); - - return miopenStatusSuccess; -} - -template -int DropoutDriver::RunBackwardCPU() -{ - RunDropoutBackwardEmulator( - DropoutDesc, outputTensor, dout.data, inputTensor, din_host.data, reservespace_host); - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_out_cpu.bin", din_host.data.data(), din_host.data.size()); - } - - return miopenStatusSuccess; -} - -template -int DropoutDriver::VerifyForward() -{ - RunForwardCPU(); - - auto error = miopen::rms_range(outhost.data, out.data); - - const double tolerance = std::is_same{} ? 5e-4 : 1e-6; - if(!std::isfinite(error) || error > tolerance) - { - std::cout << "Forward Dropout FAILED: " << error << std::endl; - } - else - { - std::cout << "Forward Dropout Verifies on CPU and GPU (" << error << ')' << std::endl; - } - - return miopenStatusSuccess; -} - -template -int DropoutDriver::VerifyBackward() -{ - RunBackwardCPU(); - - auto error = miopen::rms_range(din_host.data, din.data); - - const double tolerance = std::is_same{} ? 5e-4 : 1e-6; - if(!std::isfinite(error) || error > tolerance) - { - std::cout << "Backward Dropout FAILED: " << error << std::endl; - } - else - { - std::cout << "Backward Dropout Verifies on CPU and GPU (" << error << ')' << std::endl; - } - - return miopenStatusSuccess; -} - -#endif // GUARD_MIOPEN_DROPOUT_DRIVER_HPP diff --git a/driver/dropout_gpu_emulator.hpp b/driver/dropout_gpu_emulator.hpp index d6ee776e56..e69de29bb2 100644 --- a/driver/dropout_gpu_emulator.hpp +++ b/driver/dropout_gpu_emulator.hpp @@ -1,272 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_DROPOUT_GPU_EMULATOR_HPP -#define GUARD_MIOPEN_DROPOUT_GPU_EMULATOR_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// disable __device__ qualifiers -#ifdef FQUALIFIERS -#error rocrand FQUALIFIERS defined externally, probably one of rocrand device header included prior to this -#endif -#define FQUALIFIERS inline -#include "../src/kernels/miopen_rocrand.hpp" - -static void InitKernelStateEmulator(std::vector& states, - const miopenDropoutDescriptor_t dropoutDesc) -{ - size_t states_num = miopen::deref(dropoutDesc).stateSizeInBytes / sizeof(rocrand_state_xorwow); - size_t wk_grp_num = std::min(size_t(MAX_PRNG_STATE / 256), (states_num + 255) / 256); - size_t glb_sz = wk_grp_num * 256; - - for(size_t j = 0; j < (states_num + glb_sz - 1) / glb_sz; j++) - { - for(size_t i = 0; i < glb_sz; i++) - { - size_t gid = i + j * glb_sz; - rocrand_state_xorwow state_gid; - rocrand_init(miopen::deref(dropoutDesc).seed, gid, 0ULL, &state_gid); - states[gid] = state_gid; - } - } -} - -template -inline void ExpandTensorDim(std::vector x_len, - std::vector x_str, - std::vector y_len, - std::vector y_str, - std::vector& in_len, - std::vector& in_str, - std::vector& out_len, - std::vector& out_str) -{ - auto itr_xl = x_len.end() - 1; - auto itr_yl = y_len.end() - 1; - auto itr_xs = x_str.end() - 1; - auto itr_ys = y_str.end() - 1; - auto itr_il = in_len.end() - 1; - auto itr_ol = out_len.end() - 1; - auto itr_is = in_str.end() - 1; - auto itr_os = out_str.end() - 1; - - while(itr_xl >= x_len.begin() && itr_il >= in_len.begin()) - *(itr_il--) = *(itr_xl--); - - while(itr_yl >= y_len.begin() && itr_ol >= out_len.begin()) - *(itr_ol--) = *(itr_yl--); - - while(itr_xs >= x_str.begin() && itr_is >= in_str.begin()) - *(itr_is--) = *(itr_xs--); - - while(itr_ys >= y_str.begin() && itr_os >= out_str.begin()) - *(itr_os--) = *(itr_ys--); - - while(itr_is >= in_str.begin()) - *(itr_is--) = *(itr_is + 1) * *(itr_is + 1 - in_str.begin() + in_len.begin()); - - while(itr_os >= out_str.begin()) - *(itr_os--) = *(itr_os + 1) * *(itr_os + 1 - out_str.begin() + out_len.begin()); - - if(!std::equal(in_len.begin(), in_len.end(), out_len.begin())) - { - printf("CPU verification: Input/Output tensor lengths do not match\n"); - } -} - -template -void RunDropoutForwardEmulator(miopenHandle_t handle, - const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t noise_shape, - const miopenTensorDescriptor_t inputTensor, - std::vector& in, - const miopenTensorDescriptor_t outputTensor, - std::vector& out, - std::vector& reservespace, - std::vector& states, - size_t in_offset = 0, - size_t out_offset = 0, - size_t rsvsp_offset = 0) -{ - (void)noise_shape; - auto in_dim = miopen::deref(inputTensor).GetNumDims(); - auto out_dim = miopen::deref(outputTensor).GetNumDims(); - if(in_dim != out_dim) - { - printf("CPU verification: Input/Output dimension does not match\n"); - return; - } - - if(in_dim > 5) - { - printf("CPU verification: Only support 1D to 5D tensors\n"); - } - - if(miopen::deref(inputTensor).GetElementSize() != miopen::deref(outputTensor).GetElementSize()) - { - printf("CPU verification: Input/Output element size does not match\n"); - } - - auto use_mask = miopen::deref(dropoutDesc).use_mask; - auto dropout_rate = miopen::deref(dropoutDesc).dropout; - if(dropout_rate < 0.0 || dropout_rate > 1.0) - { - printf("CPU verification: Invalid dropout rate\n"); - } - - // support up to 5D tensor - std::vector in_len(5, 1); - std::vector in_str(5, 1); - std::vector out_len(5, 1); - std::vector out_str(5, 1); - - ExpandTensorDim(miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides(), - miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides(), - in_len, - in_str, - out_len, - out_str); - - size_t glb_sz = - std::min( - size_t(std::min(size_t(MAX_PRNG_STATE), miopen::deref(handle).GetImage3dMaxWidth()) / - 256), - ((in_len[4] * in_len[3] * in_len[2] * in_len[1] * in_len[0] + 255) / 256)) * - 256; - - for(int i0 = 0; i0 < in_len[0]; i0++) - for(int i1 = 0; i1 < in_len[1]; i1++) - for(int i2 = 0; i2 < in_len[2]; i2++) - for(int i3 = 0; i3 < in_len[3]; i3++) - for(int i4 = 0; i4 < in_len[4]; i4++) - { - size_t oi = out_offset + i0 * out_str[0] + i1 * out_str[1] + - i2 * out_str[2] + i3 * out_str[3] + i4; - size_t ii = in_offset + i0 * in_str[0] + i1 * in_str[1] + i2 * in_str[2] + - i3 * in_str[3] + i4; - size_t si = i0 * in_len[1] * in_len[2] * in_len[3] * in_len[4] + - i1 * in_len[2] * in_len[3] * in_len[4] + - i2 * in_len[3] * in_len[4] + i3 * in_len[4] + i4; - size_t ri = rsvsp_offset + si; - - if(!use_mask) - reservespace[ri] = - prng::xorwow_uniform(&states[si % glb_sz]) > dropout_rate; - - out[oi] = bool(reservespace[ri]) && !miopen::float_equal(dropout_rate, 1.0) - ? static_cast(in[ii] / (1 - dropout_rate)) - : 0; - } -} - -template -void RunDropoutBackwardEmulator(const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t outputTensor, - std::vector& dout, - const miopenTensorDescriptor_t inputTensor, - std::vector& din, - std::vector& reservespace, - size_t in_offset = 0, - size_t out_offset = 0, - size_t rsvsp_offset = 0) -{ - auto in_dim = miopen::deref(inputTensor).GetNumDims(); - auto out_dim = miopen::deref(outputTensor).GetNumDims(); - if(in_dim != out_dim) - { - printf("CPU verification: Input/Output dimension does not match\n"); - return; - } - - if(in_dim > 5) - { - printf("CPU verification: Only support 1D to 5D tensors\n"); - } - - if(miopen::deref(inputTensor).GetElementSize() != miopen::deref(outputTensor).GetElementSize()) - { - printf("CPU verification: Input/Output element size does not match\n"); - } - - auto dropout_rate = miopen::deref(dropoutDesc).dropout; - if(dropout_rate < 0.0 || dropout_rate > 1.0) - { - printf("CPU verification: Invalid dropout rate\n"); - } - - // support up to 5D tensor - std::vector in_len(5, 1); - std::vector in_str(5, 1); - std::vector out_len(5, 1); - std::vector out_str(5, 1); - - ExpandTensorDim(miopen::deref(inputTensor).GetLengths(), - miopen::deref(inputTensor).GetStrides(), - miopen::deref(outputTensor).GetLengths(), - miopen::deref(outputTensor).GetStrides(), - in_len, - in_str, - out_len, - out_str); - - for(int i0 = 0; i0 < in_len[0]; i0++) - for(int i1 = 0; i1 < in_len[1]; i1++) - for(int i2 = 0; i2 < in_len[2]; i2++) - for(int i3 = 0; i3 < in_len[3]; i3++) - for(int i4 = 0; i4 < in_len[4]; i4++) - { - size_t oi = out_offset + i0 * out_str[0] + i1 * out_str[1] + - i2 * out_str[2] + i3 * out_str[3] + i4; - size_t ii = in_offset + i0 * in_str[0] + i1 * in_str[1] + i2 * in_str[2] + - i3 * in_str[3] + i4; - size_t ri = rsvsp_offset + - i0 * in_len[1] * in_len[2] * in_len[3] * in_len[4] + - i1 * in_len[2] * in_len[3] * in_len[4] + - i2 * in_len[3] * in_len[4] + i3 * in_len[4] + i4; - - din[ii] = static_cast(bool(reservespace[ri]) && - !miopen::float_equal(dropout_rate, 1.0) - ? dout[oi] / (1 - dropout_rate) - : 0); - } -} - -#endif // GUARD_MIOPEN_DROPOUT_GPU_EMULATOR_HPP diff --git a/driver/glu_driver.hpp b/driver/glu_driver.hpp index 38deb2d69e..e69de29bb2 100644 --- a/driver/glu_driver.hpp +++ b/driver/glu_driver.hpp @@ -1,482 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include "InputFlags.hpp" -#include "driver.hpp" -#include "tensor_driver.hpp" -#include "timer.hpp" -#include "random.hpp" - -#include -#include -#include -#include -#include - -#include <../test/verify.hpp> - -#include -#include - -template -T sigmoid(T x) -{ - return 1.0f / (1.0f + exp(-x)); -} - -template -int mloGLUForwardContiguousDim0RunHost(const Tgpu* input, - miopenTensorDescriptor_t outputDesc, - Tcheck* outputHost) -{ - auto output_numel = miopen::deref(outputDesc).GetElementSize(); - auto inputFirstHalf = input; - auto inputSecondHalf = input + output_numel; - - int ret = 0; - - for(size_t o = 0; o < output_numel; o++) - { - Tcheck valA = static_cast(inputFirstHalf[o]); - Tcheck valB = static_cast(inputSecondHalf[o]); - Tcheck val = valA * sigmoid(valB); - outputHost[o] = val; - } - - return ret; -} - -template -int mloGLUBackwardCongiguousDim0RunHost(const Tgpu* input, - miopenTensorDescriptor_t outputGradDesc, - const Tgpu* outputGrad, - Tcheck* inputGradHost) -{ - int ret = 0; - - auto outputGrad_numel = miopen::deref(outputGradDesc).GetElementSize(); - auto inputFirstHalf = input; - auto inputSecondHalf = input + outputGrad_numel; - auto inputFistHalf_grad = inputGradHost; - auto inputSecondHalf_grad = inputGradHost + outputGrad_numel; - - for(size_t o = 0; o < outputGrad_numel; o++) - { - Tcheck inputFirstHalf_v = static_cast(inputFirstHalf[o]); - Tcheck sigmoid_v = sigmoid(static_cast(inputSecondHalf[o])); - Tcheck grad_v = static_cast(outputGrad[o]); - - inputFistHalf_grad[o] = sigmoid_v * grad_v; - inputSecondHalf_grad[o] = (1 - sigmoid_v) * sigmoid_v * grad_v * inputFirstHalf_v; - } - - return ret; -} - -template -class GLUDriver : public Driver -{ -public: - GLUDriver() : Driver() - { - miopenCreateTensorDescriptor(&inputTensor); - miopenCreateTensorDescriptor(&outputTensor); - miopenCreateTensorDescriptor(&inputTensorGrad); - miopenCreateTensorDescriptor(&outputTensorGrad); - - data_type = miopen_type{}; - } - - int AddCmdLineArgs() override; - int ParseCmdLineArgs(int argc, char* argv[]) override; - InputFlags& GetInputFlags() override { return inflags; } - - int GetandSetData() override; - std::vector GetInputTensorLengthsFromCmdLine(); - - int AllocateBuffersAndCopy() override; - - int RunForwardGPU() override; - int RunForwardCPU(); // Verify implements it - - int RunBackwardGPU() override; - int RunBackwardCPU(); // Verify implements it - - Tref GetTolerance(); - - int VerifyBackward() override; - int VerifyForward() override; - ~GLUDriver() override - { - miopenDestroyTensorDescriptor(outputTensor); - miopenDestroyTensorDescriptor(inputTensor); - miopenDestroyTensorDescriptor(inputTensorGrad); - miopenDestroyTensorDescriptor(outputTensorGrad); - } - -private: - InputFlags inflags; - - int forw; - - miopenTensorDescriptor_t inputTensor; - miopenTensorDescriptor_t outputTensor; - - // Backwards - miopenTensorDescriptor_t inputTensorGrad; - miopenTensorDescriptor_t outputTensorGrad; - - std::unique_ptr in_dev; - std::unique_ptr out_dev; - - std::unique_ptr inGrad_dev; - std::unique_ptr outGrad_dev; - - std::vector in; - std::vector out; - std::vector outhost; - - std::vector inGrad; - std::vector outGrad; - std::vector inGradhost; - - uint32_t dim; -}; - -template -int GLUDriver::ParseCmdLineArgs(int argc, char* argv[]) -{ - inflags.Parse(argc, argv); - - if(inflags.GetValueInt("time") == 1) - { - miopenEnableProfiling(GetHandle(), true); - } - - forw = inflags.GetValueInt("forw"); - - if(forw != 0 && forw != 1) - { - MIOPEN_THROW("Invalid Forward Mode"); - } - - return miopenStatusSuccess; -} - -template -int GLUDriver::GetandSetData() -{ - std::vector in_len = inflags.GetValueTensor("dim_lengths").lengths; - dim = inflags.GetValueInt("dim_to_split"); - - SetTensorNd(inputTensor, in_len, data_type); - - std::vector out_len; - - for(int i = 0; i < in_len.size(); i++) - { - if(i != dim) - { - out_len.push_back(in_len[i]); - } - else - { - out_len.push_back(in_len[i] / 2); - } - } - - if(out_len.empty()) - out_len.push_back(1); - - SetTensorNd(outputTensor, out_len, data_type); - - // Backwards - SetTensorNd(inputTensorGrad, in_len, data_type); - SetTensorNd(outputTensorGrad, out_len, data_type); - - return miopenStatusSuccess; -} - -template -int GLUDriver::AddCmdLineArgs() -{ - inflags.AddInputFlag("forw", - 'F', - "1", - "Run only Forward (1) or Run both Forward and Backward (0) (Default=1)", - "int"); - inflags.AddTensorFlag( - "dim_lengths", 'D', "256x512", "The dimensional lengths of the input tensor"); - inflags.AddInputFlag("dim_to_split", - 'R', - "0", - "The indice of the dimensions to be split half(Default=0)", - "int"); - inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); - inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); - inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); - inflags.AddInputFlag( - "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); - - return miopenStatusSuccess; -} - -template -int GLUDriver::AllocateBuffersAndCopy() -{ - uint32_t ctx = 0; - - size_t in_sz = GetTensorSpace(inputTensor); - - if(forw == 1) - { - size_t out_sz = GetTensorSpace(outputTensor); - - // GPU allocation - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - - // GPU host allocation - in = std::vector(in_sz, static_cast(0)); - out = std::vector(out_sz, static_cast(0)); - - // CPU allocation - outhost = std::vector(out_sz, static_cast(0)); - - for(int i = 0; i < in_sz; i++) - { - in[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); - } - - if(in_dev->ToGPU(GetStream(), in.data()) != 0) - std::cerr << "Error copying (input) to GPU, size: " << in_dev->GetSize() << std::endl; - - if(out_dev->ToGPU(GetStream(), out.data()) != 0) - std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; - } - - if(forw == 0) - { - size_t out_sz = GetTensorSpace(outputTensor); - size_t inGrad_sz = GetTensorSpace(inputTensorGrad); - size_t outGrad_sz = GetTensorSpace(outputTensorGrad); - - // GPU allocation - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - inGrad_dev = std::unique_ptr(new GPUMem(ctx, inGrad_sz, sizeof(Tgpu))); - outGrad_dev = std::unique_ptr(new GPUMem(ctx, outGrad_sz, sizeof(Tgpu))); - - // GPU host allocation - in = std::vector(in_sz, static_cast(0)); - out = std::vector(out_sz, static_cast(0)); - inGrad = std::vector(inGrad_sz, static_cast(0)); - outGrad = std::vector(outGrad_sz, static_cast(0)); - - // CPU allocation - outhost = std::vector(out_sz, static_cast(0)); - inGradhost = std::vector(inGrad_sz, static_cast(0)); - - for(int i = 0; i < in_sz; i++) - { - in[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); - } - for(int i = 0; i < outGrad_sz; i++) - { - outGrad[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); - } - - if(in_dev->ToGPU(GetStream(), in.data()) != 0) - std::cerr << "Error copying (input) to GPU, size: " << in_dev->GetSize() << std::endl; - if(out_dev->ToGPU(GetStream(), out.data()) != 0) - std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; - if(outGrad_dev->ToGPU(GetStream(), outGrad.data()) != 0) - std::cerr << "Error copying (output gradient) to GPU, size: " << outGrad_dev->GetSize() - << std::endl; - if(inGrad_dev->ToGPU(GetStream(), inGrad.data()) != 0) - std::cerr << "Error copying (input gradient) to GPU, size: " << inGrad_dev->GetSize() - << std::endl; - } - - return miopenStatusSuccess; -} - -template -int GLUDriver::RunForwardGPU() -{ - float kernel_total_time = 0; - float kernel_first_time = 0; - - Timer t; - START_TIME - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - miopenStatus_t status = miopenGLUForward( - GetHandle(), inputTensor, in_dev->GetMem(), outputTensor, out_dev->GetMem(), dim); - - MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenGLUForward"); - - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - - if(inflags.GetValueInt("time") == 1) - { - STOP_TIME - int iter = inflags.GetValueInt("iter"); - if(WALL_CLOCK) - std::cout << "Wall-clock Time Forward GLU Elapsed: " << t.gettime_ms() / iter - << " ms\n"; - - float kernel_average_time = - iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; - std::cout << "GPU Kernel Time Forward GLU Elapsed: " << kernel_average_time << " ms\n"; - } - - if(out_dev->FromGPU(GetStream(), out.data()) != 0) - std::cerr << "Error copying (out_dev) from GPU, size: " << out_dev->GetSize() << std::endl; - - return miopenStatusSuccess; -} - -template -int GLUDriver::RunForwardCPU() -{ - MIOPEN_THROW_IF(dim != 0, "This driver only supports dim = 0"); - mloGLUForwardContiguousDim0RunHost(in.data(), outputTensor, outhost.data()); - - return miopenStatusSuccess; -} - -template -int GLUDriver::RunBackwardGPU() -{ - float kernel_total_time = 0; - float kernel_first_time = 0; - Timer t; - START_TIME; - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - miopenStatus_t status = miopenGLUBackward(GetHandle(), - inputTensor, - in_dev->GetMem(), - outputTensorGrad, - outGrad_dev->GetMem(), - inputTensorGrad, - inGrad_dev->GetMem(), - dim); - - MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenGLUBackward"); - - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - kernel_total_time += time; - if(i == 0) - kernel_first_time = time; - } - - if(inflags.GetValueInt("time") == 1) - { - STOP_TIME - int iter = inflags.GetValueInt("iter"); - if(WALL_CLOCK) - std::cout << "Wall-clock Time Backward GLU Elapsed: " << t.gettime_ms() / iter - << " ms\n"; - float kernel_average_time = - iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; - std::cout << "GPU Kernel Time Backward GLU Elapsed: " << kernel_average_time << " ms\n"; - } - - if(inGrad_dev->FromGPU(GetStream(), inGrad.data()) != 0) - std::cerr << "Error copying (out_dev) from GPU, size: " << inGrad_dev->GetSize() - << std::endl; - - return miopenStatusSuccess; -} - -template -Tref GLUDriver::GetTolerance() -{ - Tref tolerance = std::numeric_limits::epsilon() * 10; - return tolerance; -} - -template -int GLUDriver::VerifyForward() -{ - RunForwardCPU(); - const Tref tolerance = GetTolerance(); - auto error = miopen::rms_range(outhost, out); - - if(!std::isfinite(error) || error > tolerance) - { - std::cout << "Forward GLU FAILED: " << error << " > " << tolerance << std::endl; - return EC_VerifyFwd; - } - else - { - std::cout << "Forward GLU Verifies OK on CPU reference (" << error << " < " << tolerance - << ')' << std::endl; - } - - return miopenStatusSuccess; -} - -template -int GLUDriver::RunBackwardCPU() -{ - MIOPEN_THROW_IF(dim != 0, "This driver only supports dim = 0"); - mloGLUBackwardCongiguousDim0RunHost( - in.data(), outputTensorGrad, outGrad.data(), inGradhost.data()); - - return miopenStatusSuccess; -} - -template -int GLUDriver::VerifyBackward() -{ - RunBackwardCPU(); - const Tref tolerance = GetTolerance(); - auto error = miopen::rms_range(inGradhost, inGrad); - if(!std::isfinite(error) || error > tolerance) - { - std::cout << "Backward GLU FAILED: " << error << " > " << tolerance << std::endl; - return EC_VerifyBwd; - } - else - { - std::cout << "Backward GLU Verifies OK on CPU reference (" << error << " < " << tolerance - << ')' << std::endl; - } - - return miopenStatusSuccess; -} diff --git a/driver/gru_verify_gemm.hpp b/driver/gru_verify_gemm.hpp index 6266070be8..e69de29bb2 100644 --- a/driver/gru_verify_gemm.hpp +++ b/driver/gru_verify_gemm.hpp @@ -1,1966 +0,0 @@ -#ifndef GUARD_MIOPEN_GRU_VERIFY_GEMM_HPP -#define GUARD_MIOPEN_GRU_VERIFY_GEMM_HPP - -#define ADNN_MM_TRANSPOSE 1 - -#include "dropout_gpu_emulator.hpp" -#include "mloConvHost.hpp" // ADNN_mm_cpu - -#include <../test/rnn_util.hpp> - -#include -#include -#include - -template -void RunGRUForwardGEMMCPUVerify(miopenHandle_t handle, - std::vector& in, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& hy_host, // current/final hidden state - std::vector& hx, // initial hidden state - std::vector& out_host, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) for - // unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by hy_h - // related function for bidirection - int inputMode, - std::vector& rsvspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool hx_is_null = false) -{ - int batch_n = sumvc(in_n); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int out_stride = out_h; - int wei_stride = bi * 3 * hy_h; - int hy_stride = bi * 4 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - std::vector hid_state(numlayer * batch_n * hy_stride * 2, static_cast(0)); - - Tref* out_state = new Tref[batch_n * out_h]; - memset(out_state, 0, batch_n * out_h * sizeof(Tref)); - - // initial input - Tref* in_state = new Tref[batch_n * in_h]; - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state[h * in_stride + w] = in[h * in_stride + w]; - } - } - - // initial hidden states - Tref* hy_state = new Tref[hy_d * hy_n * hy_h]; - memset(hy_state, 0, hy_d * hy_n * hy_h * sizeof(Tref)); - Tref* hx_state = new Tref[hy_d * hy_n * hy_h]; - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state[h] = hx[h]; - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - int wei_len = wei_shift_bias; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial weights - Tref* wei_state = new Tref[wei_len]; - for(int h = 0; h < wei_len; h++) - { - wei_state[h] = wei[h]; - } - - // initial dropoput - std::vector dropout_states_host; - std::vector dropout_reservespace_host; - std::vector dropout_hid_state; - miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; - if(use_dropout) - { - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - dropout_states_host = std::vector(states_size); - InitKernelStateEmulator(dropout_states_host, dropoutDesc); - - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - std::array drop_out_str = {{hy_h * bi, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenCreateTensorDescriptor(&dropout_outputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - miopenSetTensorDescriptor( - dropout_outputTensor, miopenFloat, 2, drop_in_len.data(), drop_out_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(1)); - - dropout_hid_state = - std::vector((numlayer - 1) * batch_n * hy_h * bi, static_cast(0)); - } - - // forward emulator - for(int li = 0; li < numlayer; li++) - { - int hid_shift = li * batch_n * hy_stride; - int hx_shift = li * in_n[0] * h_stride; - int wei_shift_bias_temp = wei_shift_bias + li * 2 * wei_stride; - - // from input - if(li == 0) - { - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 3; gi++) - { - hid_state[hid_shift + bs * hy_stride + gi * hy_h + h] += - in_state[bs * in_stride + h]; - if(bidirection) - { - hid_state[hid_shift + bs * hy_stride + (gi + 3) * hy_h + h] += - in_state[bs * in_stride + h]; - } - } - } - } - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias + h]; - } - } - } - } - else - { - ADNN_mm_cpu(const_cast(in_state), - in_h, - batch_n, - in_stride, - 0, - const_cast(wei_state), - in_h, - hy_h * bi * 3, - in_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi * 3, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias + h]; - } - } - } - } - } - else - { - int wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - int prelayer_shift = (li - 1) * batch_n * hy_stride + bi * 3 * hy_h; - if(use_dropout) - { - auto dropout_states_tmp = dropout_states_host; - size_t drop_out_offset = (li - 1) * batch_n * hy_h * bi; - - RunDropoutForwardEmulator(handle, - dropoutDesc, - dropout_inputTensor, - dropout_inputTensor, - hid_state, - dropout_outputTensor, - dropout_hid_state, - dropout_reservespace_host, - dropout_states_tmp, - prelayer_shift, - drop_out_offset, - drop_out_offset); - - prelayer_shift = drop_out_offset; - } - - ADNN_mm_cpu(use_dropout ? &dropout_hid_state[prelayer_shift] - : &hid_state[prelayer_shift], - hy_h * bi, - batch_n, - use_dropout ? hy_h * bi : hy_stride, - 0, - const_cast(&wei_state[wei_shift]), - hy_h * bi, - hy_h * bi * 3, - bi_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi * 3, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state[hid_shift + bs * hy_stride + h] += wei[wei_shift_bias_temp + h]; - } - } - } - } - - // from hidden state - bacc = 0; - baccbi = batch_n; - for(int ti = 0; ti < seqLength; ti++) - { - baccbi -= in_n[seqLength - 1 - ti]; - int wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int pretime_shift; - - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(const_cast(&hx_state[hx_shift]), - hy_h, - in_n[ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift]), - hy_h, - hy_h * 2, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h * 2, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 2; gi++) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + gi * hy_h + - h] += - wei[wei_shift_bias_temp + wei_stride + gi * hy_h + h]; - } - } - } - } - - ADNN_mm_cpu( - const_cast(&hx_state[hx_shift]), - hy_h, - in_n[ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 2 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride + bi * 3 * hy_h], - hy_h, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + - h] += - wei[wei_shift_bias_temp + wei_stride + 2 * hy_h + h]; - } - } - } - - if(bidirection) - { - ADNN_mm_cpu( - const_cast(&hx_state[hx_shift + hy_n * hy_h]), - hy_h, - in_n[seqLength - 1 - ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 3 * hy_h * uni_stride]), - hy_h, - hy_h * 2, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + 3 * hy_h], - hy_h * 2, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 2; gi++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - (3 + gi) * hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h + - h]; - } - } - } - } - - ADNN_mm_cpu( - const_cast(&hx_state[hx_shift + hy_n * hy_h]), - hy_h, - in_n[seqLength - 1 - ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 5 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h]; - } - } - } - } - } - } - else - { - ADNN_mm_cpu(const_cast(&hy_state[hx_shift]), - hy_h, - in_n[ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift]), - hy_h, - hy_h * 2, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h * 2, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 2; gi++) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + gi * hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + gi * hy_h + h]; - } - } - } - } - - ADNN_mm_cpu(const_cast(&hy_state[hx_shift]), - hy_h, - in_n[ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 2 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride + bi * 3 * hy_h], - hy_h, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + 2 * hy_h + h]; - } - } - } - - if(bidirection) - { - - if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti)) - { - ADNN_mm_cpu( - const_cast( - &hx_state[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h]), - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 3 * hy_h * uni_stride]), - hy_h, - hy_h * 2, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride + - 3 * hy_h], - hy_h * 2, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti); - bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 2; gi++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - (3 + gi) * hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h + - h]; - } - } - } - } - - ADNN_mm_cpu( - const_cast( - &hx_state[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h]), - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 5 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride + - bi * 3 * hy_h + hy_h], - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti); - bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h]; - } - } - } - } - - ADNN_mm_cpu( - const_cast(&hy_state[hx_shift + hy_n * hy_h]), - hy_h, - in_n[seqLength - ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 3 * hy_h * uni_stride]), - hy_h, - hy_h * 2, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + 3 * hy_h], - hy_h * 2, - in_n[seqLength - ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[seqLength - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 2; gi++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - (3 + gi) * hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + (3 + gi) * hy_h + h]; - } - } - } - } - - ADNN_mm_cpu( - const_cast(&hy_state[hx_shift + hy_n * hy_h]), - hy_h, - in_n[seqLength - ti], - uni_stride, - 0, - const_cast(&wei_state[wei_shift + 5 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h], - hy_h, - in_n[seqLength - ti], - hy_stride, - 0, - 1, - 1); - - if(biased) - { - for(int bs = 0; bs < in_n[seqLength - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] += - wei[wei_shift_bias_temp + wei_stride + 5 * hy_h + h]; - } - } - } - } - } - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h + - numlayer * batch_n * hy_stride] = - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h]; - - hid_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] += - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2) * - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h]; - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] = 0; - - if(ti == 0) - { - if(!hx_is_null) - { - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - ((1 - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], - 2)) * - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + - 2 * hy_h + h], - 1) + - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], 2) * - hx[hx_shift + bs * uni_stride + h]); - } - else - { - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - ((1 - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], - 2)) * - activfunc( - hid_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], - 1)); - } - } - else - { - - pretime_shift = li * batch_n * hy_stride + - (bacc - in_n[ti - 1]) * hy_stride + bi * 3 * hy_h; - - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - ((1 - - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], 2)) * - activfunc( - hid_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], - 1) + - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], 2) * - hid_state[pretime_shift + bs * hy_stride + h]); - } - - hid_state[hid_shift + (bacc + bs) * hy_stride + h + - numlayer * batch_n * hy_stride] = - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + h], 2); - hid_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h + - numlayer * batch_n * hy_stride] = - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2); - hid_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h + - numlayer * batch_n * hy_stride] = - activfunc(hid_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], 1); - - hy_state[hx_shift + bs * uni_stride + h] = - hid_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h]; - } - } - - if(bidirection) - { - pretime_shift = li * batch_n * hy_stride + - (baccbi + in_n[seqLength - 1 - ti]) * hy_stride + bi * 3 * hy_h + - hy_h; - - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + h + - numlayer * batch_n * hy_stride] = - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h]; - - hid_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] += - activfunc( - hid_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], - 2) * - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h]; - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h] = 0; - - if(ti == 0) - { - if(!hx_is_null) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] += - ((1 - - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2)) * - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 1) + - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2) * - hx[hx_shift + bs * uni_stride + hy_n * hy_h + h]); - } - else - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] += - ((1 - - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2)) * - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 1)); - } - } - else - { - if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti)) - { - if(bs >= in_n.at(seqLength - ti)) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] += - (activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2) * - hx[hx_shift + bs * uni_stride + hy_n * hy_h + h]); - } - } - - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h] += - ((1 - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2)) * - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 1)); - - if(bs < in_n[seqLength - ti]) - { - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] += - (activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2) * - hid_state[pretime_shift + bs * hy_stride + h]); - } - } - - hid_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h + - numlayer * batch_n * hy_stride] = - activfunc( - hid_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h], 2); - hid_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h + - numlayer * batch_n * hy_stride] = - activfunc( - hid_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], 2); - hid_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h + - numlayer * batch_n * hy_stride] = - activfunc( - hid_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h], 1); - - hy_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] = - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h]; - } - } - } - - bacc += in_n[ti]; - } - } - - // output - int prelayer_shift = (numlayer - 1) * batch_n * hy_stride + bi * 3 * hy_h; - - for(int i = 0; i < numlayer * batch_n * hy_stride * 2; i++) - { - rsvspace_host[i] = hid_state[i]; - } - if(use_dropout) - { - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - rsvspace_host.at(numlayer * batch_n * hy_stride * 2 + i) = dropout_hid_state.at(i); - } - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - *(p_drop_rsv + i) = dropout_reservespace_host.at(i); - } - } - - for(int i = 0; i < hy_d * hy_n * hy_h; i++) - { - hy_host[i] = hy_state[i]; - } - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - out_host[bs * out_stride + h] = hid_state[prelayer_shift + bs * hy_stride + h]; - } - } - - delete[] out_state; - delete[] in_state; - delete[] hx_state; - delete[] hy_state; - delete[] wei_state; -} - -template -void RunGRUBackwardDataGEMMCPUVerify(std::vector& din_host, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& dhy, // current/final hidden state - std::vector& dhx_host, - std::vector& hx, // initial hidden state - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) - // for unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by - // hy_h related function for bidirection - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool hx_is_null = false, - bool dhy_is_null = false) -{ - int batch_n = sumvc(in_n); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int out_stride = out_h; - int wei_stride = bi * 3 * hy_h; - int hy_stride = bi * 4 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - std::vector dh_state(numlayer * batch_n * hy_stride, static_cast(0)); - - Tref* din_state = new Tref[batch_n * in_h]; - memset(din_state, 0, batch_n * in_h * sizeof(Tref)); - - // initial dout - Tref* dout_state = new Tref[batch_n * out_h]; - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state[h * out_stride + w] = dout[h * out_stride + w]; - } - } - - // initial hidden states - Tref* dhx_state = new Tref[hy_d * hy_n * hy_h]; - memset(dhx_state, 0, hy_d * hy_n * hy_h * sizeof(Tref)); - Tref* dcx_state = new Tref[hy_d * hy_n * hy_h]; - memset(dcx_state, 0, hy_d * hy_n * hy_h * sizeof(Tref)); - Tref* dhy_state = new Tref[hy_d * hy_n * hy_h]; - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - dhy_state[h] = dhy[h]; - } - Tref* hx_state = new Tref[hy_d * hy_n * hy_h]; - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state[h] = hx[h]; - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_len = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial weights - Tref* wei_state = new Tref[wei_len]; - for(int h = 0; h < wei_len; h++) - { - wei_state[h] = wei[h]; - } - - // initial dropoput - miopenTensorDescriptor_t dropout_inputTensor{}; - std::vector dropout_reservespace_host; - if(use_dropout) - { - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(0)); - - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - dropout_reservespace_host.at(i) = *(p_drop_rsv + i); - } - } - - // bwd data emulator - for(int li = numlayer - 1; li >= 0; li--) - { - int wei_shift = (in_h + hy_h) * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int hid_shift = li * batch_n * hy_stride; - int hx_shift = li * in_n[0] * h_stride; - int weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - if(li == numlayer - 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - dh_state[hid_shift + bi * 3 * hy_h + bs * hy_stride + h] += - dout_state[bs * out_stride + h]; - } - } - } - else - { - int prelayer_shift = (li + 1) * batch_n * hy_stride; - - ADNN_mm_cpu(&dh_state[prelayer_shift], - hy_h * bi * 3, - batch_n, - hy_stride, - 0, - const_cast(&wei_state[wei_shift]), - hy_h * bi, - hy_h * bi * 3, - bi_stride, - 0, - &dh_state[hid_shift + bi * 3 * hy_h], - hy_h * bi, - batch_n, - hy_stride, - 0, - 1, - 1); - - if(use_dropout) - { - RunDropoutBackwardEmulator(dropoutDesc, - dropout_inputTensor, - dh_state, - dropout_inputTensor, - dh_state, - dropout_reservespace_host, - hid_shift + bi * 3 * hy_h, - hid_shift + bi * 3 * hy_h, - li * batch_n * hy_h * bi); - } - } - - // from hidden state - bacc = batch_n; - baccbi = 0; - for(int ti = seqLength - 1; ti >= 0; ti--) - { - bacc -= in_n[ti]; - - if(ti == seqLength - 1) - { - if(!dhy_is_null) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - dhy_state[hx_shift + bs * uni_stride + h]; - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] += - dhy_state[hx_shift + bs * uni_stride + hy_n * hy_h + h]; - } - } - } - } - } - else - { - if(!dhy_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - for(int bs = in_n.at(ti + 1); bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - dhy_state[hx_shift + bs * uni_stride + h]; - } - } - } - - int pretime_shift = li * batch_n * hy_stride + (bacc + in_n[ti]) * hy_stride; - - ADNN_mm_cpu(&dh_state[pretime_shift], - hy_h * 2, - in_n[ti + 1], - hy_stride, - 0, - const_cast(&wei_state[weitime_shift]), - hy_h, - hy_h * 2, - uni_stride, - 0, - &dh_state[hid_shift + bacc * hy_stride + bi * 3 * hy_h], - hy_h, - in_n[ti + 1], - hy_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n[ti + 1]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] += - dh_state[pretime_shift + bs * hy_stride + bi * 3 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + h], 2); - - dh_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] = - dh_state[pretime_shift + bs * hy_stride + 2 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + hy_h + h], 2); - } - } - - ADNN_mm_cpu( - &dh_state[hid_shift + bacc * hy_stride + 2 * hy_h], - hy_h, - in_n[ti + 1], - hy_stride, - 0, - const_cast(&wei_state[weitime_shift + 2 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - 0, - &dh_state[hid_shift + bacc * hy_stride + bi * 3 * hy_h], - hy_h, - in_n[ti + 1], - hy_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n[ti + 1]; bs++) - { - for(int hs = 0; hs < hy_h; hs++) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + hs] = 0; - } - } - - if(bidirection) - { - pretime_shift = li * batch_n * hy_stride + - (baccbi - in_n[seqLength - 2 - ti]) * hy_stride + hy_h * 3; - - ADNN_mm_cpu( - &dh_state[pretime_shift], - hy_h * 2, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - const_cast(&wei_state[weitime_shift + hy_h * 3 * uni_stride]), - hy_h, - hy_h * 2, - uni_stride, - 0, - &dh_state[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h] += - dh_state[pretime_shift + bs * hy_stride + 3 * hy_h + hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + h], 2); - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] = - dh_state[pretime_shift + bs * hy_stride + 2 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + hy_h + h], - 2); - } - } - - ADNN_mm_cpu( - &dh_state[hid_shift + baccbi * hy_stride + 5 * hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - const_cast(&wei_state[weitime_shift + 5 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - 0, - &dh_state[hid_shift + baccbi * hy_stride + bi * 3 * hy_h + hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int hs = 0; hs < hy_h; hs++) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + hs] = 0; - } - } - } - } - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] * - (1 - activfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], 2)) * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], 1); - - dh_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h] = - (rsvspace_host[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h + - numlayer * batch_n * hy_stride] * - dh_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2)); - - if(ti == 0) - { - if(!hx_is_null) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] * - hx_state[hx_shift + bs * uni_stride + h] * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], 2); - } - dh_state[hid_shift + (bacc + bs) * hy_stride + h] -= - (dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] * - activfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], - 1) * - dervactivfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], - 2)); - } - else - { - dh_state[hid_shift + (bacc + bs) * hy_stride + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 3 * hy_h + h] * - (rsvspace_host[hid_shift + (bacc - in_n[ti - 1] + bs) * hy_stride + - bi * 3 * hy_h + h] - - activfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], - 1)) * - dervactivfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], - 2); - } - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + hy_h + - h] * - (1 - activfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2)) * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h], - 1); - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h] = - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h + numlayer * batch_n * hy_stride]; - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h] *= - (dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] * - dervactivfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 4 * hy_h + h], - 2)); - - if(ti == 0) - { - if(!hx_is_null) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] += - (dh_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] * - hx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h]); - } - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] -= - (dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] * - activfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 1)); - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] *= - dervactivfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2); - } - else - { - if(!hx_is_null && - in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti) && - bs >= in_n.at(seqLength - ti)) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] += - (dh_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] * - hx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h]); - } - - if(bs < in_n[seqLength - ti]) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] += - (dh_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 3 * hy_h + hy_h + h] * - rsvspace_host[hid_shift + - (baccbi + in_n[seqLength - 1 - ti] + bs) * - hy_stride + - bi * 3 * hy_h + hy_h + h]); - } - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] -= - (dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 3 * hy_h + - hy_h + h] * - activfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 1)); - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 3 * hy_h + h] *= - dervactivfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 3 * hy_h + h], - 2); - } - } - } - } - - baccbi += in_n[seqLength - 1 - ti]; - } - - // dhx - int pretime_shift = li * batch_n * hy_stride; - - ADNN_mm_cpu(&dh_state[pretime_shift], - hy_h * 2, - in_n[0], - hy_stride, - 0, - const_cast(&wei_state[weitime_shift]), - hy_h, - hy_h * 2, - uni_stride, - 0, - &dhx_state[hx_shift], - hy_h, - in_n[0], - uni_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n[0]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - dhx_state[hx_shift + bs * uni_stride + h] += - dh_state[pretime_shift + bs * hy_stride + bi * 3 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + h], 2); - - dcx_state[hx_shift + bs * uni_stride + h] = - dh_state[pretime_shift + bs * hy_stride + 2 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + hy_h + h], 2); - } - } - - ADNN_mm_cpu(const_cast(&dcx_state[hx_shift]), - hy_h, - in_n[0], - uni_stride, - 0, - const_cast(&wei_state[weitime_shift + 2 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - 0, - &dhx_state[hx_shift], - hy_h, - in_n[0], - uni_stride, - 0, - 1, - 1); - - if(bidirection) - { - int ti = seqLength - 1, cur_bat = 0, pre_bat = batch_n; - - while(ti >= 0) - { - pre_bat -= in_n.at(ti); - if(in_n.at(ti) > cur_bat) - { - pretime_shift = li * batch_n * hy_stride + (pre_bat + cur_bat) * hy_stride; - - ADNN_mm_cpu( - &dh_state[pretime_shift + 3 * hy_h], - hy_h * 2, - (in_n.at(ti) - cur_bat), - hy_stride, - 0, - const_cast(&wei_state[weitime_shift + 3 * hy_h * uni_stride]), - hy_h, - hy_h * 2, - uni_stride, - 0, - &dhx_state[hx_shift + hy_n * hy_h + cur_bat * hy_h], - hy_h, - (in_n.at(ti) - cur_bat), - uni_stride, - 0, - 1, - 1); - - for(int bs = cur_bat; bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dhx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] += - dh_state[pretime_shift + (bs - cur_bat) * hy_stride + - bi * 3 * hy_h + hy_h + h] * - activfunc(rsvspace_host[pretime_shift + (bs - cur_bat) * hy_stride + - 3 * hy_h + h], - 2); - - dcx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] = - dh_state[pretime_shift + (bs - cur_bat) * hy_stride + 5 * hy_h + - h] * - activfunc(rsvspace_host[pretime_shift + (bs - cur_bat) * hy_stride + - 4 * hy_h + h], - 2); - } - } - - ADNN_mm_cpu( - const_cast(&dcx_state[hx_shift + hy_n * hy_h + cur_bat * hy_h]), - hy_h, - (in_n.at(ti) - cur_bat), - uni_stride, - 0, - const_cast(&wei_state[weitime_shift + 5 * hy_h * uni_stride]), - hy_h, - hy_h, - uni_stride, - 0, - &dhx_state[hx_shift + hy_n * hy_h + cur_bat * hy_h], - hy_h, - (in_n.at(ti) - cur_bat), - uni_stride, - 0, - 1, - 1); - } - cur_bat = in_n.at(ti--); - } - } - } - - // dinput - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 3; gi++) - { - din_state[bs * in_stride + h] += dh_state[bs * hy_stride + gi * hy_h + h]; - if(bidirection) - { - din_state[bs * in_stride + h] += - dh_state[bs * hy_stride + (gi + 3) * hy_h + h]; - } - } - } - } - } - else - { - ADNN_mm_cpu(&dh_state[0], - hy_h * bi * 3, - batch_n, - hy_stride, - 0, - const_cast(wei_state), - in_h, - hy_h * bi * 3, - in_stride, - 0, - &din_state[0], - in_h, - batch_n, - in_stride, - 0, - 1, - 1); - } - - for(int i = 0; i < numlayer * batch_n * hy_stride; i++) - { - wkspace_host[i] = dh_state[i]; - } - - for(int i = 0; i < hy_d * hy_n * hy_h; i++) - { - dhx_host[i] = dhx_state[i]; - } - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < in_stride; h++) - { - din_host[bs * in_stride + h] = din_state[bs * in_stride + h]; - } - } - - delete[] dout_state; - delete[] din_state; - delete[] hx_state; - delete[] dhx_state; - delete[] dcx_state; - delete[] dhy_state; - delete[] wei_state; -} - -template -void RunGRUBackwardWeightGEMMCPUVerify(std::vector& in, - std::vector& dwei_host, // [ input_state_weight_trans - // hidden_state_weight0_trans - // input1_trans hidden1_trans ... - // output_weight; bidirectional - // reversed weights ] - std::vector& hx, // initial hidden state - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden - // layers) for unidirection, 2 by numlayer for - // bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 - // by hy_h related function for bidirection - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - bool hx_is_null = false) -{ - int batch_n = sumvc(in_n); - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int wei_stride = bi * 3 * hy_h; - int hy_stride = bi * 4 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - // initial input - Tref* in_state = new Tref[batch_n * in_h]; - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state[h * in_h + w] = in[h * in_h + w]; - } - } - - // initial output difference - Tref* dout_state = new Tref[batch_n * out_h]; - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state[h * out_h + w] = dout[h * out_h + w]; - } - } - - // initial saved data - Tref* wkspace_state = new Tref[numlayer * batch_n * hy_stride]; - for(int h = 0; h < numlayer * batch_n * hy_stride; h++) - { - wkspace_state[h] = wkspace_host[h]; - } - size_t rsvsp_sz = use_dropout ? rsvspace_host.size() : numlayer * batch_n * hy_stride; - Tref* rsvspace_state = new Tref[rsvsp_sz]; - for(int h = 0; h < rsvsp_sz; h++) - { - rsvspace_state[h] = rsvspace_host[h]; - } - - // initial hidden states - Tref* hx_state = new Tref[hy_d * hy_n * hy_h]; - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state[h] = hx[h]; - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - int wei_len = wei_shift_bias; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial dwei - Tref* dwei_state = new Tref[wei_len]; - memset(dwei_state, 0, wei_len * sizeof(Tref)); - - // bwd weights emulator - for(int li = 0; li < numlayer; li++) - { - // between layers - if(li == 0) - { - if(inputMode == 0) - { - ADNN_mm_cpu(const_cast(wkspace_state), - hy_h * bi * 3, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(in_state), - in_h, - batch_n, - in_stride, - 0, - &dwei_state[0], - in_h, - hy_h * bi * 3, - in_stride, - 0, - 1, - 1); - } - - if(biased) - { - for(int h = 0; h < wei_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state[wei_shift_bias + h] += wkspace_state[w * hy_stride + h]; - } - } - } - } - else - { - int prelayer_shift = - use_dropout ? 2 * numlayer * batch_n * hy_stride + (li - 1) * batch_n * hy_h * bi - : (li - 1) * batch_n * hy_stride + bi * hy_h * 3; - int hid_shift = li * batch_n * hy_stride; - int wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift]), - hy_h * bi * 3, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(&rsvspace_state[prelayer_shift]), - hy_h * bi, - batch_n, - use_dropout ? hy_h * bi : hy_stride, - 0, - &dwei_state[wei_shift], - hy_h * bi, - hy_h * bi * 3, - bi_stride, - 0, - 1, - 1); - - if(biased) - { - wei_shift = wei_shift_bias + li * 2 * wei_stride; - - for(int h = 0; h < wei_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state[wei_shift + h] += wkspace_state[hid_shift + w * hy_stride + h]; - } - } - } - } - - // between time - bacc = 0; - for(int ti = 0; ti < seqLength; ti++) - { - int hid_shift = li * batch_n * hy_stride + bacc * hy_stride; - int hx_shift = li * in_n[0] * h_stride; - int wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int pretime_shift; - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - wkspace_state[hid_shift + bs * hy_stride + 2 * hy_h + h] *= - activfunc(rsvspace_state[hid_shift + bs * hy_stride + hy_h + h], 2); - } - } - - // between time - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift]), - hy_h * 3, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(&hx_state[hx_shift]), - hy_h, - in_n[ti], - uni_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h * 3, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 3; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + h] += - wkspace_state[hid_shift + w * hy_stride + h]; - } - } - } - } - } - else - { - pretime_shift = - li * batch_n * hy_stride + (bacc - in_n[ti - 1]) * hy_stride + bi * 3 * hy_h; - - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift]), - hy_h * 3, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(&rsvspace_state[pretime_shift]), - hy_h, - in_n[ti], - hy_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h * 3, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 3; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + h] += - wkspace_state[hid_shift + w * hy_stride + h]; - } - } - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - wkspace_state[hid_shift + bs * hy_stride + 5 * hy_h + h] *= - activfunc(rsvspace_state[hid_shift + bs * hy_stride + 4 * hy_h + h], 2); - } - } - - if(ti == seqLength - 1) - { - if(!hx_is_null) - { - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift + 3 * hy_h]), - hy_h * 3, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(&hx_state[hx_shift + hy_n * hy_h]), - hy_h, - in_n[ti], - uni_stride, - 0, - &dwei_state[wei_shift + 3 * hy_h * uni_stride], - hy_h, - hy_h * 3, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 3; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + 3 * hy_h + h] += - wkspace_state[hid_shift + 3 * hy_h + w * hy_stride + h]; - } - } - } - } - } - else - { - if(!hx_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - ADNN_mm_cpu( - const_cast( - &wkspace_state[hid_shift + 3 * hy_h + in_n.at(ti + 1) * hy_stride]), - hy_h * 3, - (in_n.at(ti) - in_n.at(ti + 1)), - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast( - &hx_state[hx_shift + hy_n * hy_h + in_n.at(ti + 1) * hy_h]), - hy_h, - (in_n.at(ti) - in_n.at(ti + 1)), - uni_stride, - 0, - &dwei_state[wei_shift + 3 * hy_h * uni_stride], - hy_h, - hy_h * 3, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 3; h++) - { - for(int w = in_n.at(ti + 1); w < in_n.at(ti); w++) - { - dwei_state[bias_shift + 3 * hy_h + h] += - wkspace_state[hid_shift + 3 * hy_h + w * hy_stride + h]; - } - } - } - } - - pretime_shift = - li * batch_n * hy_stride + (bacc + in_n[ti]) * hy_stride + bi * 3 * hy_h; - - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift + 3 * hy_h]), - hy_h * 3, - in_n[ti + 1], - hy_stride, - ADNN_MM_TRANSPOSE, - const_cast(&rsvspace_state[pretime_shift + hy_h]), - hy_h, - in_n[ti + 1], - hy_stride, - 0, - &dwei_state[wei_shift + 3 * hy_h * uni_stride], - hy_h, - hy_h * 3, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 3; h++) - { - for(int w = 0; w < in_n.at(ti + 1); w++) - { - dwei_state[bias_shift + 3 * hy_h + h] += - wkspace_state[hid_shift + 3 * hy_h + w * hy_stride + h]; - } - } - } - } - } - - bacc += in_n[ti]; - } - } - - for(int i = 0; i < wei_len; i++) - { - dwei_host[i] = dwei_state[i]; - } - - delete[] dwei_state; - delete[] in_state; - delete[] dout_state; - delete[] wkspace_state; - delete[] rsvspace_state; - delete[] hx_state; -} - -#endif // GUARD_MIOPEN_GRU_VERIFY_GEMM_HPP diff --git a/driver/kthvalue_driver.hpp b/driver/kthvalue_driver.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/driver/lstm_verify_gemm.hpp b/driver/lstm_verify_gemm.hpp index 9077c25af9..e69de29bb2 100644 --- a/driver/lstm_verify_gemm.hpp +++ b/driver/lstm_verify_gemm.hpp @@ -1,1706 +0,0 @@ -#ifndef GUARD_MIOPEN_LSTM_VERIFY_GEMM_HPP -#define GUARD_MIOPEN_LSTM_VERIFY_GEMM_HPP - -#define ADNN_MM_TRANSPOSE 1 - -#include "dropout_gpu_emulator.hpp" -#include "mloConvHost.hpp" // ADNN_mm_cpu - -#include <../test/rnn_util.hpp> - -#include -#include -#include - -template -void RunLSTMForwardGEMMCPUVerify(miopenHandle_t handle, - std::vector& in, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& hy_host, // current/final hidden state - std::vector& hx, // initial hidden state - std::vector& cy_host, // current/final cell state - std::vector& cx, // initial cell state - std::vector& out_host, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) for - // unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by - // hy_h related function for bidirection - int inputMode, - std::vector& rsvspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool hx_is_null = false, - bool cx_is_null = false) -{ - size_t batch_n = sumvc(in_n); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - size_t bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int out_stride = out_h; - int wei_stride = bi * 4 * hy_h; - int hy_stride = bi * 6 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - std::vector hid_state(numlayer * batch_n * hy_stride * 2, static_cast(0)); - std::vector out_state(batch_n * out_h, static_cast(0)); - - // initial input - std::vector in_state(batch_n * in_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state.at(h * in_stride + w) = in.at(h * in_stride + w); - } - } - - // initial hidden states - std::vector hy_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector hx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state.at(h) = hx.at(h); - } - std::vector cy_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector cx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - cx_state.at(h) = cx.at(h); - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - int wei_len = wei_shift_bias; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial weights - std::vector wei_state(wei_len, static_cast(0)); - for(int h = 0; h < wei_len; h++) - { - wei_state.at(h) = wei.at(h); - } - - // initial dropoput - std::vector dropout_states_host; - std::vector dropout_reservespace_host; - std::vector dropout_hid_state; - miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; - if(use_dropout) - { - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - dropout_states_host = std::vector(states_size); - InitKernelStateEmulator(dropout_states_host, dropoutDesc); - - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - std::array drop_out_str = {{hy_h * bi, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenCreateTensorDescriptor(&dropout_outputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - miopenSetTensorDescriptor( - dropout_outputTensor, miopenFloat, 2, drop_in_len.data(), drop_out_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(1)); - - dropout_hid_state = - std::vector((numlayer - 1) * batch_n * hy_h * bi, static_cast(0)); - } - - // forward emulator - for(int li = 0; li < numlayer; li++) - { - size_t hid_shift = li * batch_n * hy_stride; - size_t hx_shift = li * in_n.at(0) * h_stride; - - // from input - if(li == 0) - { - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 4; gi++) - { - hid_state.at(hid_shift + bs * hy_stride + gi * hy_h + h) += - in_state.at(bs * in_stride + h); - if(bidirection) - { - hid_state.at(hid_shift + bs * hy_stride + (gi + 4) * hy_h + h) += - in_state.at(bs * in_stride + h); - } - } - } - } - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias + h); - } - } - } - } - else - { - ADNN_mm_cpu(in_state.data(), - in_h, - batch_n, - in_stride, - 0, - wei_state.data(), - in_h, - hy_h * bi * 4, - in_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi * 4, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias + h); - } - } - } - } - } - else - { - size_t wei_shift = - (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - size_t prelayer_shift = (li - 1) * batch_n * hy_stride + bi * 5 * hy_h; - if(use_dropout) - { - auto dropout_states_tmp = dropout_states_host; - size_t drop_out_offset = (li - 1) * batch_n * hy_h * bi; - - RunDropoutForwardEmulator(handle, - dropoutDesc, - dropout_inputTensor, - dropout_inputTensor, - hid_state, - dropout_outputTensor, - dropout_hid_state, - dropout_reservespace_host, - dropout_states_tmp, - prelayer_shift, - drop_out_offset, - drop_out_offset); - - prelayer_shift = drop_out_offset; - } - - ADNN_mm_cpu(use_dropout ? &dropout_hid_state[prelayer_shift] - : &hid_state[prelayer_shift], - hy_h * bi, - batch_n, - use_dropout ? hy_h * bi : hy_stride, - 0, - &wei_state[wei_shift], - hy_h * bi, - hy_h * bi * 4, - bi_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi * 4, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - size_t wei_shift_bias_temp = wei_shift_bias + li * 2 * wei_stride; - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < wei_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - } - - // from hidden state - bacc = 0; - baccbi = batch_n; - for(int ti = 0; ti < seqLength; ti++) - { - baccbi -= in_n.at(seqLength - 1 - ti); - size_t wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&hx_state[hx_shift], - hy_h, - in_n.at(ti), - uni_stride, - 0, - &wei_state[wei_shift], - hy_h, - hy_h * 4, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h * 4, - in_n.at(ti), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - size_t wei_shift_bias_temp = wei_shift_bias + (li * 2 + 1) * wei_stride; - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < 4 * hy_h; h++) - { - hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - - if(bidirection) - { - ADNN_mm_cpu(&hx_state[hx_shift + hy_n * hy_h], - hy_h, - in_n.at(seqLength - 1 - ti), - uni_stride, - 0, - &wei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + 4 * hy_h], - hy_h * 4, - in_n.at(seqLength - 1 - ti), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + (li * 2 + 1) * wei_stride; - - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - for(int h = 0; h < 4 * hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + 4 * hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + 4 * hy_h + h); - } - } - } - } - } - } - else - { - ADNN_mm_cpu(&hy_state[hx_shift], - hy_h, - in_n.at(ti), - uni_stride, - 0, - &wei_state[wei_shift], - hy_h, - hy_h * 4, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h * 4, - in_n.at(ti), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - size_t wei_shift_bias_temp = wei_shift_bias + (li * 2 + 1) * wei_stride; - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < 4 * hy_h; h++) - { - hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - - if(bidirection) - { - - if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti)) - { - ADNN_mm_cpu( - &hx_state[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h], - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - uni_stride, - 0, - &wei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride + - 4 * hy_h], - hy_h * 4, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - size_t wei_shift_bias_temp = wei_shift_bias + (li * 2 + 1) * wei_stride; - - for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti); - bs++) - { - for(int h = 0; h < 4 * hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + 4 * hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + 4 * hy_h + h); - } - } - } - } - - ADNN_mm_cpu(&hy_state[hx_shift + hy_n * hy_h], - hy_h, - in_n.at(seqLength - ti), - uni_stride, - 0, - &wei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + 4 * hy_h], - hy_h * 4, - in_n.at(seqLength - ti), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - size_t wei_shift_bias_temp = wei_shift_bias + (li * 2 + 1) * wei_stride; - - for(int bs = 0; bs < in_n.at(seqLength - ti); bs++) - { - for(int h = 0; h < 4 * hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + 4 * hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + 4 * hy_h + h); - } - } - } - } - } - - for(int bs = 0; bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h) += - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + h), 2) * - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h), - 1); - if(ti == 0) - { - if(!cx_is_null) - { - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h) += - activfunc( - hid_state.at(hid_shift + (bacc + bs) * hy_stride + hy_h + h), - 2) * - cx_state.at(hx_shift + bs * uni_stride + h); - } - } - else - { - size_t prec_shift = li * batch_n * hy_stride + - (bacc - in_n.at(ti - 1)) * hy_stride + bi * 4 * hy_h; - - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h) += - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + hy_h + h), - 2) * - hid_state.at(prec_shift + bs * hy_stride + h); - } - - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h) += - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h), - 2) * - activfunc( - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h), - 1); - - hid_state.at(hid_shift + (bacc + bs) * hy_stride + h + - numlayer * batch_n * hy_stride) = - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + h), 2); - hid_state.at(hid_shift + (bacc + bs) * hy_stride + hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + hy_h + h), 2); - hid_state.at(hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h), - 2); - hid_state.at(hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc(hid_state.at(hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h), - 1); - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc( - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h), - 1); - - cy_state.at(hx_shift + bs * uni_stride + h) = - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h); - hy_state.at(hx_shift + bs * uni_stride + h) = - hid_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h); - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h) += - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h), - 2) * - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h), - 1); - if(ti == 0) - { - if(!cx_is_null) - { - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h) += - activfunc(hid_state.at(hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h), - 2) * - cx_state.at(hx_shift + bs * uni_stride + hy_n * hy_h + h); - } - } - else - { - - if(!cx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti)) - { - if(bs >= in_n.at(seqLength - ti)) - { - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + - bi * 4 * hy_h + hy_h + h) += - activfunc(hid_state.at(hid_shift + - (baccbi + bs) * hy_stride + - 5 * hy_h + h), - 2) * - cx_state.at(hx_shift + bs * uni_stride + hy_n * hy_h + h); - } - } - - if(bs < in_n.at(seqLength - ti)) - { - size_t prec_shift = - li * batch_n * hy_stride + - (baccbi + in_n.at(seqLength - 1 - ti)) * hy_stride + - bi * 4 * hy_h + hy_h; - - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h) += - activfunc(hid_state.at(hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h), - 2) * - hid_state.at(prec_shift + bs * hy_stride + h); - } - } - - hid_state[hid_shift + (baccbi + bs) * hy_stride + bi * 5 * hy_h + hy_h + - h] += - activfunc( - hid_state[hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h], - 2) * - activfunc(hid_state[hid_shift + (baccbi + bs) * hy_stride + - bi * 4 * hy_h + hy_h + h], - 1); - - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h), - 2); - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h), - 2); - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h), - 2); - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h + - numlayer * batch_n * hy_stride) = - activfunc( - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h), - 1); - hid_state.at(hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h + numlayer * batch_n * hy_stride) = - activfunc(hid_state.at(hid_shift + (baccbi + bs) * hy_stride + - bi * 4 * hy_h + hy_h + h), - 1); - - cy_state.at(hx_shift + bs * uni_stride + hy_n * hy_h + h) = hid_state.at( - hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + h); - hy_state.at(hx_shift + bs * uni_stride + hy_n * hy_h + h) = hid_state.at( - hid_shift + (baccbi + bs) * hy_stride + bi * 5 * hy_h + hy_h + h); - } - } - } - - bacc += in_n.at(ti); - } - } - - // output - size_t prelayer_shift = (numlayer - 1) * batch_n * hy_stride + bi * 5 * hy_h; - - for(int i = 0; i < numlayer * batch_n * hy_stride * 2; i++) - { - rsvspace_host.at(i) = hid_state.at(i); - } - if(use_dropout) - { - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - rsvspace_host.at(numlayer * batch_n * hy_stride * 2 + i) = dropout_hid_state.at(i); - } - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - *(p_drop_rsv + i) = dropout_reservespace_host.at(i); - } - } - - for(int i = 0; i < hy_d * hy_n * hy_h; i++) - { - hy_host.at(i) = hy_state.at(i); - cy_host.at(i) = cy_state.at(i); - } - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - out_host.at(bs * out_stride + h) = hid_state.at(prelayer_shift + bs * hy_stride + h); - } - } -} - -template -void RunLSTMBackwardDataGEMMCPUVerify( - std::vector& din_host, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& dhy, // current/final hidden state - std::vector& dhx_host, - std::vector& hx, // initial hidden state - std::vector& dcy, // current/final cell state - std::vector& dcx_host, - std::vector& cx, - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) - // for unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by - // hy_h related function for bidirection - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool cx_is_null = false, - bool dhy_is_null = false, - bool dcy_is_null = false) -{ - size_t batch_n = sumvc(in_n); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - size_t bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int out_stride = out_h; - int wei_stride = bi * 4 * hy_h; - int hy_stride = bi * 6 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - std::vector dh_state(numlayer * batch_n * hy_stride, static_cast(0)); - std::vector din_state(batch_n * in_h, static_cast(0)); - - // initial dout - std::vector dout_state(batch_n * out_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state[h * out_stride + w] = dout[h * out_stride + w]; - } - } - - // initial hidden states - std::vector dhx_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector dhy_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - dhy_state[h] = dhy[h]; - } - std::vector dcx_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector dcy_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - dcy_state[h] = dcy[h]; - } - std::vector hx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state[h] = hx[h]; - } - std::vector cx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - cx_state[h] = cx[h]; - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_len = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial weights - std::vector wei_state(wei_len, static_cast(0)); - for(int h = 0; h < wei_len; h++) - { - wei_state[h] = wei[h]; - } - - // initial dropoput - miopenTensorDescriptor_t dropout_inputTensor{}; - std::vector dropout_reservespace_host; - if(use_dropout) - { - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(0)); - - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - dropout_reservespace_host.at(i) = *(p_drop_rsv + i); - } - } - - // bwd data emulator - for(int li = numlayer - 1; li >= 0; li--) - { - size_t wei_shift = (in_h + hy_h) * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - size_t hid_shift = li * batch_n * hy_stride; - size_t hx_shift = li * in_n[0] * h_stride; - - if(li == numlayer - 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - dh_state[hid_shift + bi * 5 * hy_h + bs * hy_stride + h] += - dout_state[bs * out_stride + h]; - } - } - } - else - { - size_t prelayer_shift = (li + 1) * batch_n * hy_stride; - - ADNN_mm_cpu(&dh_state[prelayer_shift], - hy_h * bi * 4, - batch_n, - hy_stride, - 0, - &wei_state[wei_shift], - hy_h * bi, - hy_h * bi * 4, - bi_stride, - 0, - &dh_state[hid_shift + bi * 5 * hy_h], - hy_h * bi, - batch_n, - hy_stride, - 0, - 1, - 1); - - if(use_dropout) - { - RunDropoutBackwardEmulator(dropoutDesc, - dropout_inputTensor, - dh_state, - dropout_inputTensor, - dh_state, - dropout_reservespace_host, - hid_shift + bi * 5 * hy_h, - hid_shift + bi * 5 * hy_h, - li * batch_n * hy_h * bi); - } - } - - // from hidden state - bacc = batch_n; - baccbi = 0; - for(int ti = seqLength - 1; ti >= 0; ti--) - { - bacc -= in_n[ti]; - - if(ti == seqLength - 1) - { - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - if(!dhy_is_null) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h] += - dhy_state[hx_shift + bs * uni_stride + h]; - } - if(!dcy_is_null) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] += - dcy_state[hx_shift + bs * uni_stride + h]; - } - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - if(!dhy_is_null) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 5 * hy_h + - hy_h + h] += - dhy_state[hx_shift + bs * uni_stride + hy_n * hy_h + h]; - } - if(!dcy_is_null) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h] += - dcy_state[hx_shift + bs * uni_stride + hy_n * hy_h + h]; - } - } - } - } - } - else - { - - if(!dhy_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - for(int bs = in_n.at(ti + 1); bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h) += - dhy_state.at(hx_shift + bs * uni_stride + h); - } - } - } - - if(!dcy_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - for(int bs = in_n.at(ti + 1); bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h) += - dcy_state[hx_shift + bs * uni_stride + h]; - } - } - } - - size_t pretime_shift = li * batch_n * hy_stride + (bacc + in_n[ti]) * hy_stride; - size_t weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - ADNN_mm_cpu(&dh_state[pretime_shift], - hy_h * 4, - in_n[ti + 1], - hy_stride, - 0, - &wei_state[weitime_shift], - hy_h, - hy_h * 4, - uni_stride, - 0, - &dh_state[hid_shift + bacc * hy_stride + bi * 5 * hy_h], - hy_h, - in_n[ti + 1], - hy_stride, - 0, - 1, - 1); - - if(bidirection) - { - pretime_shift = li * batch_n * hy_stride + - (baccbi - in_n[seqLength - 2 - ti]) * hy_stride + hy_h * 4; - weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride + - hy_h * 4 * uni_stride; - - ADNN_mm_cpu( - &dh_state[pretime_shift], - hy_h * 4, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - &wei_state[weitime_shift], - hy_h, - hy_h * 4, - uni_stride, - 0, - &dh_state[hid_shift + baccbi * hy_stride + bi * 5 * hy_h + hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - } - } - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - if(ti < seqLength - 1) - { - if(bs < in_n[ti + 1]) - { - size_t pretime_shift = - li * batch_n * hy_stride + (bacc + in_n[ti]) * hy_stride; - - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] += - dh_state[pretime_shift + bs * hy_stride + bi * 4 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + hy_h + h], - 2); - } - } - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h], - 1) * - activfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], - 2); - - if(ti == 0) - { - if(!cx_is_null) - { - dh_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] * - cx_state[hx_shift + bs * uni_stride + h] * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + hy_h + h], - 2); - } - } - else - { - size_t pretime_shift = - li * batch_n * hy_stride + (bacc - in_n[ti - 1]) * hy_stride; - - dh_state[hid_shift + (bacc + bs) * hy_stride + hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] * - rsvspace_host[pretime_shift + bs * hy_stride + bi * 4 * hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + hy_h + h], 2); - } - dh_state[hid_shift + (bacc + bs) * hy_stride + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] * - activfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h], - 1) * - dervactivfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], 2); - dh_state[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 5 * hy_h + h] * - activfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h], - 1) * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 2 * hy_h + h], 2); - dh_state[hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h] += - dh_state[hid_shift + (bacc + bs) * hy_stride + bi * 4 * hy_h + h] * - activfunc(rsvspace_host[hid_shift + (bacc + bs) * hy_stride + h], 2) * - dervactivfunc( - rsvspace_host[hid_shift + (bacc + bs) * hy_stride + 3 * hy_h + h], 1); - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n[seqLength - 1 - ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - if(ti < seqLength - 1) - { - size_t pretime_shift = li * batch_n * hy_stride + - (baccbi - in_n[seqLength - 2 - ti]) * hy_stride; - - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h] += - dh_state[pretime_shift + bs * hy_stride + bi * 4 * hy_h + hy_h + - h] * - activfunc( - rsvspace_host[pretime_shift + bs * hy_stride + 5 * hy_h + h], - 2); - } - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 5 * hy_h + hy_h + - h] * - dervactivfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - bi * 4 * hy_h + hy_h + h], - 1) * - activfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h], - 2); - - if(ti == 0) - { - if(!cx_is_null) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h] * - cx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 2); - } - } - else - { - if(!cx_is_null && - in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti) && - bs >= in_n.at(seqLength - ti)) - { - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h] * - cx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 2); - } - - if(bs < in_n[seqLength - ti]) - { - size_t pretime_shift = - li * batch_n * hy_stride + - (baccbi + in_n[seqLength - 1 - ti]) * hy_stride; - - dh_state[hid_shift + (baccbi + bs) * hy_stride + 5 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + - hy_h + h] * - rsvspace_host[pretime_shift + bs * hy_stride + bi * 4 * hy_h + - hy_h + h] * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - 5 * hy_h + h], - 2); - } - } - dh_state[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h] * - activfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h], - 1) * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], - 2); - dh_state[hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 5 * hy_h + hy_h + - h] * - activfunc(rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + - bi * 4 * hy_h + hy_h + h], - 1) * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 6 * hy_h + h], - 2); - dh_state[hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h] += - dh_state[hid_shift + (baccbi + bs) * hy_stride + bi * 4 * hy_h + hy_h + - h] * - activfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 4 * hy_h + h], - 2) * - dervactivfunc( - rsvspace_host[hid_shift + (baccbi + bs) * hy_stride + 7 * hy_h + h], - 1); - } - } - } - - baccbi += in_n[seqLength - 1 - ti]; - } - - // dcx, dhx - size_t pretime_shift = li * batch_n * hy_stride; - size_t weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - ADNN_mm_cpu(&dh_state[pretime_shift], - hy_h * 4, - in_n[0], - hy_stride, - 0, - &wei_state[weitime_shift], - hy_h, - hy_h * 4, - uni_stride, - 0, - &dhx_state[hx_shift], - hy_h, - in_n[0], - uni_stride, - 0, - 1, - 1); - - for(int bs = 0; bs < in_n.at(0); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dcx_state[hx_shift + bs * uni_stride + h] += - dh_state[pretime_shift + bs * hy_stride + bi * 4 * hy_h + h] * - activfunc(rsvspace_host[pretime_shift + bs * hy_stride + hy_h + h], 2); - } - } - - if(bidirection) - { - int ti = seqLength - 1, cur_bat = 0, pre_bat = batch_n; - - while(ti >= 0) - { - pre_bat -= in_n.at(ti); - if(in_n.at(ti) > cur_bat) - { - pretime_shift = li * batch_n * hy_stride + (pre_bat + cur_bat) * hy_stride; - - ADNN_mm_cpu(&dh_state[pretime_shift + 4 * hy_h], - hy_h * 4, - (in_n.at(ti) - cur_bat), - hy_stride, - 0, - &wei_state[weitime_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - 0, - &dhx_state[hx_shift + hy_n * hy_h + cur_bat * hy_h], - hy_h, - (in_n.at(ti) - cur_bat), - uni_stride, - 0, - 1, - 1); - - for(int bs = cur_bat; bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dcx_state[hx_shift + bs * uni_stride + hy_n * hy_h + h] += - dh_state[pretime_shift + (bs - cur_bat) * hy_stride + - bi * 4 * hy_h + hy_h + h] * - activfunc(rsvspace_host[pretime_shift + (bs - cur_bat) * hy_stride + - 5 * hy_h + h], - 2); - } - } - } - cur_bat = in_n.at(ti--); - } - } - } - - // dinput - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - for(int gi = 0; gi < 4; gi++) - { - din_state[bs * in_stride + h] += dh_state[bs * hy_stride + gi * hy_h + h]; - if(bidirection) - { - din_state[bs * in_stride + h] += - dh_state[bs * hy_stride + (gi + 4) * hy_h + h]; - } - } - } - } - } - else - { - ADNN_mm_cpu(dh_state.data(), - hy_h * bi * 4, - batch_n, - hy_stride, - 0, - wei_state.data(), - in_h, - hy_h * bi * 4, - in_stride, - 0, - din_state.data(), - in_h, - batch_n, - in_stride, - 0, - 1, - 1); - } - - for(int i = 0; i < numlayer * batch_n * hy_stride; i++) - { - wkspace_host[i] = dh_state[i]; - } - - for(int i = 0; i < hy_d * hy_n * hy_h; i++) - { - dhx_host[i] = dhx_state[i]; - dcx_host[i] = dcx_state[i]; - } - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < in_stride; h++) - { - din_host[bs * in_stride + h] = din_state[bs * in_stride + h]; - } - } -} - -template -void RunLSTMBackwardWeightGEMMCPUVerify(std::vector& in, - std::vector& dwei_host, // [ input_state_weight_trans - // hidden_state_weight0_trans - // input1_trans hidden1_trans ... - // output_weight; bidirectional - // reversed weights ] - std::vector& hx, // initial hidden state - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden - // layers) for unidirection, 2 by numlayer for - // bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 - // by hy_h related function for bidirection - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - bool hx_is_null = false) -{ - size_t batch_n = sumvc(in_n); - int numlayer = bidirection ? hy_d / 2 : hy_d; - size_t bacc; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int wei_stride = bi * 4 * hy_h; - int hy_stride = bi * 6 * hy_h; - int h_stride = bi * hy_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - // initial input - std::vector in_state(batch_n * in_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state[h * in_h + w] = in[h * in_h + w]; - } - } - - // initial output difference - std::vector dout_state(batch_n * out_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state[h * out_h + w] = dout[h * out_h + w]; - } - } - - // initial saved data - std::vector wkspace_state(numlayer * batch_n * hy_stride, static_cast(0)); - for(int h = 0; h < numlayer * batch_n * hy_stride; h++) - { - wkspace_state[h] = wkspace_host[h]; - } - std::vector rsvspace_state( - use_dropout ? rsvspace_host.size() : numlayer * batch_n * hy_stride, static_cast(0)); - for(int h = 0; h < rsvspace_state.size(); h++) - { - rsvspace_state[h] = rsvspace_host[h]; - } - - // initial hidden states - std::vector hx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state[h] = hx[h]; - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - size_t wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (numlayer - 1)) * wei_stride; - int wei_len = wei_shift_bias; - if(biased) - { - int in_bias = 2; - wei_len += (in_bias + (numlayer - 1) * 2) * wei_stride; - } - - // initial dwei - std::vector dwei_state(wei_len, static_cast(0)); - - // bwd weights emulator - for(int li = 0; li < numlayer; li++) - { - // between layers - if(li == 0) - { - if(inputMode != 1) - { - ADNN_mm_cpu(wkspace_state.data(), - hy_h * bi * 4, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - in_state.data(), - in_h, - batch_n, - in_stride, - 0, - dwei_state.data(), - in_h, - hy_h * bi * 4, - in_stride, - 0, - 1, - 1); - } - - if(biased) - { - for(int h = 0; h < wei_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state[wei_shift_bias + h] += wkspace_host[w * hy_stride + h]; - } - } - } - } - else - { - size_t prelayer_shift = - use_dropout ? 2 * numlayer * batch_n * hy_stride + (li - 1) * batch_n * hy_h * bi - : (li - 1) * batch_n * hy_stride + bi * hy_h * 5; - size_t hid_shift = li * batch_n * hy_stride; - size_t wei_shift = - (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h * bi * 4, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[prelayer_shift], - hy_h * bi, - batch_n, - use_dropout ? hy_h * bi : hy_stride, - 0, - &dwei_state[wei_shift], - hy_h * bi, - hy_h * bi * 4, - bi_stride, - 0, - 1, - 1); - - if(biased) - { - wei_shift = wei_shift_bias + li * 2 * wei_stride; - - for(int h = 0; h < wei_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state[wei_shift + h] += wkspace_host[hid_shift + w * hy_stride + h]; - } - } - } - } - - // between time - bacc = 0; - for(int ti = 0; ti < seqLength; ti++) - { - size_t hid_shift = li * batch_n * hy_stride + bacc * hy_stride; - size_t hx_shift = li * in_n[0] * h_stride; - size_t wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int pretime_shift; - - // between time - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h * 4, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift], - hy_h, - in_n[ti], - uni_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h * 4, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - size_t bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 4; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + h] += - wkspace_host[hid_shift + w * hy_stride + h]; - } - } - } - } - } - else - { - pretime_shift = - li * batch_n * hy_stride + (bacc - in_n[ti - 1]) * hy_stride + bi * 5 * hy_h; - - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h * 4, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[pretime_shift], - hy_h, - in_n[ti], - hy_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h * 4, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - size_t bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 4; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + h] += - wkspace_host[hid_shift + w * hy_stride + h]; - } - } - } - } - - if(bidirection) - { - if(ti == seqLength - 1) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&wkspace_state[hid_shift + 4 * hy_h], - hy_h * 4, - in_n[ti], - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift + hy_n * hy_h], - hy_h, - in_n[ti], - uni_stride, - 0, - &dwei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - size_t bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 4; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state[bias_shift + hy_h * 4 + h] += - wkspace_host[hid_shift + hy_h * 4 + w * hy_stride + h]; - } - } - } - } - } - else - { - if(!hx_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - ADNN_mm_cpu( - &wkspace_state[hid_shift + 4 * hy_h + in_n.at(ti + 1) * hy_stride], - hy_h * 4, - (in_n.at(ti) - in_n.at(ti + 1)), - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift + hy_n * hy_h + in_n.at(ti + 1) * hy_h], - hy_h, - (in_n.at(ti) - in_n.at(ti + 1)), - uni_stride, - 0, - &dwei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - size_t bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 4; h++) - { - for(int w = in_n.at(ti + 1); w < in_n.at(ti); w++) - { - dwei_state.at(bias_shift + hy_h * 4 + h) += - wkspace_host.at(hid_shift + hy_h * 4 + w * hy_stride + h); - } - } - } - } - - pretime_shift = - li * batch_n * hy_stride + (bacc + in_n[ti]) * hy_stride + bi * 5 * hy_h; - - ADNN_mm_cpu(&wkspace_state[hid_shift + 4 * hy_h], - hy_h * 4, - in_n[ti + 1], - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[pretime_shift + hy_h], - hy_h, - in_n[ti + 1], - hy_stride, - 0, - &dwei_state[wei_shift + 4 * hy_h * uni_stride], - hy_h, - hy_h * 4, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - size_t bias_shift = wei_shift_bias + li * 2 * wei_stride + wei_stride; - - for(int h = 0; h < hy_h * 4; h++) - { - for(int w = 0; w < in_n.at(ti + 1); w++) - { - dwei_state[bias_shift + hy_h * 4 + h] += - wkspace_host[hid_shift + hy_h * 4 + w * hy_stride + h]; - } - } - } - } - } - - bacc += in_n[ti]; - } - } - - for(int i = 0; i < wei_len; i++) - { - dwei_host[i] = dwei_state[i]; - } -} - -#endif // GUARD_MIOPEN_LSTM_VERIFY_GEMM_HPP diff --git a/driver/main.cpp b/driver/main.cpp index 6389cb2a84..e69de29bb2 100644 --- a/driver/main.cpp +++ b/driver/main.cpp @@ -1,112 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include "driver.hpp" -#include "registry_driver_maker.hpp" - -#include -#include - -#include -#include - -int main(int argc, char* argv[]) -{ - - std::string base_arg = ParseBaseArg(argc, argv); - - if(base_arg == "--version") - { - size_t major, minor, patch; - miopenGetVersion(&major, &minor, &patch); - std::cout << "MIOpen (version: " << major << "." << minor << "." << patch << ")" - << std::endl; - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - // show command - std::cout << "MIOpenDriver"; - for(int i = 1; i < argc; i++) - std::cout << " " << argv[i]; - std::cout << std::endl; - - Driver* drv = nullptr; - for(auto f : rdm::GetRegistry()) - { - drv = f(base_arg); - if(drv != nullptr) - break; - } - if(drv == nullptr) - { - printf("Incorrect BaseArg\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - drv->AddCmdLineArgs(); - int rc = drv->ParseCmdLineArgs(argc, argv); - if(rc != 0) - { - std::cout << "ParseCmdLineArgs() FAILED, rc = " << rc << std::endl; - return rc; - } - drv->GetandSetData(); - rc = drv->AllocateBuffersAndCopy(); - if(rc != 0) - { - std::cout << "AllocateBuffersAndCopy() FAILED, rc = " << rc << std::endl; - return rc; - } - - int fargval = - !miopen::StartsWith(base_arg, "CBAInfer") ? drv->GetInputFlags().GetValueInt("forw") : 1; - bool bnFwdInVer = (fargval == 2 && miopen::StartsWith(base_arg, "bnorm")); - bool verifyarg = (drv->GetInputFlags().GetValueInt("verify") == 1); - int cumulative_rc = 0; // Do not stop running tests in case of errors. - - if(fargval & 1 || fargval == 0 || bnFwdInVer) - { - rc = drv->RunForwardGPU(); - cumulative_rc |= rc; - if(rc != 0) - std::cout << "RunForwardGPU() FAILED, rc = " - << "0x" << std::hex << rc << std::dec << std::endl; - if(verifyarg) // Verify even if Run() failed. - cumulative_rc |= drv->VerifyForward(); - } - - if(fargval != 1) - { - rc = drv->RunBackwardGPU(); - cumulative_rc |= rc; - if(rc != 0) - std::cout << "RunBackwardGPU() FAILED, rc = " - << "0x" << std::hex << rc << std::dec << std::endl; - if(verifyarg) // Verify even if Run() failed. - cumulative_rc |= drv->VerifyBackward(); - } - - return cumulative_rc; -} diff --git a/driver/mloConvHost.hpp b/driver/mloConvHost.hpp index c0f6272801..e69de29bb2 100644 --- a/driver/mloConvHost.hpp +++ b/driver/mloConvHost.hpp @@ -1,1176 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MLO_CONVHOST_H_ -#define MLO_CONVHOST_H_ - -#include - -#include -#include - -#include "calcerr.hpp" - -//#if 0 // disable functions -#if 1 -//////////////////////////////////////////////////////////// -// -/////////////////////////////////////////////////////////// -#define ADNN_MM_TRANSPOSE 1 -template -void ADNN_mm_cpu(const Dtype* a_ptr, - size_t a_cols, - size_t a_rows, - size_t a_stride, - int a_flags, - const Dtype* b_ptr, - size_t b_cols, - size_t b_rows, - size_t b_stride, - int b_flags, - Dtype* c_ptr, - size_t c_cols, - size_t c_rows, - size_t c_stride, - int /*c_flags*/, - double d_alpha, - double d_beta) -{ - // mA - - // mB - - // mC - Dtype alpha = Dtype(d_alpha); - Dtype beta = Dtype(d_beta); - if((!(a_flags & ADNN_MM_TRANSPOSE) && !(b_flags & ADNN_MM_TRANSPOSE) && - ((a_cols != b_rows) || (a_rows != c_rows) || (b_cols != c_cols))) || - ((a_flags & ADNN_MM_TRANSPOSE) && (b_flags & ADNN_MM_TRANSPOSE) && - ((a_rows != b_cols) || (a_cols != c_rows) || (b_rows != c_cols))) || - ((a_flags & ADNN_MM_TRANSPOSE) && !(b_flags & ADNN_MM_TRANSPOSE) && - ((a_rows != b_rows) || (a_cols != c_rows) || (b_cols != c_cols))) || - (!(a_flags & ADNN_MM_TRANSPOSE) && (b_flags & ADNN_MM_TRANSPOSE) && - ((a_cols != b_cols) || (a_rows != c_rows) || (b_rows != c_cols)))) - { - printf("MM_CPU ERROR; %zu %zu %zu %zu %zu %zu\n", - a_cols, - a_rows, - b_cols, - b_rows, - c_rows, - c_cols); - return; - } - - size_t inner_loop = (!(a_flags & ADNN_MM_TRANSPOSE)) ? a_cols : a_rows; - - if(!(a_flags & ADNN_MM_TRANSPOSE) && !(b_flags & ADNN_MM_TRANSPOSE)) - { - for(size_t n = 0; n < c_rows; ++n) - { - for(size_t k = 0; k < c_cols; ++k) - { - Dtype mm_e = static_cast(0); - for(size_t m = 0; m < inner_loop; ++m) - { - mm_e += a_ptr[n * a_stride + m] * b_ptr[m * b_stride + k]; - } - c_ptr[n * c_stride + k] = beta * c_ptr[n * c_stride + k] + alpha * mm_e; - } - } - } - else if((a_flags & ADNN_MM_TRANSPOSE) && !(b_flags & ADNN_MM_TRANSPOSE)) - { - for(size_t n = 0; n < c_rows; ++n) - { - for(size_t k = 0; k < c_cols; ++k) - { - - Dtype mm_e = static_cast(0); - for(size_t m = 0; m < inner_loop; ++m) - { - mm_e += a_ptr[m * a_stride + n] * b_ptr[m * b_stride + k]; -#if 0 - if ( - (n == 0 && k == 33 - || n == 1 && k == 32 - || n == 3 && k == 1 - || n == 4 && k == 0 - - ) - && a_ptr[m*a_stride + n] * b_ptr[m*b_stride + k] != 0 - ) - { - printf("C:mm:%d %d %d %11.9f %11.9f %11.9f %11.9f\n", - n, k, m, - mm_e, a_ptr[m*a_stride + n], b_ptr[m*b_stride + k], a_ptr[m*a_stride + n] * b_ptr[m*b_stride + k]); - } -#endif - } - c_ptr[n * c_stride + k] = beta * c_ptr[n * c_stride + k] + alpha * mm_e; - } - } - } - else if(!(a_flags & ADNN_MM_TRANSPOSE) && (b_flags & ADNN_MM_TRANSPOSE)) - { - for(size_t n = 0; n < c_rows; ++n) - { - for(size_t k = 0; k < c_cols; ++k) - { - Dtype mm_e = static_cast(0); - - for(size_t m = 0; m < inner_loop; ++m) - { - mm_e += a_ptr[n * a_stride + m] * b_ptr[k * b_stride + m]; -#if 0 - if (n == 0 && k == 6 && a_ptr[n*a_stride + m] * b_ptr[k*b_stride + m] != 0) - { - printf("%4d %11.9f %11.9f %11.9f\n", m, mm_e, a_ptr[n*a_stride + m], b_ptr[k*b_stride + m]); - } -#endif - } - c_ptr[n * c_stride + k] = beta * c_ptr[n * c_stride + k] + alpha * mm_e; - } - } - } - else - { - for(size_t n = 0; n < c_rows; ++n) - { - for(size_t k = 0; k < c_cols; ++k) - { - Dtype mm_e = static_cast(0); - for(size_t m = 0; m < inner_loop; ++m) - { - c_ptr[n * c_stride + k] += a_ptr[m * a_stride + n] * b_ptr[k * b_stride + m]; - } - c_ptr[n * c_stride + k] = beta * c_ptr[n * c_stride + k] + alpha * mm_e; - } - } - } -} - -template -void ADNN_im2col_cpu(const Dtype* data_im, - const int channels, - const int height, - const int width, - const int ksize_h, - const int ksize_w, - const int pad, - const int stride, - Dtype* data_col, - int stride_col = 0) -{ - int height_col = (height + 2 * pad - ksize_h) / stride + 1; - int width_col = (width + 2 * pad - ksize_w) / stride + 1; - height_col = (height_col < 0) ? 1 : height_col; - width_col = (width_col < 0) ? 1 : width_col; - stride_col = (stride_col == 0) ? height_col * width_col : stride_col; - int channels_col = channels * ksize_h * ksize_w; - for(int c = 0; c < channels_col; ++c) - { - int w_offset = c % ksize_w; - int h_offset = (c / ksize_w) % ksize_h; - int c_im = c / ksize_h / ksize_w; - for(int h = 0; h < height_col; ++h) - { - for(int w = 0; w < width_col; ++w) - { - int h_pad = h * stride - pad + h_offset; - int w_pad = w * stride - pad + w_offset; - if(h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) - { - data_col[c * stride_col + h * width_col + w] = - data_im[(c_im * height + h_pad) * width + w_pad]; - } - else - { - data_col[c * stride_col + h * width_col + w] = 0; - } - } - } - } -} - -template -void ADNN_col2im_cpu(const Dtype* data_col, - const int channels, - const int height, - const int width, - const int ksize_h, - const int ksize_w, - const int pad, - const int stride, - Dtype* data_im) -{ - memset(data_im, 0, sizeof(Dtype) * height * width * channels); - int height_col = (height + 2 * pad - ksize_h) / stride + 1; - int width_col = (width + 2 * pad - ksize_w) / stride + 1; - height_col = (height_col < 0) ? 1 : height_col; - width_col = (width_col < 0) ? 1 : width_col; - int channels_col = channels * ksize_h * ksize_w; - for(int c = 0; c < channels_col; ++c) - { - int w_offset = c % ksize_w; - int h_offset = (c / ksize_w) % ksize_h; - int c_im = c / ksize_h / ksize_w; - for(int h = 0; h < height_col; ++h) - { - for(int w = 0; w < width_col; ++w) - { - int h_pad = h * stride - pad + h_offset; - int w_pad = w * stride - pad + w_offset; - if(h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) - { - data_im[(c_im * height + h_pad) * width + w_pad] += - data_col[(c * height_col + h) * width_col + w]; -#if 0 - if (c_im == 3 && h_pad == 30 && w_pad == 23) - { - printf("C:c2i: %d %d %d %d %d %d %14.12f %14.12f\n", c, h * width_col + w, w, h, w_pad, h_pad, data_im[(c_im * height + h_pad) * width + w_pad], data_col[(c * height_col + h) * width_col + w]); - } -#endif - } - } - } - } -} - -template -int mloConvForwarDirectOnHost( - T_ padding_value, // padding value - int filter_size_h, // kernel 1 dim - int pad_h, // padding size - int conv_stride_h, // scale factor - int filter_size_w, // kernel 1 dim - int pad_w, // padding size - int conv_stride_w, // scale factor - int n_batchs, - int n_outputs, - int n_inputs, - int top_height, - int top_width, - int top_batch_stride, - int top_channel_stride, - int top_stride, - int bot_height, - int bot_width, - int bot_batch_stride, - int bot_channel_stride, - int bot_stride, - int weights_stride, - const T_* bot_ptr, // input "tensor" - batch x channels (input images, feature maps, slices) x - // width x height - T_* top_ptr, // output "te4nsor" - batch x channels (output images, feature maps, slices) x - // width (scaled) x height (scaled) - const T_* - weights_ptr, // weights n output channels x n input channels x filter size_y x filter size_x - const T_* bias_ptr = NULL // bias, NULL if no bias -) -{ - int ret = 0; - const T_* run_bot_ptr = bot_ptr; - T_* run_top_ptr = top_ptr; - const T_* run_weights_ptr = weights_ptr; - - // over all batches - for(int b = 0; b < n_batchs; - b++, run_bot_ptr += bot_batch_stride, run_top_ptr += top_batch_stride) - { - run_weights_ptr = weights_ptr; - // over all output channels - for(int o = 0; o < n_outputs; o++) - { - // sum up convolutions - // over output image (scaled input) - for(int j = 0; j < top_height; j++) - { - for(int i = 0; i < top_width; i++) - { - // over all input channels - T_ accum = 0; - for(int c = 0; c < n_inputs; c++) - { - // do convolution with kernel kernel_size x kerenl_size - // with padding - left, right, top, bottom = pad, and value = 0 - for(int k_j = 0; k_j < filter_size_h; k_j++) - { - - int in_y = (j * conv_stride_h + k_j - pad_h); - for(int k_i = 0; k_i < filter_size_w; k_i++) - { - int in_x = (i * conv_stride_w + k_i - pad_w); - T_ data_val = padding_value; - if(!(in_y < 0 || in_x < 0 || in_y >= bot_height || - in_x >= bot_width)) - { - int in_data_off = - c * bot_channel_stride + in_y * bot_stride + in_x; - data_val = run_bot_ptr[in_data_off]; - } - - T_ wei_val = run_weights_ptr[o * weights_stride + - c * filter_size_h * filter_size_w + - k_j * filter_size_w + k_i]; - - accum += data_val * wei_val; -#if 0 - if (b == 0 && o == 0 && j == 1 && i == 0) - { - printf("c: %f %f %f\n", - accum/* + bias_ptr[o]*/, - data_val, - wei_val - ); - } -#endif - } - } - } - - T_ final_val = (bias_ptr) ? accum + bias_ptr[o] : accum; - run_top_ptr[o * top_channel_stride + j * top_stride + i] = final_val; - } - } - } - } - - return (ret); -} - -template -int mloBackwardMMOnHost(int kernel_size_h, - int kernel_size_w, - int pad, - int stride, - const T_* weights_ptr, - int weights_height, - int weights_width, - int weights_stride, - const T_* top_df_ptr, - int top_height, - int top_width, - int outputs, - int batch_sz, - int top_df_batch_stride, - int top_df_channel_stride, - int /*top_df_stride*/, - T_* bot_df_ptr, - int bot_height, - int bot_width, - int inputs, - int bot_df_batch_stride, - int /*bot_df_channel_stride*/, - int /*bot_df_stride*/ - -) -{ - - int col_we_df_width = top_width * top_height; - int col_we_df_height = weights_width; // - bias - int col_we_batch_stride = col_we_df_width * col_we_df_height; - int col_we_stride = col_we_df_width; - T_* col_we_df_ptr = new T_[col_we_batch_stride * batch_sz]; - - assert(col_we_df_ptr); - - for(int b = 0; b < batch_sz; ++b) - { - ADNN_mm_cpu(weights_ptr, - weights_width, - weights_height, - weights_stride, - ADNN_MM_TRANSPOSE, - &T_(&top_df_ptr[top_df_batch_stride * b]), - top_width * top_height, - outputs, - top_df_channel_stride, - 0, - &col_we_df_ptr[col_we_batch_stride * b], - col_we_df_width, - col_we_df_height, - col_we_stride, - 0, - 1, - 0); //- bias - - ADNN_col2im_cpu(&col_we_df_ptr[col_we_batch_stride * b], - inputs, - bot_height, - bot_width, - kernel_size_h, - kernel_size_w, - pad, - stride, - &bot_df_ptr[bot_df_batch_stride * b]); - } - if(col_we_df_ptr) - { - delete[] col_we_df_ptr; - } - - return (0); -} - -template -int mloBackwardDirectOnHost( - T_ /*padding_value*/, // padding value - // TO DO: check top, bot dim are equal - int filter_size_h, // kernel 1 dim - int pad_h, // padding size - int conv_stride_h, // scale factor - int filter_size_w, // kernel 1 dim - int pad_w, // padding size - int conv_stride_w, // scale factor - int n_batchs, - int n_outputs, - int n_inputs, - int top_height, - int top_width, - int top_batch_stride, - int top_channel_stride, - int top_stride, - int bot_width, - int bot_height, - int bot_batch_stride, - int bot_channel_stride, - int bot_stride, - int weights_stride, - T_* bot_ptr, // input "tensor" - batch x channels (input images, feature maps, slices) x width x - // height - const T_* top_ptr, // output "te4nsor" - batch x channels (output images, feature maps, slices) - // x width (scaled) x height (scaled) - const T_* - weights_ptr // weights n output channels x n input channels x filter size_y x filter size_x -) -{ - int ret = 0; - T_* run_bot_ptr = bot_ptr; - const T_* run_top_ptr = top_ptr; - const T_* run_weights_ptr = weights_ptr; - - // over all batches - for(int b = 0; b < n_batchs; - b++, run_bot_ptr += bot_batch_stride, run_top_ptr += top_batch_stride) - { - run_weights_ptr = weights_ptr; - // over all output channels - for(int c = 0; c < n_inputs; ++c) - { - // sum up convolutions - for(int o = 0; o < n_outputs; ++o) - { - - for(int j = 0; j < top_height; ++j) - { - - for(int i = 0; i < top_width; ++i) - { - - int out_data_off = o * top_channel_stride + j * top_stride + i; - T_ data_val = run_top_ptr[out_data_off]; - // over all input channels - T_ accum = 0; - // do convolution with kernel kernel_size x kerenl_size - // with padding - left, right, top, bottom = pad, and value = 0 - for(int k_j = 0; k_j < filter_size_h; ++k_j) - { - int bot_y = (j * conv_stride_h + k_j - pad_h); - // int top_y = (j + filter_size_h - 1 - k_j); - for(int k_i = 0; k_i < filter_size_w; ++k_i) - { - // int top_x = (i + filter_size_w - 1 - k_i); - int bot_x = (i * conv_stride_w + k_i - pad_w); - if(!(bot_y < 0 || bot_x < 0 || bot_y >= bot_height || - bot_x >= bot_width)) - { - T_ wei_val = run_weights_ptr[o * weights_stride + - c * filter_size_h * filter_size_w + - k_j * filter_size_w + k_i]; - - int bot_data_off = - c * bot_channel_stride + bot_y * bot_stride + bot_x; - run_bot_ptr[bot_data_off] += data_val * wei_val; - -#if 0 - if (b == 0 && o == 0 && bot_y == 0 && bot_x == 0) - { - printf("c: %d %d %d %d %d %d %f %f %f\n", - bot_data_off, - k_j, - k_i, - j, - i, - out_data_off, - run_bot_ptr[bot_data_off], - data_val, - wei_val - ); - } -#endif - } - } - } - } - } - } - } - } - - return (ret); -} - -template -void mloPrepad(int o, - int c, - int j, - int i, - T_ padding_value, // padding value - int pad_w, // padding size - int pad_h, // padding size - int bot_batch_stride, - int bot_channel_stride, - int bot_stride, - int bot_height, - int bot_width, - int new_bot_batch_stride, - int new_bot_channel_stride, - int new_bot_stride, - int /*new_bot_height*/, - int /*new_bot_width*/, - T_* new_bot_ptr, - const T_* bot_ptr // input "tensor" - batch x channels (input images, feature maps, - // slices) x width x height -) -{ - int src_j = j - pad_h; - int src_i = i - pad_w; - int src_off = o * bot_batch_stride + c * bot_channel_stride + src_j * bot_stride + src_i; - int dst_off = o * new_bot_batch_stride + c * new_bot_channel_stride + j * new_bot_stride + i; - - if(src_i >= 0 && src_i < bot_width && src_j >= 0 && src_j < bot_height) - { - new_bot_ptr[dst_off] = bot_ptr[src_off]; - } - else - { - new_bot_ptr[dst_off] = padding_value; - } -} - -template -int mloDirectSPHostPrepad(T_ padding_value, // padding value - int pad_w, // padding size - int pad_h, // padding size - int n_batchs, - int n_inputs, - int bot_batch_stride, - int bot_channel_stride, - int bot_stride, - int bot_height, - int bot_width, - int new_bot_batch_stride, - int new_bot_channel_stride, - int new_bot_stride, - int new_bot_height, - int new_bot_width, - T_* new_bot_ptr, - const T_* bot_ptr // input "tensor" - batch x channels (input images, - // feature maps, slices) x width x height -) -{ - int ret = 0; - for(int o = 0; o < n_batchs; ++o) - { - for(int c = 0; c < n_inputs; ++c) - { - for(int j = 0; j < new_bot_height; ++j) - { - for(int i = 0; i < new_bot_width; ++i) - { - mloPrepad(o, - c, - j, - i, - padding_value, // padding value - pad_w, // padding size - pad_h, // padding size - bot_batch_stride, - bot_channel_stride, - bot_stride, - bot_height, - bot_width, - new_bot_batch_stride, - new_bot_channel_stride, - new_bot_stride, - new_bot_height, - new_bot_width, - new_bot_ptr, - bot_ptr); - } - } - } - } - - return (ret); -} - -/* -weihts interleave -n inputs - n output blocks (1 block == n output tiles) - rows 1 * n output tiles - rows 2 * n output tiles - rows 3 * n output tiles -*/ - -template -void mloInterleavelWeightsInOutputs(int c, - int o_block, - int o, - int k_j, - int k_i, - int filter_size_w, // kernel 1 dim - int filter_size_h, // kernel 1 dim - int n_outputs, - int MLO_N_OUT_TILES, - int n_inputs, - const T_* wei_ptr, - T_* new_wei_ptr) -{ - int src_off = (o_block * MLO_N_OUT_TILES + o) * n_inputs * filter_size_h * filter_size_w + - c * filter_size_h * filter_size_w + k_j * filter_size_w + k_i; - int dst_off = c * n_outputs * filter_size_h * filter_size_w + - o_block * MLO_N_OUT_TILES * filter_size_h * filter_size_w + - MLO_N_OUT_TILES * k_j * filter_size_w + o * filter_size_w + k_i; - new_wei_ptr[dst_off] = wei_ptr[src_off]; -} - -template -void mloDirectSPHostIntlWeights(bool forward, // forwad = 1, backward = 0 - int filter_size_w, // kernel 1 dim - int filter_size_h, // kernel 1 dim - int n_outputs, - int n_inputs, - int MLO_N_OUT_TILES, - const T_* wei_ptr, - T_* new_wei_ptr) -{ - if(forward) - { - // interleave all - // outputs per input - int o_loops = (n_outputs + MLO_N_OUT_TILES - 1) / MLO_N_OUT_TILES; - - for(int c = 0; c < n_inputs; ++c) - { - for(int o_block = 0; o_block < o_loops; ++o_block) - { - for(int o = 0; o < MLO_N_OUT_TILES && o_block * MLO_N_OUT_TILES + o < n_outputs; - ++o) - { - for(int k_j = 0; k_j < filter_size_h; ++k_j) - { - for(int k_i = 0; k_i < filter_size_w; ++k_i) - { - mloInterleavelWeightsInOutputs(c, - o_block, - o, - k_j, - k_i, - filter_size_w, // kernel 1 dim - filter_size_h, // kernel 1 dim - n_outputs, - MLO_N_OUT_TILES, - n_inputs, - wei_ptr, - new_wei_ptr); - } - } - } - } - } - } -} - -template -int mloDirectSPConvHost5x5(int MLO_GRP_SZ1, - int MLO_GRP_SZ0, - int MLO_OUT_TILE1, - int MLO_OUT_TILE0, - int MLO_N_OUT_TILES, - int MLO_KERNEL_SZ1, // kernel 1 dim - int MLO_FILTER_SIZE1, // kernel 1 dim - int MLO_N_BATCHS, - int MLO_N_OUTPUTS, - int MLO_TOP_BATCH_STRIDE, - int MLO_TOP_CHANNEL_STRIDE, - int MLO_TOP_STRIDE, - int MLO_TOP_HEIGHT, - int MLO_TOP_WIDTH, - int MLO_N_INPUTS, - int MLO_BOT_BATCH_STRIDE, - int MLO_BOT_CHANNEL_STRIDE, - int MLO_BOT_STRIDE, - int /*MLO_BOT_HEIGHT*/, - int /*MLO_BOT_WIDTH*/, - const T_* bot_ptr, // input "tensor" - batch x channels (input images, - // feature maps, slices) x width x height - // interleaved weights - const T_* wei_ptr, // weights n output channels x n input channels x - // filter size_y x filter size_x - T_* top_ptr, // output "te4nsor" - batch x channels (output images, - // feature maps, slices) x width (scaled) x height (scaled) - const T_* /*bias_ptr = NULL*/ // bias, NULL if no bias - -) -{ - int j_loops = - (MLO_TOP_HEIGHT + MLO_GRP_SZ1 * MLO_OUT_TILE1 - 1) / (MLO_GRP_SZ1 * MLO_OUT_TILE1); - int i_loops = (MLO_TOP_WIDTH + MLO_GRP_SZ0 * MLO_OUT_TILE0 - 1) / (MLO_GRP_SZ0 * MLO_OUT_TILE0); - int o_loops = (MLO_N_OUTPUTS + MLO_N_OUT_TILES - 1) / MLO_N_OUT_TILES; - - T_* bot_stage = new T_[MLO_OUT_TILE1 * (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1)]; - T_* wei_stage = new T_[MLO_FILTER_SIZE1]; - T_* out_tiles = new T_[MLO_OUT_TILE1 * MLO_OUT_TILE0 * MLO_N_OUT_TILES]; - - for(int g2 = 0; g2 < o_loops * MLO_N_BATCHS; ++g2) - { - for(int g1 = 0; g1 < j_loops; ++g1) - { - for(int g0 = 0; g0 < i_loops; ++g0) - { - for(int l1 = 0; - l1 < MLO_GRP_SZ1 && (g1 * MLO_GRP_SZ1 + l1) * MLO_OUT_TILE1 < MLO_TOP_HEIGHT; - ++l1) - { - for(int l0 = 0; - l0 < MLO_GRP_SZ0 && (g0 * MLO_GRP_SZ0 + l0) * MLO_OUT_TILE1 < MLO_TOP_WIDTH; - ++l0) - { - // KERNEL - int glbl1 = (g1 * MLO_GRP_SZ1 + l1) * MLO_OUT_TILE1; - int glbl0 = (g0 * MLO_GRP_SZ0 + l0) * MLO_OUT_TILE0; - int o_block = g2 / MLO_N_BATCHS; - int b = g2 - o_block * MLO_N_BATCHS; // batch - // position of on the map of the top-left input pixel - // bot stride may include additional padding zeros from prepadding - int bot_off = b * MLO_BOT_BATCH_STRIDE + MLO_BOT_STRIDE * glbl1 + glbl0; - // weight are interleaved - int o_base = o_block * MLO_N_OUT_TILES; - int wei_off = o_base * MLO_KERNEL_SZ1 * MLO_FILTER_SIZE1; - for(int ii = 0; ii < MLO_OUT_TILE1 * MLO_OUT_TILE0 * MLO_N_OUT_TILES; ++ii) - { - out_tiles[ii] = 0; - } - // the only place where we jump - for(int c = 0; c < MLO_N_INPUTS; ++c, - bot_off += MLO_BOT_CHANNEL_STRIDE, - wei_off += MLO_KERNEL_SZ1 * MLO_FILTER_SIZE1 * MLO_N_OUTPUTS) - { - int bot_off1 = bot_off; - - // read first MLO_OUT_TILE1 - 1 lines of input - for(int j = 0; j < MLO_OUT_TILE1 - 1; ++j, bot_off1 += MLO_BOT_STRIDE) - { - for(int i = 0; i < (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1); ++i) - { - bot_stage[j * (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1) + i] = - bot_ptr[bot_off1 + i]; - } - } - // now all weights are sequencially located - // see transformed layout - int wei_off1 = wei_off; - for(int k_j = 0; k_j < MLO_KERNEL_SZ1; - ++k_j, bot_off1 += MLO_BOT_STRIDE) - { - // insertn new line - for(int i = 0; i < (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1); ++i) - { - bot_stage[(MLO_OUT_TILE1 - 1) * - (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1) + - i] = bot_ptr[bot_off1 + i]; - } - // loop over outputs - - for(int o_i = 0; o_i < MLO_N_OUT_TILES; ++o_i) - { - - // read filter coeff row - for(int w_i = 0; w_i < MLO_FILTER_SIZE1; ++w_i) - { - // moving along the weights - wei_stage[w_i] = wei_ptr[wei_off1++]; - } - // convolve - for(int k_i = 0; k_i < MLO_FILTER_SIZE1; ++k_i) - { - for(int m = 0; m < MLO_OUT_TILE1; ++m) - { - for(int l = 0; l < MLO_OUT_TILE0; ++l) - { - out_tiles[o_i * MLO_OUT_TILE1 * MLO_OUT_TILE0 + - m * MLO_OUT_TILE0 + l] += - bot_stage[m * (MLO_OUT_TILE0 + - MLO_FILTER_SIZE1 - 1) + - k_i + l] * - wei_stage[k_i]; -#if 0 - if (o_i == 1 && l1 == 0 && l0 == 0 && g0==0 && g1==0 && g2==0) - { - printf("ek: %f %f %f\n", - out_tiles[o_i * MLO_OUT_TILE1 * MLO_OUT_TILE0 + m*MLO_OUT_TILE0 + l], - bot_stage[m * (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1) + k_i + l], - wei_stage[k_i] - ); - } -#endif - } - } - } - - } // for (int o_i = 0; o_i < MLO_N_OUT_TILES; ++o_i, wei_off1 += - // MLO_FILTER_SIZE1) - - // move data up - for(int up = 0; up < MLO_OUT_TILE1 - 1; ++up) - { - for(int r = 0; r < (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1); ++r) - { - bot_stage[up * (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1) + r] = - bot_stage[(up + 1) * - (MLO_OUT_TILE0 + MLO_FILTER_SIZE1 - 1) + - r]; - } - } - - } // for (int k_j = 0; k_j < MLO_KERNEL_SZ1; ++k_j, bot_off1 += - // MLO_BOT_STRIDE, wei_off += MLO_FILTER_SIZE1 * MLO_N_OUTPUTS * - // MLO_N_INPUTS) - } // for (int c = 0; c < MLO_N_INPUTS; ++c, bot_off += - // MLO_BOT_CHANNEL_STRIDE) - - // output - int out_off = b * MLO_TOP_BATCH_STRIDE + o_base * MLO_TOP_CHANNEL_STRIDE + - glbl1 * MLO_TOP_STRIDE + glbl0; - for(int o = 0; o < MLO_N_OUT_TILES && o_base + o < MLO_N_OUTPUTS; - ++o, out_off += MLO_TOP_CHANNEL_STRIDE) - { - int out_off1 = out_off; - for(int j = 0; j < MLO_OUT_TILE1; ++j, out_off1 += MLO_TOP_STRIDE) - { - for(int i = 0; i < MLO_OUT_TILE0; ++i) - { - top_ptr[out_off1 + i] = - out_tiles[o * MLO_OUT_TILE1 * MLO_OUT_TILE0 + - j * MLO_OUT_TILE0 + i]; - } - } - } - - } // for (int l0 = 0; l0 < MLO_SPC_GRP0 && (g0*MLO_SPC_GRP0 + l0) * - // MLO_OUT_TILE1 < MLO_TOP_WIDTH; ++l0) - } - } - } - } - - delete[] bot_stage; - delete[] out_tiles; - delete[] wei_stage; - - return (0); -} - -template -int mloDirectSPHost( - - int MLO_SPC_GRP1, - int MLO_SPC_GRP0, - int MLO_OUT_TILE1, - int MLO_OUT_TILE0, - int MLO_N_OUT_TILES, - - bool forward, - bool do_input_copy, - T_ padding_value, // padding value - // TO DO: check top, bot dim are equal - int filter_size_w, // kernel 1 dim - int pad_w, // padding size - int /*conv_stride_w*/, // scale factor - int filter_size_h, // kernel 1 dim - int pad_h, // padding size - int /*conv_stride_h*/, // scale factor - int n_batchs, - int n_outputs, - int top_batch_stride, - int top_channel_stride, - int top_stride, - int top_height, - int top_width, - int n_inputs, - int bot_batch_stride, - int bot_channel_stride, - int bot_stride, - int bot_height, - int bot_width, - int new_bot_batch_stride, - int new_bot_channel_stride, - int new_bot_stride, - int new_bot_height, - int new_bot_width, - T_* new_bot_ptr, - const T_* bot_ptr, // input "tensor" - batch x channels (input images, feature maps, slices) x - // width x height - - const T_* - wei_ptr, // weights n output channels x n input channels x filter size_y x filter size_x - T_* new_wei_ptr, - T_* top_ptr, // output "te4nsor" - batch x channels (output images, feature maps, slices) x - // width (scaled) x height (scaled) - const T_* bias_ptr = NULL // bias, NULL if no bias - -) -{ - int ret = 0; - if(do_input_copy) - { - mloDirectSPHostPrepad(padding_value, // padding value - pad_w, // padding size - pad_h, // padding size - n_batchs, - n_inputs, - bot_batch_stride, - bot_channel_stride, - bot_stride, - bot_height, - bot_width, - new_bot_batch_stride, - new_bot_channel_stride, - new_bot_stride, - new_bot_height, - new_bot_width, - new_bot_ptr, - bot_ptr // input "tensor" - batch x channels (input images, - // feature maps, slices) x width x height - ); - } - - mloDirectSPHostIntlWeights(forward, // forwad = 1, backward = 0 - filter_size_w, // kernel 1 dim - filter_size_h, // kernel 1 dim - n_outputs, - n_inputs, - MLO_N_OUT_TILES, - wei_ptr, - new_wei_ptr); - - mloDirectSPConvHost5x5(MLO_SPC_GRP1, - MLO_SPC_GRP0, - MLO_OUT_TILE1, - MLO_OUT_TILE0, - MLO_N_OUT_TILES, - filter_size_h, - filter_size_w, - n_batchs, - n_outputs, - top_batch_stride, - top_channel_stride, - top_stride, - top_height, - top_width, - n_inputs, - new_bot_batch_stride, - new_bot_channel_stride, - new_bot_stride, - new_bot_height, - new_bot_width, - - new_bot_ptr, - new_wei_ptr, - top_ptr, // output "te4nsor" - batch x channels (output images, - // feature maps, slices) x width (scaled) x height (scaled) - bias_ptr); - return (ret); -} - -#endif // disable functions - -template -bool mloVerify(const miopenTensorDescriptor_t& cpu_, - const miopenTensorDescriptor_t& gpu_, - const Tcheck_* c_ptr, - const Tgpu_* g_ptr, - float ulps_tolerance, - Tcheck_ diff_tolerance, - double rms_tolerance, - bool check_ulps, - double& report_err) -{ - const auto& cpu = miopen::deref(cpu_); - const auto& gpu = miopen::deref(gpu_); - - const auto spatial_dim = cpu.GetLengths().size() - 2; - - size_t n_batchs, n_channels, depth, height, width; - size_t c_batch_stride, c_channel_stride, c_depth_stride, c_height_stride, c_width_stride; - size_t g_batch_stride, g_channel_stride, g_depth_stride, g_height_stride, g_width_stride; - - std::tie(n_batchs, n_channels, depth, height, width) = - miopen::GetNCDHW(spatial_dim, cpu.GetLengths()); - std::tie(c_batch_stride, c_channel_stride, c_depth_stride, c_height_stride, c_width_stride) = - miopen::GetNCDHW(spatial_dim, cpu.GetStrides()); - std::tie(g_batch_stride, g_channel_stride, g_depth_stride, g_height_stride, g_width_stride) = - miopen::GetNCDHW(spatial_dim, gpu.GetStrides()); - - bool match = true; - double rms_accum = 0.0; - Tcheck_ worst_c_val = static_cast(0); - Tcheck_ worst_g_val = static_cast(0); - Tcheck_ worst_diff = static_cast(0); - size_t worst_b = 0, worst_c = 0, worst_i = 0, worst_j = 0, worst_k = 0; - - for(size_t b = 0; b < n_batchs; ++b) - { - for(size_t c = 0; c < n_channels; ++c) - { - for(size_t k = 0; k < depth; ++k) - { - for(size_t j = 0; j < height; ++j) - { - for(size_t i = 0; i < width; ++i) - { - Tcheck_ c_val = - c_ptr[b * c_batch_stride + c * c_channel_stride + k * c_depth_stride + - j * c_height_stride + i * c_width_stride]; - Tcheck_ g_val = static_cast( - g_ptr[b * g_batch_stride + c * g_channel_stride + k * g_depth_stride + - j * g_height_stride + i * g_width_stride]); - - Tcheck_ diff = std::abs(c_val - g_val); - rms_accum += diff * diff; - // Register worst (max) abs error and its position. - // This info will be used to show additional diagnostics, - // but only if sgr_accum is too big. - if(diff > worst_diff) - { - worst_diff = diff; - worst_c_val = c_val; - worst_g_val = g_val; - worst_b = b; - worst_c = c; - worst_i = i; - worst_j = j; - worst_k = k; - } - } - } - } - } - } - - const double rms = std::sqrt( - rms_accum / (static_cast(n_batchs * n_channels * depth * height * width))); - report_err = rms; - - if(rms > rms_tolerance || std::isnan(rms) || !std::isfinite(rms)) - { - match = false; - - std::cout << "RMS too big: " << rms << ". Max diff: " << worst_diff << " at {" << worst_b - << ',' << worst_c << ','; - if(spatial_dim == 3) - std::cout << worst_k << ','; - std::cout << worst_j << ',' << worst_i << "}, cpu_v = " << worst_c_val - << " vs gpu_v = " << worst_g_val << std::endl; - } - - if(check_ulps) - { - static int n_logged = 0; - for(size_t b = 0; b < n_batchs && match; ++b) - { - for(size_t c = 0; c < n_channels && match; ++c) - { - for(size_t k = 0; k < depth && match; ++k) - { - for(size_t j = 0; j < height && match; ++j) - { - for(size_t i = 0; i < width && match; ++i) - { - auto c_val = - static_cast(c_ptr[b * c_batch_stride + c * c_channel_stride + - k * c_depth_stride + j * c_height_stride + - i * c_width_stride]); - auto g_val = - static_cast(g_ptr[b * g_batch_stride + c * g_channel_stride + - k * g_depth_stride + j * g_height_stride + - i * g_width_stride]); - - const auto diff = std::abs(c_val - g_val); - const auto ulps = ApproxUlps(c_val, g_val); - const bool check_failed = - (diff > diff_tolerance && ulps > ulps_tolerance) // - || std::isnan(c_val) // - || std::isnan(g_val) // - || !std::isfinite(c_val) // - || !std::isfinite(g_val); - - if(check_failed) - match = false; - - if(check_failed) - { - if(!(n_logged >= 10)) - { - std::cout << "ULPs: " << ulps; - if(check_failed) - std::cout << " is too large (> " << ulps_tolerance << ")"; - std::cout << " at {" << b << ',' << c << ','; - if(spatial_dim == 3) - std::cout << k << ','; - std::cout << j << ',' << i << "}, cpu_val = " << c_val - << ", gpu_val = " << g_val << " (diff = " << diff - << ')' << std::endl; - ++n_logged; - if(n_logged >= 10) - std::cout << "(too many lines logged, truncating output...)" - << std::endl; - } - } - } - } - } - } - } - } - return match; -} - -#endif diff --git a/driver/mloCumulativeReductionHost.hpp b/driver/mloCumulativeReductionHost.hpp new file mode 100644 index 0000000000..6b4c60d993 --- /dev/null +++ b/driver/mloCumulativeReductionHost.hpp @@ -0,0 +1,151 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include <../test/ford.hpp> + +#include +#include +#include + +#include + +inline constexpr void update() {} + +template +inline constexpr void update(T& a, T b, Ts&... c, Ts... d) +{ + a = b; + update(c..., d...); +} + +template +struct reduce_func_base +{ + reduce_func_base(){}; + virtual ~reduce_func_base(){}; + virtual inline bool isbetter(const T& /*a*/, const T& /*b*/) const { return false; } + virtual inline void combine(T& a, T b) const { a = b; } + inline constexpr void calculate(T& a, T b, Ts&... c, Ts... d) const + { + if(!isbetter(a, b)) + { + combine(a, b); + update(c..., d...); + } + } +}; + +template +struct reduce_func : reduce_func_base +{ + virtual ~reduce_func(){}; +}; + +template +struct reduce_func : reduce_func_base +{ + const float START_VAL = -std::numeric_limits::max(); + inline bool isbetter(const T& a, const T& b) const { return a > b; } +}; + +template +struct reduce_func : reduce_func_base +{ + const float START_VAL = std::numeric_limits::max(); + inline bool isbetter(const T& a, const T& b) const { return a < b; } +}; + +template +struct reduce_func : reduce_func_base +{ + const float START_VAL = 0.0f; + inline void combine(T& a, T b) const { a += b; } +}; + +template +struct reduce_func : reduce_func_base +{ + const float START_VAL = 1.0f; + inline void combine(T& a, T b) const { a *= b; } +}; + +template +int32_t mloCumulativeReductionForwardRunHost(const miopenTensorDescriptor_t inputDesc, + const miopenTensorDescriptor_t outputDesc, + const miopenTensorDescriptor_t indicesDesc, + const Tgpu* input, + Tcheck* output_host, + int* indices_host, + const int dim, + const bool exclusive, + const bool reverse) +{ + const int ndims = miopen::deref(inputDesc).GetNumDims(); + const auto exec_dim = ((dim % ndims) + ndims) % ndims; + + auto input_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(inputDesc)); + auto output_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(outputDesc)); + auto indices_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(indicesDesc)); + + auto size = miopen::deref(inputDesc).GetElementSize(); + auto inner_size = miopen::deref(inputDesc).GetLengths()[exec_dim]; + auto outer_size = size / inner_size; + + auto op_worker = reduce_func{}; + + tensor_view_t<5> ignore_dim_input_tv = input_tv; + ignore_dim_input_tv.size[exec_dim] = 1; + + par_ford(outer_size)([&](int gid) { + auto tensor_layout = tensor_layout_t<5>(ignore_dim_input_tv, gid); + float cum_val = op_worker.START_VAL; + int cum_idx = (reverse ? input_tv.size[exec_dim] - 1 : 0); + + ford(inner_size)([&](int idx) { + int tmp_idx = + (reverse ? input_tv.size[exec_dim] - (idx - exclusive) - 1 : (idx - exclusive)); + float tmp_val = op_worker.START_VAL; + if(0 <= tmp_idx && tmp_idx < inner_size) + { + tensor_layout.layout[exec_dim] = tmp_idx; + tmp_val = static_cast(input[input_tv.get_tensor_view_idx(tensor_layout)]); + } + + op_worker.calculate(cum_val, tmp_val, cum_idx, tmp_idx); + + tensor_layout.layout[exec_dim] = (reverse ? input_tv.size[exec_dim] - idx - 1 : idx); + if(output_host) + output_host[output_tv.get_tensor_view_idx(tensor_layout)] = + static_cast(cum_val); + if(indices_host) + indices_host[indices_tv.get_tensor_view_idx(tensor_layout)] = cum_idx; + }); + }); + + return miopenStatusSuccess; +} diff --git a/driver/rnn_driver.hpp b/driver/rnn_driver.hpp index 22f3d63131..e69de29bb2 100644 --- a/driver/rnn_driver.hpp +++ b/driver/rnn_driver.hpp @@ -1,1777 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_RNN_DRIVER_HPP -#define GUARD_MIOPEN_RNN_DRIVER_HPP - -#include "InputFlags.hpp" -#include "driver.hpp" -#include "gru_verify_gemm.hpp" -#include "lstm_verify_gemm.hpp" -#include "mloConvHost.hpp" -#include "random.hpp" -#include "rnn_verify_gemm.hpp" -#include "tensor_driver.hpp" -#include "timer.hpp" -#include "util_driver.hpp" -#include "util_file.hpp" - -#include <../test/verify.hpp> - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -template -class RNNDriver : public Driver -{ -public: - RNNDriver() : Driver() - { - miopenCreateTensorDescriptor(&inputTensor); - miopenCreateTensorDescriptor(&hiddenTensor); - miopenCreateTensorDescriptor(&weightTensor); - miopenCreateTensorDescriptor(&outputTensor); - - miopenCreateRNNDescriptor(&rnnDesc); - workspace_dev = nullptr; - reservespace_dev = nullptr; - data_type = (sizeof(Tgpu) == 4) ? miopenFloat : miopenHalf; - - miopenCreateDropoutDescriptor(&DropoutDesc); - } - - int AddCmdLineArgs() override; - int ParseCmdLineArgs(int argc, char* argv[]) override; - InputFlags& GetInputFlags() override { return inflags; } - - int GetandSetData() override; - std::vector GetInputTensorLengthsFromCmdLine(); - std::vector GetHiddenTensorLengthsFromCmdLine(); - std::vector GetWeightTensorLengthsFromCmdLine(); - std::vector GetOutputTensorLengthsFromCmdLine(); - - int SetRNNDescriptorFromCmdLineArgs(); - int AllocateBuffersAndCopy() override; - - int RunForwardGPU() override; - int RunForwardCPU(); - int RunBackwardGPU() override; - int RunBackwardDataCPU(); - int RunBackwardWeightsCPU(); - int VerifyBackward() override; - int VerifyForward() override; - ~RNNDriver() override - { - miopenDestroyTensorDescriptor(outputTensor); - miopenDestroyTensorDescriptor(weightTensor); - miopenDestroyTensorDescriptor(hiddenTensor); - miopenDestroyTensorDescriptor(inputTensor); - - miopenDestroyRNNDescriptor(rnnDesc); - } - -private: - InputFlags inflags; - - std::vector inputTensors; - std::vector outputTensors; - miopenTensorDescriptor_t inputTensor; - miopenTensorDescriptor_t hiddenTensor; - miopenTensorDescriptor_t weightTensor; - miopenTensorDescriptor_t outputTensor; - - std::unique_ptr in_dev; - std::unique_ptr din_dev; - std::unique_ptr wei_dev; - std::unique_ptr dwei_dev; - std::unique_ptr out_dev; - std::unique_ptr dout_dev; - std::unique_ptr hx_dev; - std::unique_ptr cx_dev; - std::unique_ptr hy_dev; - std::unique_ptr cy_dev; - std::unique_ptr dhx_dev; - std::unique_ptr dcx_dev; - std::unique_ptr dhy_dev; - std::unique_ptr dcy_dev; - std::unique_ptr workspace_dev; - std::unique_ptr reservespace_dev; - std::unique_ptr dropout_states_dev; - - std::vector in; - std::vector din; - std::vector wei; - std::vector dwei; - std::vector out; - std::vector dout; - std::vector hx; - std::vector cx; - std::vector hy; - std::vector cy; - std::vector dhx; - std::vector dcx; - std::vector dhy; - std::vector dcy; - std::vector workspace; - std::vector reservespace; - std::vector outhost; - std::vector workspace_host; - std::vector reservespace_host; - std::vector din_host; - std::vector dwei_host; - std::vector hy_host; - std::vector cy_host; - std::vector dhx_host; - std::vector dcx_host; - std::vector dropout_states_host; - - miopenRNNDescriptor_t rnnDesc; - - int batchsize; - int adjustedSeqLen; - std::vector batchseq; - - miopenDropoutDescriptor_t DropoutDesc; - float dropout_rate; - unsigned long long dropout_seed; - - // std::string GetVerificationCacheFileName() const; - // bool TryReadVerificationCache(const std::string& file_name, - // miopenTensorDescriptor_t& tensorDesc, - // Tgpu* data) const; - // void TrySaveVerificationCache(const std::string& file_name, std::vector& data) - // const; -}; - -static inline bool CheckGuard(const int& in_h, - const int& out_h, - const int& hy_d, - const int& hy_n, - const int& hy_h, - const miopenRNNDirectionMode_t& dirMode, - const miopenRNNInputMode_t& inputMode) -{ - return (in_h == 0 || out_h == 0 || hy_d == 0 || hy_n == 0 || hy_h == 0 || - out_h != ((dirMode + 1) * hy_h) || (inputMode == miopenRNNskip && in_h != hy_h)); -} - -template -int RNNDriver::ParseCmdLineArgs(int argc, char* argv[]) -{ - inflags.Parse(argc, argv); - - return miopenStatusSuccess; -} - -template -int RNNDriver::GetandSetData() -{ - std::vector in_len = GetInputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - std::vector wei_len = GetWeightTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - - for(int i = 0; i < in_len.size() - 1; i++) - { - std::array in_lens = {{in_len[i], in_len.back()}}; - miopenCreateTensorDescriptor(&inputTensor); - miopenSetTensorDescriptor(inputTensor, data_type, 2, in_lens.data(), nullptr); - inputTensors.push_back(inputTensor); - - std::array out_lens = {{in_len[i], out_len[0]}}; - miopenCreateTensorDescriptor(&outputTensor); - miopenSetTensorDescriptor(outputTensor, data_type, 2, out_lens.data(), nullptr); - outputTensors.push_back(outputTensor); - } - - std::array hid_lens = {{hid_len[0], in_len[0], hid_len[1]}}; - miopenSetTensorDescriptor(hiddenTensor, data_type, 3, hid_lens.data(), nullptr); - - SetRNNDescriptorFromCmdLineArgs(); - miopenGetRNNParamsDescriptor(handle, rnnDesc, inputTensor, weightTensor, data_type); - - return miopenStatusSuccess; -} - -template -int RNNDriver::AddCmdLineArgs() -{ - inflags.AddInputFlag("forw", - 'F', - "0", - "Run only Forward RNN == 1 or only Backward Data RNN == 2, Backward " - "Weights = 4 or both == 0 (Default=0)", - "int"); - inflags.AddInputFlag("num_layer", 'l', "1", "Number of hidden stacks (Default=1)", "int"); - inflags.AddInputFlag( - "seq_len", 'k', "1", "Number of iterations to unroll over (Default=10)", "int"); - inflags.AddInputFlag( - "bidirection", 'r', "0", "uni- or bi-direction, default uni- (Default=0)", "int"); - inflags.AddInputFlag("batchsize", 'n', "4", "Mini-batch size (Default=4)", "vector"); - inflags.AddInputFlag("hid_h", 'H', "32", "Hidden State Length (Default=32)", "int"); - inflags.AddInputFlag("in_h", 'W', "32", "Input Length (Default=32)", "int"); - inflags.AddInputFlag("iter", 'i', "1", "Number of Iterations (Default=1)", "int"); - inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); - /* - // DL: This is disabled below, so I'm turninig it off here - inflags.AddInputFlag("verification_cache", - 'C', - "", - "Use specified directory to cache verification data. Off by default.", - "string"); - */ - inflags.AddInputFlag( - "wall", - 'w', - "0", - "Wall-clock, for host and gpu, Time Each Layer, Disabled = 0,\ - OldWallClock = 1,\ - SeparateClocksSynced = 2,\ - SeparateClocksNotSynced = 3 \ - (Default = 0) ", - "int"); - inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); - /* // DL: These have not been implemented. Removing them for now. - inflags.AddInputFlag("in_data", 'd', "", "Input data filename (Default=)", "string"); - inflags.AddInputFlag("weights", 'e', "", "Input weights filename (Default=)", "string");*/ - inflags.AddInputFlag("bias", 'b', "", "Use Bias (Default=0)", "int"); - inflags.AddInputFlag( - "mode", 'm', "tanh", "RNN Mode (relu, tanh, lstm, gru) (Default=tanh)", "str"); - inflags.AddInputFlag("inputmode", 'p', "0", "linear == 0 or skip == 1, (Default=0)", "int"); - inflags.AddInputFlag( - "use_padding", 'q', "0", "packed tensors == 0 or padded == 1, (Default=0)", "int"); - inflags.AddInputFlag("rnnalgo", 'a', "0", "default, fundamental (Default=0)", "int"); - inflags.AddInputFlag("fwdtype", - 'c', - "0", - "RNN forward being training or inference, Default training (Default=0)", - "int"); - inflags.AddInputFlag("datatype", 'f', "1", "16-bit or 32-bit fp (Default=1)", "int"); - - inflags.AddInputFlag( - "use_dropout", 'U', "0", "Use dropout: 1; Not use dropout: 0 (Default=0)", "int"); - inflags.AddInputFlag("dropout", 'P', "0.0", "Dropout rate (Default=0.0)", "float"); - inflags.AddInputFlag( - "seed_low", 'L', "0", "Least significant 32 bits of seed (Default=0)", "int"); - inflags.AddInputFlag( - "seed_high", 'M', "0", "Most significant 32 bits of seed (Default=0)", "int"); - - return 0; -} - -template -std::vector RNNDriver::GetInputTensorLengthsFromCmdLine() -{ - int nseq = inflags.GetValueInt("seq_len"); - int in_h = inflags.GetValueInt("in_h"); - std::vector in_n(nseq, 0); - std::string batchstr = inflags.GetValueStr("batchsize"); - - std::stringstream ss(batchstr); - int cont = 0; - - int element; - while(ss >> element) - { - if(cont >= nseq) - { - printf("Length of data sequence is longer than required unrolled time sequence " - "provided.\n" - "The data sequence will be truncated to match unrolled time sequence.\n"); - break; - } - - if(ss.peek() == ',' || ss.peek() == ' ') - { - ss.ignore(); - } - - if(cont > 0 && in_n[cont] > in_n[cont - 1]) - { - printf("Incorrect input batch size at time %d\n", cont); - return std::vector({0}); - } - else - { - in_n[cont] = element; - cont++; - } - } - - adjustedSeqLen = nseq; - - if(nseq > cont) - { - printf("length of data sequence == %d is short than time sequence == %d, padding the rest " - "of data sequence with %d\n", - cont, - nseq, - in_n[cont - 1]); - for(int i = cont; i < nseq; i++) - { - in_n[i] = in_n[cont - 1]; - } - } - - in_n.push_back(in_h); - - return in_n; -} - -template -std::vector RNNDriver::GetHiddenTensorLengthsFromCmdLine() -{ - int hid_h = inflags.GetValueInt("hid_h"); - int hid_l = inflags.GetValueInt("num_layer"); - if((inflags.GetValueInt("bidirection")) == 1) - hid_l *= 2; - - return std::vector({hid_l, hid_h}); -} - -template -std::vector RNNDriver::GetWeightTensorLengthsFromCmdLine() -{ - int wei_ih = inflags.GetValueInt("in_h"); - int wei_hh = inflags.GetValueInt("hid_h"); - int wei_oh; - int wei_l = inflags.GetValueInt("num_layer"); - int wei_bi = 1; - if((inflags.GetValueInt("bidirection")) == 1) - wei_bi = 2; - wei_oh = wei_hh * wei_bi; - - int wei_sc = 1; - if((inflags.GetValueStr("mode")) == "lstm") - wei_sc = 4; - else if((inflags.GetValueStr("mode")) == "gru") - wei_sc = 3; - - return std::vector({wei_bi, wei_l, wei_ih, wei_hh, wei_oh, wei_sc}); -} - -template -int RNNDriver::SetRNNDescriptorFromCmdLineArgs() -{ - - int layer = inflags.GetValueInt("num_layer"); - int wei_hh = inflags.GetValueInt("hid_h"); // hidden state size - - miopenRNNMode_t mode; - - if((inflags.GetValueStr("mode")) == "relu") - { - mode = miopenRNNRELU; - } - else if((inflags.GetValueStr("mode")) == "tanh") - { - mode = miopenRNNTANH; - } - else if((inflags.GetValueStr("mode")) == "lstm") - { - mode = miopenLSTM; - } - else if((inflags.GetValueStr("mode")) == "gru") - { - mode = miopenGRU; - } - else - { - printf("Incorrect RNN Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNBiasMode_t biasMode; - if((inflags.GetValueInt("bias")) == 0) - { - biasMode = miopenRNNNoBias; - } - else if((inflags.GetValueInt("bias")) == 1) - { - biasMode = miopenRNNwithBias; - } - else - { - printf("Incorrect bias Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNDirectionMode_t directionMode; - if((inflags.GetValueInt("bidirection")) == 0) - { - directionMode = miopenRNNunidirection; - } - else if((inflags.GetValueInt("bidirection")) == 1) - { - directionMode = miopenRNNbidirection; - } - else - { - printf("Incorrect direction Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNInputMode_t inMode; - if((inflags.GetValueInt("inputmode")) == 0) - { - inMode = miopenRNNlinear; - } - else if((inflags.GetValueInt("inputmode")) == 1) - { - inMode = miopenRNNskip; - } - else - { - printf("Incorrect input Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNAlgo_t algo; - if((inflags.GetValueInt("rnnalgo")) == 0) - { - algo = miopenRNNdefault; - } - else if((inflags.GetValueInt("rnnalgo")) == 1) - { - algo = miopenRNNfundamental; - } - else - { - printf("Incorrect RNN algorithm\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - if(inflags.GetValueInt("use_dropout")) - { - dropout_rate = static_cast(inflags.GetValueDouble("dropout")); - auto dropout_seed_low = - static_cast(std::max(inflags.GetValueInt("seed_low"), 0)); - auto dropout_seed_high = - static_cast(std::max(inflags.GetValueInt("seed_high"), 0)); - dropout_seed = dropout_seed_high << 32 | dropout_seed_low; - - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - dropout_states_dev = - std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); - - miopenSetDropoutDescriptor(DropoutDesc, - GetHandle(), - dropout_rate, - dropout_states_dev->GetMem(), - dropout_states_dev->GetSize(), - dropout_seed, - false, - false, - MIOPEN_RNG_PSEUDO_XORWOW); - - miopenSetRNNDescriptor_V2(rnnDesc, - wei_hh, - layer, - DropoutDesc, - inMode, - directionMode, - mode, - biasMode, - algo, - data_type); - } - else - { - miopenSetRNNDescriptor( - rnnDesc, wei_hh, layer, inMode, directionMode, mode, biasMode, algo, data_type); - } - if(inflags.GetValueInt("use_padding")) - { - miopenSetRNNPaddingMode(rnnDesc, miopenRNNPaddingMode_t::miopenRNNIOWithPadding); - } - - return miopenStatusSuccess; -} - -template -std::vector RNNDriver::GetOutputTensorLengthsFromCmdLine() -{ - int hid_h = inflags.GetValueInt("hid_h"); - int bi = (inflags.GetValueInt("bidirection") == 1) ? 2 : 1; - - int out_h = hid_h * bi; - return std::vector({out_h}); -} - -template -int RNNDriver::AllocateBuffersAndCopy() -{ - - size_t in_sz = 0; - size_t out_sz = 0; - size_t wei_sz = 0; - size_t hy_sz = 0; - size_t workSpace_sz; - size_t reserveSpace_sz; - - miopenGetRNNInputTensorSize(GetHandle(), rnnDesc, adjustedSeqLen, inputTensors.data(), &in_sz); - miopenGetRNNInputTensorSize( - GetHandle(), rnnDesc, adjustedSeqLen, outputTensors.data(), &out_sz); - miopenGetRNNHiddenTensorSize(GetHandle(), rnnDesc, adjustedSeqLen, inputTensors.data(), &hy_sz); - miopenGetRNNWorkspaceSize( - GetHandle(), rnnDesc, adjustedSeqLen, inputTensors.data(), &workSpace_sz); - miopenGetRNNTrainingReserveSize( - GetHandle(), rnnDesc, adjustedSeqLen, inputTensors.data(), &reserveSpace_sz); - miopenGetRNNParamsSize(GetHandle(), rnnDesc, inputTensors[0], &wei_sz, data_type); - - in_sz /= sizeof(Tgpu); - out_sz /= sizeof(Tgpu); - hy_sz /= sizeof(Tgpu); - wei_sz /= sizeof(Tgpu); - workSpace_sz /= sizeof(Tgpu); - reserveSpace_sz = (reserveSpace_sz + sizeof(Tgpu) - 1) / sizeof(Tgpu); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - hx_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - wei_dev = std::unique_ptr(new GPUMem(ctx, wei_sz, sizeof(Tgpu))); - cx_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - workspace_dev = std::unique_ptr(new GPUMem(ctx, workSpace_sz, sizeof(Tgpu))); - reservespace_dev = std::unique_ptr(new GPUMem(ctx, reserveSpace_sz, sizeof(Tgpu))); - - if(inflags.GetValueInt("forw") != 2) - { - hy_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - cy_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - } - - if(inflags.GetValueInt("forw") != 1) - { - din_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - dwei_dev = std::unique_ptr(new GPUMem(ctx, wei_sz, sizeof(Tgpu))); - dout_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - dhx_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - dcx_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - dhy_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - dcy_dev = std::unique_ptr(new GPUMem(ctx, hy_sz, sizeof(Tgpu))); - } - - in = std::vector(in_sz); - hx = std::vector(hy_sz, static_cast(0)); - wei = std::vector(wei_sz); - out = std::vector(out_sz, static_cast(0)); - cx = std::vector(hy_sz, static_cast(0)); - - if(inflags.GetValueInt("forw") != 2) - { - hy = std::vector(hy_sz, static_cast(0)); - cy = std::vector(hy_sz, static_cast(0)); - hy_host = std::vector(hy_sz, static_cast(0)); - cy_host = std::vector(hy_sz, static_cast(0)); - } - - workspace = std::vector(workSpace_sz, static_cast(0)); - reservespace = std::vector(reserveSpace_sz, static_cast(0)); - outhost = std::vector(out_sz, static_cast(0)); - workspace_host = std::vector(workSpace_sz, static_cast(0)); - - int nseq = inflags.GetValueInt("seq_len"); - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::size_t inputBatchLenSum; - inputBatchLenSum = std::accumulate(in_n.begin(), in_n.begin() + nseq, 0ULL); - - int hid_h = inflags.GetValueInt("hid_h"); - int layer = inflags.GetValueInt("num_layer"); - int bidir = inflags.GetValueInt("bidirection"); - - // not checked - // TODO remove - size_t reserveSpaceHost_sz = - 2 * - (inflags.GetValueStr("mode") == "lstm" ? 6 - : (inflags.GetValueStr("mode") == "gru" ? 4 : 1)) * - layer * inputBatchLenSum * hid_h * (bidir + 1); - if(inflags.GetValueInt("use_dropout")) - { - reserveSpaceHost_sz += (layer - 1) * inputBatchLenSum * hid_h * (bidir + 1); - reserveSpaceHost_sz *= sizeof(Tref); - reserveSpaceHost_sz += (layer - 1) * inputBatchLenSum * hid_h * (bidir + 1); - reserveSpaceHost_sz = (reserveSpaceHost_sz + sizeof(Tref) - 1) / sizeof(Tref); - } - reservespace_host = std::vector(reserveSpaceHost_sz, static_cast(0)); - - if(inflags.GetValueInt("forw") != 1) - { - din = std::vector(in_sz, static_cast(0)); - dwei = std::vector(wei_sz, static_cast(0)); - dout = std::vector(out_sz, static_cast(0)); - dhx = std::vector(hy_sz, static_cast(0)); - dcx = std::vector(hy_sz, static_cast(0)); - dhy = std::vector(hy_sz, static_cast(0)); - dcy = std::vector(hy_sz, static_cast(0)); - din_host = std::vector(in_sz, static_cast(0)); - dwei_host = std::vector(wei_sz, static_cast(0)); - dhx_host = std::vector(hy_sz, static_cast(0)); - dcx_host = std::vector(hy_sz, static_cast(0)); - } - - /* // Not implemented. - std::string inFileName = inflags.GetValueStr("in_data"); - std::string weiFileName = inflags.GetValueStr("weights");*/ - - double scale = 0.01; - - /* bool dataRead = false; - bool weiRead = false; - if(!inFileName.empty()) - { - std::cerr << "Loading input data from file is a feature which has not been implemented - in MIOpenDriver." << std::endl; - // Not implemented. - //dataRead = readBufferFromFile(in.data(), in_sz, inFileName.c_str()); - } - */ - - for(int i = 0; i < in_sz; i++) - { - in[i] = static_cast(prng::gen_0_to_B(scale)); - } - - for(int i = 0; i < hy_sz; i++) - { - hx[i] = static_cast(prng::gen_0_to_B(scale)); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - for(int i = 0; i < hy_sz; i++) - { - cx[i] = static_cast(prng::gen_0_to_B(scale)); - } - } - - if(inflags.GetValueInt("forw") != 1) - { - for(int i = 0; i < out_sz; i++) - { - dout[i] = static_cast(prng::gen_0_to_B(scale)); - } - - for(int i = 0; i < hy_sz; i++) - { - dhy[i] = static_cast(prng::gen_0_to_B(scale)); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - for(int i = 0; i < hy_sz; i++) - { - dcy[i] = static_cast(prng::gen_0_to_B(scale)); - } - } - } - - /* - if(!weiFileName.empty()) - { - std::cerr << "Loading weights from file is a feature which has not been implemented in - MIOpenDriver." << std::endl; - // Not implemented. - // weiRead = readBufferFromFile(wei.data(), wei_sz, weiFileName.c_str()); - } - */ - - for(int i = 0; i < wei_sz; i++) - { - wei[i] = static_cast(scale * prng::gen_A_to_B(-0.5, 0.5)); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_in.bin", in.data(), in_sz); - dumpBufferToFile("dump_wei.bin", wei.data(), wei_sz); - } - - status_t status; - status = in_dev->ToGPU(q, in.data()); - status |= wei_dev->ToGPU(q, wei.data()); - status |= out_dev->ToGPU(q, out.data()); - status |= cx_dev->ToGPU(q, cx.data()); - status |= hx_dev->ToGPU(q, hx.data()); - status |= workspace_dev->ToGPU(q, workspace.data()); - status |= reservespace_dev->ToGPU(q, reservespace.data()); - - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - - if(inflags.GetValueInt("forw") != 2) - { - status = hy_dev->ToGPU(q, hy.data()); - status |= cy_dev->ToGPU(q, cy.data()); - - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - } - - if(inflags.GetValueInt("forw") != 1) - { - status = din_dev->ToGPU(q, din.data()); - status |= dwei_dev->ToGPU(q, dwei.data()); - status |= dout_dev->ToGPU(q, dout.data()); - status |= dhx_dev->ToGPU(q, dhx.data()); - status |= dcx_dev->ToGPU(q, dcx.data()); - status |= dhy_dev->ToGPU(q, dhy.data()); - status |= dcy_dev->ToGPU(q, dcy.data()); - - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - } - - return miopenStatusSuccess; -} - -#include -#include -#include -#include -#include -#include - -template -int RNNDriver::RunForwardGPU() -{ - - if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) - return miopenStatusSuccess; - - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - std::fill(out.begin(), out.end(), static_cast(0)); - out_dev->ToGPU(GetStream(), out.data()); - - if(i > 0) - { - std::fill(reservespace.begin(), reservespace.end(), 0.); - std::fill(workspace.begin(), workspace.end(), 0.); - - workspace_dev->ToGPU(q, workspace.data()); - reservespace_dev->ToGPU(q, reservespace.data()); - } - t.Start(); - if(inflags.GetValueInt("fwdtype") == 0) - { - miopenRNNForwardTraining(GetHandle(), - rnnDesc, - adjustedSeqLen, - inputTensors.data(), - in_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - hiddenTensor, - cx_dev->GetMem(), - weightTensor, - wei_dev->GetMem(), - outputTensors.data(), - out_dev->GetMem(), - hiddenTensor, - hy_dev->GetMem(), - hiddenTensor, - cy_dev->GetMem(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - } - else if(inflags.GetValueInt("fwdtype") == 1) - { - if(inflags.GetValueInt("forw") != 1) - { - printf("Warning: Inference type is only valid for Forward RNN! \n"); - } - - miopenRNNForwardInference(GetHandle(), - rnnDesc, - adjustedSeqLen, - inputTensors.data(), - in_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - hiddenTensor, - cx_dev->GetMem(), - weightTensor, - wei_dev->GetMem(), - outputTensors.data(), - out_dev->GetMem(), - hiddenTensor, - hy_dev->GetMem(), - hiddenTensor, - cy_dev->GetMem(), - workspace_dev->GetMem(), - workspace_dev->GetSize()); - } - - t.StopAndPush(); - } - miopen::deref(GetHandle()).Finish(); - if(WALL_CLOCK) - { - printf("Forward RNN time results:\n"); - t.Print(); - } - - out_dev->FromGPU(GetStream(), out.data()); - hy_dev->FromGPU(GetStream(), hy.data()); - cy_dev->FromGPU(GetStream(), cy.data()); - reservespace_dev->FromGPU(GetStream(), reservespace.data()); - - /* - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_fwd_out_gpu.bin", out.data(), out.size()); - } - */ - - return miopenStatusSuccess; -} - -template -void ChangeDataPadding(std::vector& src_array, - std::vector& dst_array, - std::vector& batch_list, - int max_batch, - int sample_size, - bool is_src_packed) -{ - auto seq_len = batch_list.size(); - - auto packed_array = is_src_packed ? &src_array[0] : &dst_array[0]; - auto padded_array = is_src_packed ? &dst_array[0] : &src_array[0]; - - auto cur_padded_ptr = padded_array; - auto cur_packed_ptr = packed_array; - for(int seq_id = 0; seq_id < seq_len; seq_id++) - { - auto packed_size = batch_list[seq_id] * sample_size; - - if(is_src_packed) - { - std::copy(cur_packed_ptr, cur_packed_ptr + packed_size, cur_padded_ptr); - } - else - { - std::copy(cur_padded_ptr, cur_padded_ptr + packed_size, cur_packed_ptr); - } - - cur_padded_ptr += max_batch * sample_size; - cur_packed_ptr += packed_size; - } -} - -template -int RNNDriver::RunForwardCPU() -{ - if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) - return miopenStatusSuccess; - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - int in_h = in_n.back(); - int out_h = out_len[0]; - int hy_d = hid_len[0], hy_n = in_n[0], hy_h = hid_len[1]; - in_n.pop_back(); - - bool bidirection, biased; - int layer; - - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode)) - { - return miopenStatusBadParm; - } - miopenRNNPaddingMode_t paddingMode; - miopenGetRNNPaddingMode(rnnDesc, &paddingMode); - if(inflags.GetValueInt("use_padding") == 1 && - paddingMode == miopenRNNPaddingMode_t::miopenRNNIONotPadded) - { - printf("Error at check miopenGetRNNPaddingMode resutl.\n"); - return miopenStatusInternalError; - } - - std::vector* in_packed = ∈ - std::vector* out_packed = &outhost; - - std::vector converted_in; - std::vector converted_out; - - if(paddingMode == miopenRNNIOWithPadding) - { - size_t packedXInSize, packedYOutSize; - std::tie(packedXInSize, packedYOutSize) = GetTempPackedBuffersSize(in_n, in_h, out_h); - - converted_in.resize(packedXInSize); - converted_out.resize(packedYOutSize); - - ChangeDataPadding(in, converted_in, in_n, in_n[0], in_h, false); - - in_packed = &converted_in; - out_packed = &converted_out; - } - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn fwd \n"); - RunRNNForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - *out_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - mode, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenLSTM) - { - printf("verify lstm fwd \n"); - - RunLSTMForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - cy_host, - cx, - *out_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenGRU) - { - printf("verify gru fwd \n"); - - RunGRUForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - *out_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else - { - printf("illegal RNN mode"); - } - - if(paddingMode == miopenRNNIOWithPadding) - { - ChangeDataPadding(*out_packed, outhost, in_n, in_n[0], out_h, true); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_fwd_out_cpu.bin", outhost.data(), outhost.size()); - } - - // TrySaveVerificationCache("fwd_out", outhost); - return miopenStatusSuccess; -} - -template -int RNNDriver::RunBackwardGPU() -{ - int ret = 0; - - if(inflags.GetValueInt("forw") == 1) - return ret; // forward only - - if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) - { - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - workspace_dev->ToGPU(q, workspace.data()); - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - t.Start(); - ret = miopenRNNBackwardData(GetHandle(), - rnnDesc, - adjustedSeqLen, - outputTensors.data(), - out_dev->GetMem(), - outputTensors.data(), - dout_dev->GetMem(), - hiddenTensor, - dhy_dev->GetMem(), - hiddenTensor, - dcy_dev->GetMem(), - weightTensor, - wei_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - hiddenTensor, - cx_dev->GetMem(), - inputTensors.data(), - din_dev->GetMem(), - hiddenTensor, - dhx_dev->GetMem(), - hiddenTensor, - dcx_dev->GetMem(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - t.StopAndPush(); - } - miopen::deref(GetHandle()).Finish(); - if(WALL_CLOCK) - { - printf("Backward Data RNN time results:\n"); - t.Print(); - } - - din_dev->FromGPU(GetStream(), din.data()); - dhx_dev->FromGPU(GetStream(), dhx.data()); - dcx_dev->FromGPU(GetStream(), dcx.data()); - workspace_dev->FromGPU(GetStream(), workspace.data()); - } - - if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) - { - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - t.Start(); - ret = miopenRNNBackwardWeights(GetHandle(), - rnnDesc, - adjustedSeqLen, - inputTensors.data(), - in_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - outputTensors.data(), - dout_dev->GetMem(), - weightTensor, - dwei_dev->GetMem(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - t.StopAndPush(); - } - miopen::deref(GetHandle()).Finish(); - - if(WALL_CLOCK) - { - printf("Backward Weights RNN time results:\n"); - t.Print(); - } - - dwei_dev->FromGPU(GetStream(), dwei.data()); - } - - return ret; -} - -template -int RNNDriver::RunBackwardWeightsCPU() -{ - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - int in_h = in_n.back(); - int out_h = out_len[0]; - int hy_d = hid_len[0], hy_n = in_n[0], hy_h = hid_len[1]; - in_n.pop_back(); - - bool bidirection, biased; - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode)) - { - return miopenStatusBadParm; - } - - miopenRNNPaddingMode_t paddingMode = - (inflags.GetValueInt("use_padding") == 1) ? miopenRNNIOWithPadding : miopenRNNIONotPadded; - - std::vector converted_in; - std::vector converted_dout; - - if(paddingMode == miopenRNNIOWithPadding) - { - size_t packedXInSize, packedYOutSize; - std::tie(packedXInSize, packedYOutSize) = GetTempPackedBuffersSize(in_n, in_h, out_h); - - converted_in.resize(packedXInSize); - converted_dout.resize(packedYOutSize); - - ChangeDataPadding(in, converted_in, in_n, in_n[0], in_h, false); - ChangeDataPadding(dout, converted_dout, in_n, in_n[0], out_h, false); - } - - std::vector* in_packed = paddingMode == miopenRNNIOWithPadding ? &converted_in : ∈ - std::vector* dout_packed = - paddingMode == miopenRNNIOWithPadding ? &converted_dout : &dout; - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn bwdwei \n"); - - RunRNNBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - mode, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else if(mode == miopenLSTM) - { - printf("verify lstm bwdwei \n"); - - RunLSTMBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else if(mode == miopenGRU) - { - printf("verify gru bwdwei \n"); - - RunGRUBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else - { - printf("illegal RNN mode"); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_dwei_cpu.bin", dwei_host.data(), dwei_host.size()); - } - - // TrySaveVerificationCache("bwd_wei", dwei_host); - return miopenStatusSuccess; -} - -template -int RNNDriver::RunBackwardDataCPU() -{ - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - int in_h = in_n.back(); - int out_h = out_len[0]; - int hy_d = hid_len[0], hy_n = in_n[0], hy_h = hid_len[1]; - in_n.pop_back(); - - bool bidirection, biased; - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode)) - { - return miopenStatusBadParm; - } - - miopenRNNPaddingMode_t paddingMode = - (inflags.GetValueInt("use_padding") == 1) ? miopenRNNIOWithPadding : miopenRNNIONotPadded; - - std::vector converted_din; - std::vector converted_dout; - - if(paddingMode == miopenRNNIOWithPadding) - { - size_t packedXInSize, packedYOutSize; - std::tie(packedXInSize, packedYOutSize) = GetTempPackedBuffersSize(in_n, in_h, out_h); - - converted_din.resize(packedXInSize); - converted_dout.resize(packedYOutSize); - - ChangeDataPadding(dout, converted_dout, in_n, in_n[0], out_h, false); - } - - std::vector* din_packed = - paddingMode == miopenRNNIOWithPadding ? &converted_din : &din_host; - std::vector* dout_packed = - paddingMode == miopenRNNIOWithPadding ? &converted_dout : &dout; - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn bwddata \n"); - - RunRNNBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - mode, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenLSTM) - { - printf("verify lstm bwddata \n"); - - RunLSTMBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - dcy, - dcx_host, - cx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenGRU) - { - printf("verify gru bwddata \n"); - - RunGRUBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - *dout_packed, - in_n, - in_h, - adjustedSeqLen, - bidirection, - biased, - hy_d, - hy_n, - hy_h, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else - { - printf("illegal RNN mode"); - } - - if(paddingMode == miopenRNNIOWithPadding) - { - ChangeDataPadding(converted_din, din_host, in_n, in_n[0], in_h, true); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_din_cpu.bin", din_host.data(), din_host.size()); - } - - // TrySaveVerificationCache("bwd_dat", din_host); - return miopenStatusSuccess; -} - -/* -template -std::string RNNDriver::GetVerificationCacheFileName() const -{ - std::ostringstream ss; - - miopenRNNMode_t mode; - int seqLength, layer, bidir; - miopenGetRNNDescriptor( - rnnDesc, &mode, &seqLength, &layer, &bidir); - - const auto inputDesc = GetTensorLengths(const_cast(inputTensor)); - const auto weiDesc = GetTensorLengths(const_cast(weightTensor)); - const auto outDesc = GetTensorLengths(const_cast(outputTensor)); - - ss << inputDesc[1] //_n_inputs - << "x" << inputDesc[2] //_in_height - << "x" << inputDesc[3] //_in_width - << "x" << weiDesc[2] //_kernel_size1 - << "x" << weiDesc[3] //_kernel_size0 - << "x" << weiDesc[0] //_n_outputs - << "x" << outDesc[2] //_out_height - << "x" << outDesc[3] //_out_width - << "x" << inputDesc[0] //_batch_sz - << "_" << weiDesc[1] << "x" << pad_h << "x" << pad_w << "x" << u << "x" << v << "x" << sx - << "x" << sy << "x" << inflags.GetValueInt("pad_val"); - - return ss.str(); -} - -template -bool RNNDriver::TryReadVerificationCache(const std::string& file_name, - miopenTensorDescriptor_t& tensorDesc, - Tgpu* data) const -{ - const auto verification_cache_path = inflags.GetValueStr("verification_cache"); - - if(!verification_cache_path.empty()) - { - const auto file_path = - verification_cache_path + "/" + file_name + "_" + GetVerificationCacheFileName(); - if(std::ifstream(file_path).good()) - { - if(readBufferFromFile(data, GetTensorSize(tensorDesc), file_path.c_str())) - { - return true; - } - } - } - - return false; -} - -template -void RNNDriver::TrySaveVerificationCache(const std::string& file_name, - std::vector& data) const -{ - const auto verification_cache_path = inflags.GetValueStr("verification_cache"); - if(!verification_cache_path.empty()) - { - const auto file_path = - verification_cache_path + "/" + file_name + "_" + GetVerificationCacheFileName(); - dumpBufferToFile(file_path.c_str(), data.data(), data.size()); - } -} -*/ - -template -int RNNDriver::VerifyForward() -{ - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - int in_h = in_n.back(); - int out_h = out_len[0]; - int hy_d = hid_len[0], hy_n = in_n[0], hy_h = hid_len[1]; - - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode)) - { - printf("Bad Parameters! Verification FAILED\n"); - return miopenStatusBadParm; - } - - // if(!TryReadVerificationCache("fwd_out", outputTensor, outhost.data())) - { - RunForwardCPU(); - } - - auto error = miopen::rms_range(outhost, out); - - Tref tolerance = (sizeof(Tgpu) == 4 ? static_cast(1e-6) : static_cast(5e-2)); - - if(!std::isfinite(error) || error > tolerance) - { - std::cout << std::string("Forward RNN FAILED: ") << error << std::endl; - } - else - { - printf("Forward RNN Verifies on CPU and GPU\n"); - } - - auto error2 = miopen::rms_range(hy_host, hy); - - if(!std::isfinite(error2) || error2 > tolerance) - { - std::cout << std::string("final hidden state FAILED: ") << error2 << std::endl; - } - else - { - printf("final hidden Verifies on CPU and GPU\n"); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - auto error3 = miopen::rms_range(cy_host, cy); - - if(!std::isfinite(error3) || error3 > tolerance) - { - std::cout << std::string("final cell state FAILED: ") << error3 << std::endl; - } - else - { - printf("final cell Verifies on CPU and GPU\n"); - } - } - - return miopenStatusSuccess; -} - -template -int RNNDriver::VerifyBackward() -{ - std::vector in_n = GetInputTensorLengthsFromCmdLine(); - std::vector out_len = GetOutputTensorLengthsFromCmdLine(); - std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - int in_h = in_n.back(); - int out_h = out_len[0]; - int hy_d = hid_len[0], hy_n = in_n[0], hy_h = hid_len[1]; - - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode)) - { - printf("Bad Parameters! Verification FAILED\n"); - return miopenStatusBadParm; - } - - if(inflags.GetValueInt("fwdtype") == 1 && inflags.GetValueInt("forw") != 1) - { - return miopenStatusSuccess; - } - - Tref tolerance = (sizeof(Tgpu) == 4 ? static_cast(1e-6) : static_cast(5e-2)); - - // if(!TryReadVerificationCache("bwd_dat", inputTensor, din_host.data())) - if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) - { - { - RunBackwardDataCPU(); - } - - auto error_data = miopen::rms_range(din_host, din); - - if(!std::isfinite(error_data) || error_data > tolerance) - { - std::cout << std::string("Backward RNN Data FAILED: ") << error_data << std::endl; - } - else - { - printf("Backward RNN Data Verifies on CPU and GPU\n"); - } - - auto error_data2 = miopen::rms_range(dhx_host, dhx); - - if(!std::isfinite(error_data2) || error_data2 > tolerance) - { - std::cout << std::string("difference at inital hidden state FAILED: ") << error_data2 - << std::endl; - } - else - { - printf("initial hidden state Verifies on CPU and GPU\n"); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - auto error_data3 = miopen::rms_range(dcx_host, dcx); - - if(!std::isfinite(error_data3) || error_data3 > tolerance) - { - std::cout << std::string("difference at inital cell state FAILED: ") << error_data3 - << std::endl; - } - else - { - printf("inital cell state Verifies on CPU and GPU\n"); - } - } - } - - // if(!TryReadVerificationCache("bwd_wei", weightTensor, dwei_host.data())) - if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) - { - { - RunBackwardWeightsCPU(); - } - - auto error_weights = miopen::rms_range(dwei_host, dwei); - if(!std::isfinite(error_weights) || error_weights > tolerance) - { - std::cout << std::string("Backward RNN Weights FAILED: ") << error_weights << std::endl; - } - else - { - printf("Backward RNN Weights Verifies on CPU and GPU\n"); - } - } - - return miopenStatusSuccess; -} - -#endif // GUARD_MIOPEN_RNN_DRIVER_HPP diff --git a/driver/rnn_seq_driver.hpp b/driver/rnn_seq_driver.hpp index 2a05741b2f..e69de29bb2 100644 --- a/driver/rnn_seq_driver.hpp +++ b/driver/rnn_seq_driver.hpp @@ -1,1832 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include "InputFlags.hpp" -#include "driver.hpp" -#include "gru_verify_gemm.hpp" -#include "lstm_verify_gemm.hpp" -#include "random.hpp" -#include "rnn_verify_gemm.hpp" -#include "tensor_driver.hpp" -#include "timer.hpp" -#include "util_driver.hpp" -#include "util_file.hpp" - -#include <../test/verify.hpp> - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -std::vector get_default_time_strides(int vec_size, const std::vector& seq_array) -{ - std::vector sum_v(seq_array.size()); - sum_v[0] = 0; - std::partial_sum(seq_array.begin(), std::prev(seq_array.end()), std::next(sum_v.begin())); - for(auto& it : sum_v) - it *= vec_size; - return sum_v; -}; - -template -void TransformIODefaultLayaoutToTarget(std::vector& def_array, - std::vector& target_array, - const std::vector& def_batch_sz_per_time, - const std::vector& target_batch_order, - size_t seq_len_padded, - size_t io_vec_size, - miopenRNNBaseLayout_t target_layout, - bool is_dst_target) -{ - assert(target_layout == miopenRNNDataBatchMajorPadded || - target_layout == miopenRNNDataSeqMajorPadded); - - assert(def_batch_sz_per_time[0] == target_batch_order.size()); - - size_t non_zero_seq_len = def_batch_sz_per_time.size(); - size_t batch_size = def_batch_sz_per_time[0]; - - const std::vector default_time_strides = - get_default_time_strides(io_vec_size, def_batch_sz_per_time); - const size_t default_batch_stride = io_vec_size; - - size_t target_batch_stride = - io_vec_size * (target_layout == miopenRNNDataBatchMajorPadded ? seq_len_padded : 1); - size_t target_time_stride = - io_vec_size * (target_layout == miopenRNNDataBatchMajorPadded ? 1 : batch_size); - - for(size_t time_it = 0; time_it < non_zero_seq_len; time_it++) - { - for(size_t batch_it = 0; batch_it < def_batch_sz_per_time[time_it]; batch_it++) - { - const size_t def_time_offset = default_time_strides[time_it]; - const size_t target_time_off = time_it * target_time_stride; - - const size_t def_offset = def_time_offset + default_batch_stride * batch_it; - const size_t target_offset = - target_time_off + target_batch_stride * target_batch_order[batch_it]; - - if(is_dst_target) - { - std::copy(&def_array[def_offset], - &def_array[def_offset + io_vec_size], - &target_array[target_offset]); - } - else - { - std::copy(&target_array[target_offset], - &target_array[target_offset + io_vec_size], - &def_array[def_offset]); - } - } - } -} - -template -void HiddenTensorReorder(std::vector& src_array, - std::vector& dst_array, - const std::vector& batch_order, - const std::vector hid_len, - bool is_dst_direct_order) -{ - const size_t copy_size = hid_len[2]; - - const size_t batch_stride = hid_len[2]; - const size_t layer_stride = batch_stride * hid_len[1]; - - for(size_t batch_id = 0; batch_id < hid_len[1]; batch_id++) - { - const auto src_batch_off = - batch_stride * (is_dst_direct_order ? batch_order[batch_id] : batch_id); - const auto dst_batch_off = - batch_stride * (is_dst_direct_order ? batch_id : batch_order[batch_id]); - - for(size_t layer_id = 0; layer_id < hid_len[0]; layer_id++) - { - const auto dst_offset = dst_batch_off + layer_id * layer_stride; - const auto src_offset = src_batch_off + layer_id * layer_stride; - - std::copy(src_array.begin() + src_offset, - src_array.begin() + src_offset + copy_size, - dst_array.begin() + dst_offset); - } - } -} - -template -class RNNSeqDriver : public Driver -{ -public: - RNNSeqDriver() : Driver() - { - miopenCreateSeqTensorDescriptor(&inputSeqTensor); - miopenCreateTensorDescriptor(&hiddenTensor); - miopenCreateTensorDescriptor(&inputTensor_dims); - miopenCreateSeqTensorDescriptor(&outputSeqTensor); - - miopenCreateRNNDescriptor(&rnnDesc); - miopenCreateDropoutDescriptor(&DropoutDesc); - workspace_dev = nullptr; - reservespace_dev = nullptr; - - InitDataType(); - } - - int AddCmdLineArgs() override; - int ParseCmdLineArgs(int argc, char* argv[]) override; - - InputFlags& GetInputFlags() override { return inflags; } - - int CheckDescriptor(miopenSeqTensorDescriptor_t desc, - miopenRNNBaseLayout_t layout, - const std::vector& src_lens, - const std::vector& src_Slens); - int CheckDescriptor(miopenTensorDescriptor_t src_desc, const std::vector& src_lens); - - int GetandSetData() override; - - std::vector GetSeqLengthsFromCmdLine(); - miopenRNNBaseLayout_t GetIODataLayoutFromCmdLine(); - miopenRNNFWDMode_t GetRNNFwdModeFromCmdLine(); - - std::vector GetInputTensorLengthsFromCmdLine(); - std::vector GetHiddenTensorLengthsFromCmdLine(); - std::vector GetOutputTensorLengthsFromCmdLine(); - - int SetRNNDescriptorFromCmdLineArgs(); - int AllocateBuffersAndCopy() override; - - int RunForwardGPU() override; - int RunForwardCPU(); - int RunBackwardGPU() override; - int RunBackwardDataCPU(); - int RunBackwardWeightsCPU(); - int VerifyBackward() override; - int VerifyForward() override; - ~RNNSeqDriver() override - { - miopenDestroySeqTensorDescriptor(outputSeqTensor); - miopenDestroyTensorDescriptor(inputTensor_dims); - miopenDestroyTensorDescriptor(hiddenTensor); - miopenDestroySeqTensorDescriptor(inputSeqTensor); - - miopenDestroyRNNDescriptor(rnnDesc); - } - -private: - InputFlags inflags; - - miopenSeqTensorDescriptor_t inputSeqTensor; - miopenSeqTensorDescriptor_t outputSeqTensor; - miopenTensorDescriptor_t hiddenTensor; - miopenTensorDescriptor_t inputTensor_dims; - - std::unique_ptr in_dev; - std::unique_ptr din_dev; - std::unique_ptr wei_dev; - std::unique_ptr dwei_dev; - std::unique_ptr out_dev; - std::unique_ptr dout_dev; - std::unique_ptr hx_dev; - std::unique_ptr cx_dev; - std::unique_ptr hy_dev; - std::unique_ptr cy_dev; - std::unique_ptr dhx_dev; - std::unique_ptr dcx_dev; - std::unique_ptr dhy_dev; - std::unique_ptr dcy_dev; - std::unique_ptr workspace_dev; - std::unique_ptr reservespace_dev; - std::unique_ptr dropout_states_dev; - - // for debug - std::vector tmp_gpu_in; - std::vector tmp_gpu_hx; - std::vector tmp_gpu_cx; - std::vector from_gpu_out; - - std::vector in; - std::vector din; - std::vector wei; - std::vector dwei; - std::vector out; - std::vector dout; - std::vector hx; - std::vector cx; - std::vector hy; - std::vector cy; - std::vector dhx; - std::vector dcx; - std::vector dhy; - std::vector dcy; - std::vector workspace; - std::vector reservespace; - - std::vector outhost; - std::vector workspace_host; - std::vector reservespace_host; - std::vector din_host; - std::vector dwei_host; - std::vector hy_host; - std::vector cy_host; - std::vector dhx_host; - std::vector dcx_host; - std::vector dropout_states_host; - - miopenRNNDescriptor_t rnnDesc; - - /////////////// - - std::vector sorted_seq_lens; - std::vector unsorted_seq_lens; - - miopenRNNFWDMode_t fwd_type; - miopenRNNBaseLayout_t io_layout; - /////////////// - - miopenDropoutDescriptor_t DropoutDesc; - float dropout_rate; - unsigned long long dropout_seed; -}; - -template -int RNNSeqDriver::ParseCmdLineArgs(int argc, char* argv[]) -{ - inflags.Parse(argc, argv); - - int nn_dir = inflags.GetValueInt("forw"); - if(inflags.GetValueInt("fwdtype") == 1 && !(nn_dir == 0 || nn_dir == 1)) - { - MIOPEN_THROW( - "Incorrect input, In Inference only fwd direction is allowed ((forw=0) OR (forw=1))."); - } - - if(inflags.GetValueInt("iter") > 1 && inflags.GetValueInt("verify") != 0) - MIOPEN_THROW( - "To use non default Number of Iterations >1 need to disable Verification -V 0."); - - if((nn_dir & 4) && !(nn_dir & 2) && !(nn_dir & 1)) - MIOPEN_THROW("Incorrect input, calculation of BackwardWeights require BackwardData and " - "ForwardData."); - if((nn_dir & 2) && !(nn_dir & 1)) - MIOPEN_THROW("Incorrect input, calculation of BackwardData require ForwardData."); - - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::CheckDescriptor(miopenSeqTensorDescriptor_t src_desc, - miopenRNNBaseLayout_t src_layout, - const std::vector& src_lens, - const std::vector& src_Slens) -{ - miopenDataType_t dt; - miopenRNNBaseLayout_t layout; - std::vector lens(src_lens.size()); - std::vector slens(src_Slens.size()); - - miopenGetRNNDataSeqTensorDescriptor( - src_desc, &dt, &layout, &lens[1], &lens[0], &lens[2], slens.size(), slens.data(), nullptr); - - if(!std::equal(std::begin(src_lens), std::end(src_lens), std::begin(lens)) || - !std::equal(std::begin(src_Slens), std::end(src_Slens), std::begin(slens)) || - layout != src_layout) - { - return miopenStatusInternalError; - } - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::CheckDescriptor(miopenTensorDescriptor_t src_desc, - const std::vector& src_lens) -{ - const std::vector lens = GetTensorLengths(src_desc); - - if(lens.size() != src_lens.size() || - !std::equal(src_lens.begin(), src_lens.end(), lens.begin())) - { - return miopenStatusInternalError; - } - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::GetandSetData() -{ - int status = 0; - io_layout = GetIODataLayoutFromCmdLine(); - - unsorted_seq_lens = GetSeqLengthsFromCmdLine(); - sorted_seq_lens = unsorted_seq_lens; - std::sort(sorted_seq_lens.begin(), sorted_seq_lens.end(), std::greater()); - - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - const std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - - { - const std::vector in_dims = {in_lens[0], in_lens[2]}; - miopenSetTensorDescriptor(inputTensor_dims, data_type, 2, in_dims.data(), nullptr); - status = CheckDescriptor(inputTensor_dims, in_dims); - } - - if(status != miopenStatusSuccess) - { - printf("Error checking TensorDescriptor:%s content.\n", "inputTensor_dims"); - return miopenStatusInternalError; - } - - Tgpu alfa = static_cast(-1); - - miopenSetRNNDataSeqTensorDescriptor(inputSeqTensor, - data_type, - io_layout, - in_lens[1], - in_lens[0], - in_lens[2], - unsorted_seq_lens.data(), - &alfa); - - status = CheckDescriptor(inputSeqTensor, io_layout, in_lens, unsorted_seq_lens); - if(status != miopenStatusSuccess) - { - printf("Error checking SeqTensorDescriptor:%s content.\n", "inputSeqTensor"); - return status; - } - - miopenSetRNNDataSeqTensorDescriptor(outputSeqTensor, - data_type, - io_layout, - out_lens[1], - out_lens[0], - out_lens[2], - unsorted_seq_lens.data(), - &alfa); - - status = CheckDescriptor(outputSeqTensor, io_layout, out_lens, unsorted_seq_lens); - if(status != miopenStatusSuccess) - { - printf("Error checking SeqTensorDescriptor:%s content.\n", "outputSeqTensor"); - return status; - } - - miopenSetTensorDescriptor(hiddenTensor, data_type, 3, hid_len.data(), nullptr); - status = CheckDescriptor(hiddenTensor, hid_len); - if(status != miopenStatusSuccess) - { - printf("Error checking TensorDescriptor:%s content.\n", "hiddenTensor"); - return miopenStatusInternalError; - } - - SetRNNDescriptorFromCmdLineArgs(); - fwd_type = GetRNNFwdModeFromCmdLine(); - - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::AddCmdLineArgs() -{ - inflags.AddInputFlag("forw", - 'F', - "0", - "Run only Forward RNN == 1 or only Backward Data RNN == 2, Backward " - "Weights = 4 or both == 0 (Default=0)", - "int"); - inflags.AddInputFlag("fwdtype", - 'c', - "0", - "RNN forward being training or inference, Default training (Default=0)", - "int"); - inflags.AddInputFlag( - "mode", 'm', "tanh", "RNN Mode (relu, tanh, lstm, gru) (Default=tanh)", "str"); - - inflags.AddInputFlag("datatype", 'f', "1", "16-bit or 32-bit fp (Default=1)", "int"); - inflags.AddInputFlag("io_layout", - 'I', - "1", - "IO data layout" - "SeqMajorNotPadded = 1" - "SeqMajorPadded = 2" - "BatchMajorPadded = 3 (Default=1)", - "int"); - - inflags.AddInputFlag("num_layer", 'l', "1", "Number of hidden stacks (Default=1)", "int"); - inflags.AddInputFlag("batch_size", 'n', "4", "Mini-batch size (Default=4)", "int"); - inflags.AddInputFlag( - "seq_len", 'k', "10", "Max number of iterations to unroll over (Default=10)", "int"); - inflags.AddInputFlag("seq_len_array", - 'K', - "", - "Array of SeqLength for each sample in batch (Default=4)", - "vector"); - - inflags.AddInputFlag("hid_h", 'H', "32", "Hidden State Length (Default=32)", "int"); - inflags.AddInputFlag("in_vec", 'W', "32", "Input Length (Default=32)", "int"); - - inflags.AddInputFlag( - "bidirection", 'r', "0", "uni- or bi-direction, default uni- (Default=0)", "int"); - inflags.AddInputFlag("bias", 'b', "", "Use Bias (Default=0)", "int"); - inflags.AddInputFlag("inputmode", 'p', "0", "linear == 0 or skip == 1, (Default=0)", "int"); - - inflags.AddInputFlag("rnnalgo", 'a', "0", "default, fundamental (Default=0)", "int"); - - inflags.AddInputFlag( - "use_dropout", 'U', "0", "Use dropout: 1; Not use dropout: 0 (Default=0)", "int"); - inflags.AddInputFlag("dropout", 'P', "0.0", "Dropout rate (Default=0.0)", "float"); - inflags.AddInputFlag( - "seed_low", 'L', "0", "Least significant 32 bits of seed (Default=0)", "int"); - inflags.AddInputFlag( - "seed_high", 'M', "0", "Most significant 32 bits of seed (Default=0)", "int"); - - inflags.AddInputFlag("iter", 'i', "1", "Number of Iterations (Default=1)", "int"); - inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); - - inflags.AddInputFlag( - "wall", - 'w', - "0", - "Wall-clock, for host and gpu, Time Each Layer, Disabled = 0,\ - OldWallClock = 1,\ - SeparateClocksSynced = 2,\ - SeparateClocksNotSynced = 3 \ - (Default = 0) ", - "int"); - inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); - /* // DL: These have not been implemented. Removing them for now. - inflags.AddInputFlag("in_data", 'd', "", "Input data filename (Default=)", "string"); - inflags.AddInputFlag("weights", 'e', "", "Input weights filename (Default=)", "string");*/ - - return 0; -} - -template -miopenRNNBaseLayout_t RNNSeqDriver::GetIODataLayoutFromCmdLine() -{ - int layout = inflags.GetValueInt("io_layout"); - switch(layout) - { - case 1: return miopenRNNDataSeqMajorNotPadded; - case 2: return miopenRNNDataSeqMajorPadded; - case 3: return miopenRNNDataBatchMajorPadded; - default: MIOPEN_THROW("Incorrect input, unsupported RNNLayout."); - } -} - -template -miopenRNNFWDMode_t RNNSeqDriver::GetRNNFwdModeFromCmdLine() -{ - int fwdtype = inflags.GetValueInt("fwdtype"); - switch(fwdtype) - { - case 0: return miopenRNNFWDMode_t::miopenRNNTraining; - case 1: return miopenRNNFWDMode_t::miopenRNNInference; - default: MIOPEN_THROW("Incorrect input, unsupported fwdtype."); - } -} - -template -std::vector RNNSeqDriver::GetSeqLengthsFromCmdLine() -{ - int batch_size = inflags.GetValueInt("batch_size"); - int seq_len_max = inflags.GetValueInt("seq_len"); - - std::vector data_seq_lens(batch_size, 0); - - std::string s_lens = inflags.GetValueStr("seq_len_array"); - std::stringstream ss(s_lens); - - auto seq_it = data_seq_lens.begin(); - int element; - - while(ss >> element) - { - if(seq_it == data_seq_lens.end()) - MIOPEN_THROW("Incorrect input, seq_len_array bigger than provided batch_size.\n"); - - if(element > seq_len_max) - MIOPEN_THROW("Length of data sequence is longer than required unrolled time sequence " - "provided.\n" - "The data sequence will be truncated to match unrolled time sequence.\n"); - - *(seq_it++) = element; - - if(ss.peek() == ',' || ss.peek() == ' ') - { - ss.ignore(); - } - } - - if(io_layout == miopenRNNDataSeqMajorNotPadded && (seq_it != data_seq_lens.begin()) && - (!std::is_sorted(data_seq_lens.begin(), seq_it, std::greater<>{}))) - { - MIOPEN_THROW("Incorrect input, seq_lens should not to increase with " - "miopenRNNDataSeqMajorNotPadded layout\n"); - } - - if(seq_it != data_seq_lens.end()) - { - auto padding_val = (seq_it != data_seq_lens.begin()) ? *(seq_it - 1) : seq_len_max; - std::cout << "sampl_lens size == " << std::distance(data_seq_lens.begin(), seq_it) - << " is smaller than time batch_size == " << batch_size - << ", padding the rest of data with " << padding_val << "\n"; - - for(; seq_it != data_seq_lens.end(); seq_it++) - *seq_it = padding_val; - } - - return data_seq_lens; -} - -template -std::vector RNNSeqDriver::GetInputTensorLengthsFromCmdLine() -{ - int batch_size = inflags.GetValueInt("batch_size"); - int seq_len = inflags.GetValueInt("seq_len"); - int in_vec = inflags.GetValueInt("in_vec"); - - return std::vector({batch_size, seq_len, in_vec}); -} - -template -std::vector RNNSeqDriver::GetOutputTensorLengthsFromCmdLine() -{ - int batch_size = inflags.GetValueInt("batch_size"); - int seq_len = inflags.GetValueInt("seq_len"); - - int hid_h = inflags.GetValueInt("hid_h"); - int bi = (inflags.GetValueInt("bidirection") == 1) ? 2 : 1; - int out_vec = hid_h * bi; - - return std::vector({batch_size, seq_len, out_vec}); -} - -template -std::vector RNNSeqDriver::GetHiddenTensorLengthsFromCmdLine() -{ - int n_layer = inflags.GetValueInt("num_layer"); - if((inflags.GetValueInt("bidirection")) == 1) - n_layer *= 2; - - int batch_size = inflags.GetValueInt("batch_size"); - int hid_vec = inflags.GetValueInt("hid_h"); - - return std::vector({n_layer, batch_size, hid_vec}); -} - -template -int RNNSeqDriver::SetRNNDescriptorFromCmdLineArgs() -{ - - const int layer = inflags.GetValueInt("num_layer"); - const int hidden_size = inflags.GetValueInt("hid_h"); - - miopenRNNMode_t mode; - - if((inflags.GetValueStr("mode")) == "relu") - { - mode = miopenRNNRELU; - } - else if((inflags.GetValueStr("mode")) == "tanh") - { - mode = miopenRNNTANH; - } - else if((inflags.GetValueStr("mode")) == "lstm") - { - mode = miopenLSTM; - } - else if((inflags.GetValueStr("mode")) == "gru") - { - mode = miopenGRU; - } - else - { - printf("Incorrect RNN Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNBiasMode_t biasMode; - if((inflags.GetValueInt("bias")) == 0) - { - biasMode = miopenRNNNoBias; - } - else if((inflags.GetValueInt("bias")) == 1) - { - biasMode = miopenRNNwithBias; - } - else - { - printf("Incorrect bias Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNDirectionMode_t directionMode; - if((inflags.GetValueInt("bidirection")) == 0) - { - directionMode = miopenRNNunidirection; - } - else if((inflags.GetValueInt("bidirection")) == 1) - { - directionMode = miopenRNNbidirection; - } - else - { - printf("Incorrect direction Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNInputMode_t inMode; - if((inflags.GetValueInt("inputmode")) == 0) - { - inMode = miopenRNNlinear; - } - else if((inflags.GetValueInt("inputmode")) == 1) - { - inMode = miopenRNNskip; - } - else - { - printf("Incorrect input Mode\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - miopenRNNAlgo_t algo; - if((inflags.GetValueInt("rnnalgo")) == 0) - { - algo = miopenRNNdefault; - } - else if((inflags.GetValueInt("rnnalgo")) == 1) - { - algo = miopenRNNfundamental; - } - else - { - printf("Incorrect RNN algorithm\n"); - exit(0); // NOLINT (concurrency-mt-unsafe) - } - - if(inflags.GetValueInt("use_dropout")) - { - dropout_rate = static_cast(inflags.GetValueDouble("dropout")); - auto dropout_seed_low = - static_cast(std::max(inflags.GetValueInt("seed_low"), 0)); - auto dropout_seed_high = - static_cast(std::max(inflags.GetValueInt("seed_high"), 0)); - dropout_seed = dropout_seed_high << 32 | dropout_seed_low; - - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - dropout_states_dev = - std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); - - miopenSetDropoutDescriptor(DropoutDesc, - GetHandle(), - dropout_rate, - dropout_states_dev->GetMem(), - dropout_states_dev->GetSize(), - dropout_seed, - false, - false, - MIOPEN_RNG_PSEUDO_XORWOW); - - miopenSetRNNDescriptor_V2(rnnDesc, - hidden_size, - layer, - DropoutDesc, - inMode, - directionMode, - mode, - biasMode, - algo, - data_type); - } - else - { - miopenSetRNNDescriptor( - rnnDesc, hidden_size, layer, inMode, directionMode, mode, biasMode, algo, data_type); - } - - if(io_layout != miopenRNNDataSeqMajorNotPadded) - { - miopenSetRNNPaddingMode(rnnDesc, miopenRNNPaddingMode_t::miopenRNNIOWithPadding); - } - - return miopenStatusSuccess; -} - -// GetTensorSize broken So this is WA -inline size_t Get3DNoVECTensorSize(miopenTensorDescriptor_t& tensor) -{ - assert(miopen::deref(tensor).IsPacked() && - "GetTensorSize should not be used on an unpacked tensor."); - const auto len = GetTensorLengths(tensor); - size_t sz = std::accumulate(len.begin(), len.end(), 1ULL, std::multiplies()); - return sz; -} - -std::vector GetSamplesIndexDescendingOrder(const std::vector& unsorted_seq_lens, - bool reverse) -{ - const auto sample_count = unsorted_seq_lens.size(); - - std::vector index_v(sample_count); - std::iota(index_v.begin(), index_v.end(), 0); - - auto seq_len_cmp = [&unsorted_seq_lens](unsigned a_id, unsigned b_id) { - return unsorted_seq_lens[a_id] > unsorted_seq_lens[b_id]; - }; - - std::stable_sort(index_v.begin(), index_v.end(), seq_len_cmp); - - auto get_reverse_index = [](const std::vector& base_index) { - std::vector reverse_index(base_index.size()); - unsigned next_rev_index = 0; - for(auto id : base_index) - reverse_index[id] = next_rev_index++; - return reverse_index; - }; - - return !reverse ? index_v : get_reverse_index(index_v); -} - -std::vector GetBatchesPerTime(const std::vector& sorted_sequence_len) -{ - std::vector batches; - auto block_begin = sorted_sequence_len.rbegin(); - auto sample_ptr = sorted_sequence_len.rbegin(); - auto batch_size = sorted_sequence_len.size(); - - batches.insert(batches.end(), *block_begin, batch_size); - - while(sample_ptr != sorted_sequence_len.rend()) - { - if(*sample_ptr != *block_begin) - { - batch_size = batch_size - (sample_ptr - block_begin); - const auto seq_count = *sample_ptr - *block_begin; - batches.insert(batches.end(), seq_count, batch_size); - block_begin = sample_ptr; - } - sample_ptr++; - } - return batches; -} - -template -int RNNSeqDriver::AllocateBuffersAndCopy() -{ - int status = 0; - - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - - const size_t vectors_cnt_host = - std::accumulate(sorted_seq_lens.begin(), sorted_seq_lens.end(), 0ULL); - const size_t vectors_cnt_gpu = - io_layout == miopenRNNDataSeqMajorNotPadded ? vectors_cnt_host : in_lens[0] * in_lens[1]; - - const size_t in_host_sz = vectors_cnt_host * in_lens[2]; - const size_t out_host_sz = vectors_cnt_host * out_lens[2]; - - const size_t in_gpu_sz = vectors_cnt_gpu * in_lens[2]; - const size_t out_gpu_sz = vectors_cnt_gpu * out_lens[2]; - - const size_t hid_sz = Get3DNoVECTensorSize(hiddenTensor); - - size_t workSpace_sz; - size_t reserveSpace_sz; - status |= miopenGetRNNTempSpaceSizes( - GetHandle(), rnnDesc, inputSeqTensor, fwd_type, &workSpace_sz, &reserveSpace_sz); - - workSpace_sz /= sizeof(Tgpu); - if(inflags.GetValueInt("use_dropout")) - reserveSpace_sz = (reserveSpace_sz + sizeof(Tgpu) - 1) / sizeof(Tgpu); - else - reserveSpace_sz /= sizeof(Tgpu); - - size_t wei_sz = 0; - status |= miopenGetRNNParamsSize(GetHandle(), rnnDesc, inputTensor_dims, &wei_sz, data_type); - wei_sz /= sizeof(Tgpu); - - if(status != HIP_SUCCESS) - { - printf("Error at getting required space for RNN rnnDesc\n"); - return miopenStatusNotInitialized; - } - - const uint32_t ctx = 0; // opencl legacy - - in_dev = std::unique_ptr(new GPUMem(ctx, in_gpu_sz, sizeof(Tgpu))); - hx_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_gpu_sz, sizeof(Tgpu))); - wei_dev = std::unique_ptr(new GPUMem(ctx, wei_sz, sizeof(Tgpu))); - cx_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - workspace_dev = std::unique_ptr(new GPUMem(ctx, workSpace_sz, sizeof(Tgpu))); - reservespace_dev = std::unique_ptr(new GPUMem(ctx, reserveSpace_sz, sizeof(Tgpu))); - - if(inflags.GetValueInt("forw") != 2) - { - hy_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - cy_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - } - - if(inflags.GetValueInt("forw") != 1) - { - din_dev = std::unique_ptr(new GPUMem(ctx, in_gpu_sz, sizeof(Tgpu))); - dwei_dev = std::unique_ptr(new GPUMem(ctx, wei_sz, sizeof(Tgpu))); - dout_dev = std::unique_ptr(new GPUMem(ctx, out_gpu_sz, sizeof(Tgpu))); - dhx_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - dcx_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - dhy_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - dcy_dev = std::unique_ptr(new GPUMem(ctx, hid_sz, sizeof(Tgpu))); - } - - in = std::vector(in_host_sz); - hx = std::vector(hid_sz); - wei = std::vector(wei_sz); - out = std::vector(out_host_sz, static_cast(0)); - cx = std::vector(hid_sz); - - if(inflags.GetValueInt("forw") != 2) - { - hy = std::vector(hid_sz, static_cast(0)); - cy = std::vector(hid_sz, static_cast(0)); - hy_host = std::vector(hid_sz, static_cast(0)); - cy_host = std::vector(hid_sz, static_cast(0)); - } - - workspace = std::vector(workSpace_sz, static_cast(0)); - reservespace = std::vector(reserveSpace_sz, static_cast(0)); - outhost = std::vector(out_host_sz, static_cast(0)); - workspace_host = std::vector(workSpace_sz, static_cast(0)); - - /// dropout legacy format - const std::size_t inputBatchLenSum = vectors_cnt_host; - - int hid_h = inflags.GetValueInt("hid_h"); - int layer = inflags.GetValueInt("num_layer"); - int bidir = inflags.GetValueInt("bidirection"); - - size_t reserveSpaceHost_sz = - 2 * - (inflags.GetValueStr("mode") == "lstm" ? 6 - : (inflags.GetValueStr("mode") == "gru" ? 4 : 1)) * - layer * inputBatchLenSum * hid_h * (bidir + 1); - if(inflags.GetValueInt("use_dropout")) - { - reserveSpaceHost_sz += (layer - 1) * inputBatchLenSum * hid_h * (bidir + 1); - reserveSpaceHost_sz *= sizeof(Tref); - reserveSpaceHost_sz += (layer - 1) * inputBatchLenSum * hid_h * (bidir + 1); - reserveSpaceHost_sz = (reserveSpaceHost_sz + sizeof(Tref) - 1) / sizeof(Tref); - } - reservespace_host = std::vector(reserveSpaceHost_sz, static_cast(0)); - // end dropout - - if(inflags.GetValueInt("forw") != 1) - { - din = std::vector(in_host_sz, static_cast(0)); - dwei = std::vector(wei_sz, static_cast(0)); - dout = std::vector(out_host_sz, static_cast(0)); - dhx = std::vector(hid_sz, static_cast(0)); - dcx = std::vector(hid_sz, static_cast(0)); - dhy = std::vector(hid_sz, static_cast(0)); - dcy = std::vector(hid_sz, static_cast(0)); - din_host = std::vector(in_host_sz, static_cast(0)); - dwei_host = std::vector(wei_sz, static_cast(0)); - dhx_host = std::vector(hid_sz, static_cast(0)); - dcx_host = std::vector(hid_sz, static_cast(0)); - } - - // Unless seed is persistent between runs validation using cache stored in file is impossible. - prng::reset_seed(); - - auto fill_array_via_gen = [](auto& dst, size_t dst_sz, double range_l, double range_r) { - for(size_t it = 0; it < dst_sz; it++) - dst[it] = prng::gen_A_to_B(static_cast(range_l), static_cast(range_r)); - }; - - const double scale = 0.01; - fill_array_via_gen(in, in_host_sz, 0.0, 1.0 * scale); - fill_array_via_gen(hx, hid_sz, 0.0, 1.0 * scale); - fill_array_via_gen(wei, wei_sz, -0.5 * scale, 0.5 * scale); - - if((inflags.GetValueStr("mode")) == "lstm") - fill_array_via_gen(cx, hid_sz, 0.0, 1.0 * scale); - - if(inflags.GetValueInt("forw") != 1) - { - fill_array_via_gen(dout, out_host_sz, 0.0, 1.0 * scale); - - fill_array_via_gen(dhy, hid_sz, 0.0, 1.0 * scale); - - if((inflags.GetValueStr("mode")) == "lstm") - fill_array_via_gen(dcy, hid_sz, 0.0, 1.0 * scale); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_in.bin", in.data(), in_host_sz); - dumpBufferToFile("dump_wei.bin", wei.data(), wei_sz); - } - - const std::vector hid_len = GetHiddenTensorLengthsFromCmdLine(); - if(io_layout != miopenRNNDataSeqMajorNotPadded) - { - const std::vector batches_per_time = GetBatchesPerTime(sorted_seq_lens); - const std::vector order_idxs = - GetSamplesIndexDescendingOrder(unsorted_seq_lens, false); - - tmp_gpu_in = std::vector(in_gpu_sz, static_cast(0)); - tmp_gpu_hx = std::vector(hid_sz, static_cast(0)); - - TransformIODefaultLayaoutToTarget( - in, tmp_gpu_in, batches_per_time, order_idxs, in_lens[1], in_lens[2], io_layout, true); - - HiddenTensorReorder(hx, tmp_gpu_hx, order_idxs, hid_len, false); - - status |= in_dev->ToGPU(q, tmp_gpu_in.data()); - status |= hx_dev->ToGPU(q, tmp_gpu_hx.data()); - - if((inflags.GetValueStr("mode")) == "lstm") - { - tmp_gpu_cx = std::vector(hid_sz, static_cast(0)); - HiddenTensorReorder(cx, tmp_gpu_cx, order_idxs, hid_len, false); - status |= cx_dev->ToGPU(q, tmp_gpu_cx.data()); - } - - if(inflags.GetValueInt("forw") != 1) - { - { - std::vector tmp_gpu_dout = - std::vector(out_gpu_sz, static_cast(0)); - TransformIODefaultLayaoutToTarget(dout, - tmp_gpu_dout, - batches_per_time, - order_idxs, - out_lens[1], - out_lens[2], - io_layout, - true); - status |= dout_dev->ToGPU(q, tmp_gpu_dout.data()); - } - { - std::vector tmp_gpu_dhy = std::vector(hid_sz, static_cast(0)); - HiddenTensorReorder(dhy, tmp_gpu_dhy, order_idxs, hid_len, false); - status |= dhy_dev->ToGPU(q, tmp_gpu_dhy.data()); - } - if((inflags.GetValueStr("mode")) == "lstm") - { - std::vector tmp_gpu_dcy = std::vector(hid_sz, static_cast(0)); - HiddenTensorReorder(dcy, tmp_gpu_dcy, order_idxs, hid_len, false); - status |= dcy_dev->ToGPU(q, tmp_gpu_dcy.data()); - } - } - if(status != HIP_SUCCESS) - { - printf("Error copying data to GPU\n"); - return miopenStatusNotInitialized; - } - } - else - { - status |= in_dev->ToGPU(q, in.data()); - status |= hx_dev->ToGPU(q, hx.data()); - status |= cx_dev->ToGPU(q, cx.data()); - if(inflags.GetValueInt("forw") != 1) - { - status |= dout_dev->ToGPU(q, dout.data()); - status |= dhy_dev->ToGPU(q, dhy.data()); - if((inflags.GetValueStr("mode")) == "lstm") - status |= dcy_dev->ToGPU(q, dcy.data()); - } - - if(status != HIP_SUCCESS) - { - printf("Error copying data to GPU\n"); - return miopenStatusNotInitialized; - } - } - - status |= wei_dev->ToGPU(q, wei.data()); - // status |= workspace_dev->ToGPU(q, workspace.data()); - // status |= reservespace_dev->ToGPU(q, reservespace.data()); - - if(status != HIP_SUCCESS) - { - printf("Error copying data to GPU\n"); - return miopenStatusNotInitialized; - } - - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::RunForwardGPU() -{ - - if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) - return miopenStatusSuccess; - - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - - from_gpu_out = std::vector(out_dev->GetSize() / sizeof(Tgpu), static_cast(0)); - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - out_dev->ToGPU(q, from_gpu_out.data()); - workspace_dev->ToGPU(q, workspace.data()); - reservespace_dev->ToGPU(q, reservespace.data()); - - t.Start(); - miopenRNNForward(GetHandle(), - rnnDesc, - fwd_type, - inputSeqTensor, - in_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - hy_dev->GetMem(), - hiddenTensor, // cdesc - cx_dev->GetMem(), - cy_dev->GetMem(), - outputSeqTensor, - out_dev->GetMem(), - wei_dev->GetMem(), - wei_dev->GetSize(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - - t.StopAndPush(); - } - - miopen::deref(GetHandle()).Finish(); - if(WALL_CLOCK) - { - printf("Forward RNN time results:\n"); - t.Print(); - } - - if(io_layout != miopenRNNDataSeqMajorNotPadded) - { - auto from_gpu_hy = std::vector(hy_dev->GetSize() / sizeof(Tgpu)); - auto from_gpu_cy = std::vector(cy_dev->GetSize() / sizeof(Tgpu)); - - out_dev->FromGPU(GetStream(), from_gpu_out.data()); - hy_dev->FromGPU(GetStream(), from_gpu_hy.data()); - cy_dev->FromGPU(GetStream(), from_gpu_cy.data()); - - const std::vector batches_per_time = GetBatchesPerTime(sorted_seq_lens); - const std::vector order_idxs = - GetSamplesIndexDescendingOrder(unsorted_seq_lens, false); - - const std::vector hid_lens = GetHiddenTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - - TransformIODefaultLayaoutToTarget(out, - from_gpu_out, - batches_per_time, - order_idxs, - out_lens[1], - out_lens[2], - io_layout, - false); - - HiddenTensorReorder(from_gpu_hy, hy, order_idxs, hid_lens, true); - HiddenTensorReorder(from_gpu_cy, cy, order_idxs, hid_lens, true); - } - else - { - out_dev->FromGPU(GetStream(), out.data()); - hy_dev->FromGPU(GetStream(), hy.data()); - cy_dev->FromGPU(GetStream(), cy.data()); - } - reservespace_dev->FromGPU(GetStream(), reservespace.data()); - - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::RunBackwardGPU() -{ - int ret = 0; - if(inflags.GetValueInt("fwdtype") == 1 || inflags.GetValueInt("forw") == 1) - { - return ret; - } - - if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) - { - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - - workspace_dev->ToGPU(q, workspace.data()); - if(inflags.GetValueInt("inputmode") == 1) - { - // skip mode bug or feature, but din=F(...)+din - auto tmp_gpu_din = - std::vector(din_dev->GetSize() / sizeof(Tgpu), static_cast(0)); - din_dev->ToGPU(GetStream(), tmp_gpu_din.data()); - } - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - t.Start(); - ret = miopenRNNBackwardSeqData(GetHandle(), - rnnDesc, - outputSeqTensor, - out_dev->GetMem(), - dout_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - dhy_dev->GetMem(), - dhx_dev->GetMem(), - hiddenTensor, - cx_dev->GetMem(), - dcy_dev->GetMem(), - dcx_dev->GetMem(), - inputSeqTensor, - din_dev->GetMem(), - wei_dev->GetMem(), - wei_dev->GetSize(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - t.StopAndPush(); - } - - miopen::deref(GetHandle()).Finish(); - if(WALL_CLOCK) - { - printf("Backward Data RNN time results:\n"); - t.Print(); - } - - if(io_layout != miopenRNNDataSeqMajorNotPadded) - { - auto from_gpu_din = std::vector(din_dev->GetSize() / sizeof(Tgpu)); - auto from_gpu_dhx = std::vector(dhx_dev->GetSize() / sizeof(Tgpu)); - auto from_gpu_dcx = std::vector(dcx_dev->GetSize() / sizeof(Tgpu)); - - din_dev->FromGPU(GetStream(), from_gpu_din.data()); - dhx_dev->FromGPU(GetStream(), from_gpu_dhx.data()); - dcx_dev->FromGPU(GetStream(), from_gpu_dcx.data()); - - const std::vector batches_per_time = GetBatchesPerTime(sorted_seq_lens); - const std::vector order_idxs = - GetSamplesIndexDescendingOrder(unsorted_seq_lens, false); - - const std::vector hid_lens = GetHiddenTensorLengthsFromCmdLine(); - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - - TransformIODefaultLayaoutToTarget(din, - from_gpu_din, - batches_per_time, - order_idxs, - in_lens[1], - in_lens[2], - io_layout, - false); - - HiddenTensorReorder(from_gpu_dhx, dhx, order_idxs, hid_lens, true); - HiddenTensorReorder(from_gpu_dcx, dcx, order_idxs, hid_lens, true); - } - else - { - din_dev->FromGPU(GetStream(), din.data()); - dhx_dev->FromGPU(GetStream(), dhx.data()); - dcx_dev->FromGPU(GetStream(), dcx.data()); - } - workspace_dev->FromGPU(GetStream(), workspace.data()); - } - - if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) - { - RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); - - for(int i = 0; i < inflags.GetValueInt("iter"); i++) - { - t.Start(); - ret = miopenRNNBackwardWeightsSeqTensor(GetHandle(), - rnnDesc, - inputSeqTensor, - in_dev->GetMem(), - hiddenTensor, - hx_dev->GetMem(), - outputSeqTensor, - dout_dev->GetMem(), - dwei_dev->GetMem(), - dwei_dev->GetSize(), - workspace_dev->GetMem(), - workspace_dev->GetSize(), - reservespace_dev->GetMem(), - reservespace_dev->GetSize()); - t.StopAndPush(); - } - - miopen::deref(GetHandle()).Finish(); - - if(WALL_CLOCK) - { - printf("Backward Weights RNN time results:\n"); - t.Print(); - } - dwei_dev->FromGPU(GetStream(), dwei.data()); - } - - /* - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_din_gpu.bin", din.data(), din.size()); - dumpBufferToFile("dump_bwd_dwei_gpu.bin", dwei.data(), dwei.size()); - } - */ - - return ret; -} - -template -int RNNSeqDriver::RunForwardCPU() -{ - if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) - return miopenStatusSuccess; - - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - const int in_vec = in_lens[2]; - const int out_h = out_lens[2]; - - const std::vector hid_lens = GetHiddenTensorLengthsFromCmdLine(); - const int n_layer = hid_lens[0], batch_size = hid_lens[1], hid_vec = hid_lens[2]; - - const auto batchs = GetBatchesPerTime(sorted_seq_lens); - std::vector in_n(batchs.begin(), batchs.end()); - - bool bidirection, biased; - int layer; - - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - std::vector* in_packed = ∈ - std::vector* out_packed = &outhost; - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn fwd \n"); - RunRNNForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - *out_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - mode, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenLSTM) - { - printf("verify lstm fwd \n"); - - RunLSTMForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - cy_host, - cx, - *out_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenGRU) - { - printf("verify gru fwd \n"); - - RunGRUForwardGEMMCPUVerify(GetHandle(), - *in_packed, - wei, - hy_host, - hx, - *out_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else - { - printf("illegal RNN mode"); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_fwd_out_cpu.bin", outhost.data(), outhost.size()); - } - - // TrySaveVerificationCache("fwd_out", outhost); - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::RunBackwardWeightsCPU() -{ - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - const int in_vec = in_lens[2]; - const int out_h = out_lens[2]; - - const std::vector hid_lens = GetHiddenTensorLengthsFromCmdLine(); - const int n_layer = hid_lens[0], batch_size = hid_lens[1], hid_vec = hid_lens[2]; - - const auto batchs = GetBatchesPerTime(sorted_seq_lens); - std::vector in_n(batchs.begin(), batchs.end()); - - bool bidirection, biased; - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - std::vector* in_packed = ∈ - std::vector* dout_packed = &dout; - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn bwdwei \n"); - - RunRNNBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - mode, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else if(mode == miopenLSTM) - { - printf("verify lstm bwdwei \n"); - - RunLSTMBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else if(mode == miopenGRU) - { - printf("verify gru bwdwei \n"); - - RunGRUBackwardWeightGEMMCPUVerify(*in_packed, - dwei_host, - hx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout"))); - } - else - { - printf("illegal RNN mode"); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_dwei_cpu.bin", dwei_host.data(), dwei_host.size()); - } - - // TrySaveVerificationCache("bwd_wei", dwei_host); - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::RunBackwardDataCPU() -{ - const std::vector in_lens = GetInputTensorLengthsFromCmdLine(); - const std::vector out_lens = GetOutputTensorLengthsFromCmdLine(); - const int in_vec = in_lens[2]; - const int out_h = out_lens[2]; - - const std::vector hid_lens = GetHiddenTensorLengthsFromCmdLine(); - const int n_layer = hid_lens[0], batch_size = hid_lens[1], hid_vec = hid_lens[2]; - - const auto batchs = GetBatchesPerTime(sorted_seq_lens); - std::vector in_n(batchs.begin(), batchs.end()); - - bool bidirection, biased; - int layer; - miopenRNNMode_t mode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNBiasMode_t biasMode; - int hiddenSize; - miopenDropoutDescriptor_t drop_desc; - - if(inflags.GetValueInt("use_dropout")) - { - miopenGetRNNDescriptor_V2(rnnDesc, - &hiddenSize, - &layer, - &drop_desc, - &inputMode, - &dirMode, - &mode, - &biasMode, - &algoMode, - nullptr); - } - else - { - miopenGetRNNDescriptor( - rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer); - } - - bidirection = (dirMode == miopenRNNbidirection); - biased = (biasMode == miopenRNNwithBias); - - std::vector* din_packed = &din_host; - std::vector* dout_packed = &dout; - - if(mode == miopenRNNRELU || mode == miopenRNNTANH) - { - printf("verify rnn bwddata \n"); - - RunRNNBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - mode, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenLSTM) - { - printf("verify lstm bwddata \n"); - - RunLSTMBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - dcy, - dcx_host, - cx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else if(mode == miopenGRU) - { - printf("verify gru bwddata \n"); - - RunGRUBackwardDataGEMMCPUVerify(*din_packed, - wei, - dhy, - dhx_host, - hx, - *dout_packed, - in_n, - in_vec, - sorted_seq_lens[0], - bidirection, - biased, - n_layer, - batch_size, - hid_vec, - out_h, - inputMode, - reservespace_host, - workspace_host, - bool(inflags.GetValueInt("use_dropout")), - DropoutDesc); - } - else - { - printf("illegal RNN mode"); - } - - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_din_cpu.bin", din_host.data(), din_host.size()); - } - - // TrySaveVerificationCache("bwd_dat", din_host); - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::VerifyForward() -{ - RunForwardCPU(); - - auto error = miopen::rms_range(outhost, out); - - Tref tolerance = (sizeof(Tgpu) == 4 ? static_cast(1e-6) : static_cast(5e-2)); - - if(!std::isfinite(error) || error > tolerance) - { - std::cout << std::string("Forward RNN FAILED: ") << error << std::endl; - } - else - { - printf("Forward RNN Verifies on CPU and GPU\n"); - } - - auto error2 = miopen::rms_range(hy_host, hy); - - if(!std::isfinite(error2) || error2 > tolerance) - { - std::cout << std::string("final hidden state FAILED: ") << error2 << std::endl; - } - else - { - printf("final hidden Verifies on CPU and GPU\n"); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - auto error3 = miopen::rms_range(cy_host, cy); - - if(!std::isfinite(error3) || error3 > tolerance) - { - std::cout << std::string("final cell state FAILED: ") << error3 << std::endl; - } - else - { - printf("final cell Verifies on CPU and GPU\n"); - } - } - - return miopenStatusSuccess; -} - -template -int RNNSeqDriver::VerifyBackward() -{ - - if(inflags.GetValueInt("fwdtype") == 1 && inflags.GetValueInt("forw") != 1) - { - return miopenStatusSuccess; - } - - Tref tolerance = (sizeof(Tgpu) == 4 ? static_cast(1e-6) : static_cast(5e-2)); - - // if(!TryReadVerificationCache("bwd_dat", inputTensor, din_host.data())) - if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) - { - { - RunBackwardDataCPU(); - } - - auto error_data = miopen::rms_range(din_host, din); - - if(!std::isfinite(error_data) || error_data > tolerance) - { - std::cout << std::string("Backward RNN Data FAILED: ") << error_data << std::endl; - } - else - { - printf("Backward RNN Data Verifies on CPU and GPU\n"); - } - - auto error_data2 = miopen::rms_range(dhx_host, dhx); - - if(!std::isfinite(error_data2) || error_data2 > tolerance) - { - std::cout << std::string("difference at inital hidden state FAILED: ") << error_data2 - << std::endl; - } - else - { - printf("initial hidden state Verifies on CPU and GPU\n"); - } - - if((inflags.GetValueStr("mode")) == "lstm") - { - auto error_data3 = miopen::rms_range(dcx_host, dcx); - - if(!std::isfinite(error_data3) || error_data3 > tolerance) - { - std::cout << std::string("difference at inital cell state FAILED: ") << error_data3 - << std::endl; - } - else - { - printf("inital cell state Verifies on CPU and GPU\n"); - } - } - } - - // if(!TryReadVerificationCache("bwd_wei", weightTensor, dwei_host.data())) - if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) - { - { - RunBackwardWeightsCPU(); - } - - auto error_weights = miopen::rms_range(dwei_host, dwei); - if(!std::isfinite(error_weights) || error_weights > tolerance) - { - std::cout << std::string("Backward RNN Weights FAILED: ") << error_weights << std::endl; - } - else - { - printf("Backward RNN Weights Verifies on CPU and GPU\n"); - } - } - - return miopenStatusSuccess; -} diff --git a/driver/rnn_verify_gemm.hpp b/driver/rnn_verify_gemm.hpp index 94f90dc579..e69de29bb2 100644 --- a/driver/rnn_verify_gemm.hpp +++ b/driver/rnn_verify_gemm.hpp @@ -1,1342 +0,0 @@ -#ifndef GUARD_MIOPEN_RNN_VERIFY_GEMM_HPP -#define GUARD_MIOPEN_RNN_VERIFY_GEMM_HPP - -#define ADNN_MM_TRANSPOSE 1 - -#include "dropout_gpu_emulator.hpp" - -#include <../test/rnn_util.hpp> - -#include -#include -#include - -template -void RunRNNForwardGEMMCPUVerify(miopenHandle_t handle, - std::vector& in, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& hy_host, // current/final hidden state - std::vector& hx, // initial hidden state - std::vector& out_host, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) for - // unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by hy_h - // related function for bidirection - int squash, - int inputMode, - std::vector& rsvspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool hx_is_null = false) -{ - // printf("FWD TRAIN CPU:\n"); - // printf("seqLen: %d, in_h: %d, hy_d: %d, hy_n: %d, hy_h: %d, out_h: %d\n", seqLength, in_h, - // hy_d, hy_n, hy_h, out_h); - // printf("dirmode: %d, hx size: %d, hy_host size: %d, reserveSpace: %d\n", bidirection ? 2 : - // 1, hx.size(), hy_host.size(), rsvspace_host.size()); - // printf("input size: %d\n", in.size()); - // printf("output size: %d\n", out_host.size()); - int batch_n = sumvc(in_n); - std::vector hid_state(hy_d * batch_n * hy_h, static_cast(0)); - std::vector wk_state(hy_d * batch_n * hy_h, static_cast(0)); - std::vector out_state(batch_n * out_h, static_cast(0)); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int hy_stride = hy_h * bi; - int out_stride = out_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - // initial input - std::vector in_state(batch_n * in_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state.at(h * in_h + w) = in.at(h * in_h + w); - } - } - - // initial hidden states - std::vector hy_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector hx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state.at(h) = hx.at(h); - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - // initial weights - int wei_len = (bi * (in_h + hy_h) + (numlayer - 1) * bi * (bi + 1) * hy_h) * hy_h; - if(biased) - { - int in_bias = 2; - wei_len += (bi * in_bias + (numlayer - 1) * bi * 2) * hy_h; - } - - std::vector wei_state(wei_len, static_cast(0)); - for(int h = 0; h < wei_len; h++) - { - wei_state.at(h) = wei[h]; - } - - int wei_shift_bias = ((in_h + hy_h) * bi + (bi * hy_h + hy_h) * bi * (numlayer - 1)) * hy_h; - - // initial dropoput - std::vector dropout_states_host; - std::vector dropout_reservespace_host; - std::vector dropout_hid_state; - miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; - if(use_dropout) - { - size_t statesSizeInBytes = 0; - miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); - dropout_states_host = std::vector(states_size); - InitKernelStateEmulator(dropout_states_host, dropoutDesc); - - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - std::array drop_out_str = {{hy_h * bi, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenCreateTensorDescriptor(&dropout_outputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - miopenSetTensorDescriptor( - dropout_outputTensor, miopenFloat, 2, drop_in_len.data(), drop_out_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(1)); - - dropout_hid_state = - std::vector((numlayer - 1) * batch_n * hy_h * bi, static_cast(0)); - } - - // forward emulator - for(int li = 0; li < numlayer; li++) - { - int hid_shift = li * batch_n * hy_h * bi; - int hx_shift = li * bi * in_n.at(0) * hy_h; - - // from input - if(li == 0) - { - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - in_state.at(bs * in_stride + h); - if(bidirection) - { - hid_state.at(hid_shift + bs * hy_stride + hy_h + h) += - in_state.at(bs * in_stride + h); - } - } - } - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias + h); - } - } - } - } - else - { - ADNN_mm_cpu(in_state.data(), - in_h, - batch_n, - in_stride, - 0, - wei_state.data(), - in_h, - hy_h * bi, - in_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias + h); - } - } - } - } - } - else - { - int wei_shift = bi * (in_h + hy_h) * hy_h + (li - 1) * bi * (bi * hy_h + hy_h) * hy_h; - int prelayer_shift = (li - 1) * batch_n * hy_h * bi; - if(use_dropout) - { - auto dropout_states_tmp = dropout_states_host; - size_t drop_out_offset = (li - 1) * batch_n * hy_h * bi; - - RunDropoutForwardEmulator(handle, - dropoutDesc, - dropout_inputTensor, - dropout_inputTensor, - wk_state, - dropout_outputTensor, - dropout_hid_state, - dropout_reservespace_host, - dropout_states_tmp, - prelayer_shift, - drop_out_offset, - drop_out_offset); - - prelayer_shift = drop_out_offset; - } - - ADNN_mm_cpu(use_dropout ? &dropout_hid_state[prelayer_shift] - : &wk_state[prelayer_shift], - hy_h * bi, - batch_n, - use_dropout ? hy_h * bi : hy_stride, - 0, - &wei_state[wei_shift], - hy_h * bi, - hy_h * bi, - bi_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift], - hy_h * bi, - batch_n, - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * li * 2 * hy_h; - - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_stride; h++) - { - hid_state.at(hid_shift + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - } - - // from hidden state - bacc = 0; - baccbi = batch_n; - for(int ti = 0; ti < seqLength; ti++) - { - baccbi -= in_n.at(seqLength - 1 - ti); - - int wei_shift = - li == 0 ? (in_h * hy_h * bi) - : (bi * (in_h + hy_h) * hy_h + (li - 1) * bi * (bi * hy_h + hy_h) * hy_h + - bi * hy_h * hy_stride); - - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&hx_state[hx_shift], - hy_h, - in_n[ti], - uni_stride, - 0, - &wei_state[wei_shift], - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * (li * 2 + 1) * hy_h; - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - - if(bidirection) - { - ADNN_mm_cpu(&hx_state[hx_shift + hy_n * hy_h], - hy_h, - in_n[seqLength - 1 - ti], - uni_stride, - 0, - &wei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + hy_h], - hy_h, - in_n[seqLength - 1 - ti], - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * (li * 2 + 1) * hy_h; - - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + hy_h + h); - } - } - } - } - } - } - else - { - ADNN_mm_cpu(&hy_state[hx_shift], - hy_h, - in_n[ti], - uni_stride, - 0, - &wei_state[wei_shift], - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + bacc * hy_stride], - hy_h, - in_n[ti], - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * (li * 2 + 1) * hy_h; - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + h); - } - } - } - - if(bidirection) - { - - if(!hx_is_null && in_n.at(seqLength - 1 - ti) > in_n.at(seqLength - ti)) - { - ADNN_mm_cpu( - &hx_state[hx_shift + hy_n * hy_h + in_n.at(seqLength - ti) * hy_h], - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - uni_stride, - 0, - &wei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + (baccbi + in_n.at(seqLength - ti)) * hy_stride + - hy_h], - hy_h, - (in_n.at(seqLength - 1 - ti) - in_n.at(seqLength - ti)), - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * (li * 2 + 1) * hy_h; - - for(int bs = in_n.at(seqLength - ti); bs < in_n.at(seqLength - 1 - ti); - bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + hy_h + h); - } - } - } - } - - ADNN_mm_cpu(&hy_state[hx_shift + hy_n * hy_h], - hy_h, - in_n[seqLength - ti], - uni_stride, - 0, - &wei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - ADNN_MM_TRANSPOSE, - &hid_state[hid_shift + baccbi * hy_stride + hy_h], - hy_h, - in_n[seqLength - ti], - hy_stride, - 0, - 1, - 1); - - // from bias - if(biased) - { - int wei_shift_bias_temp = wei_shift_bias + bi * (li * 2 + 1) * hy_h; - - for(int bs = 0; bs < in_n.at(seqLength - ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - hid_state.at(hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h) += - wei.at(wei_shift_bias_temp + hy_h + h); - } - } - } - } - } - - for(int bs = 0; bs < in_n[ti]; bs++) - { - for(int h = 0; h < hy_h; h++) - { - wk_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) = - activfunc(hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h), - squash); // squash_func - hy_state.at(hx_shift + bs * uni_stride + h) = - wk_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h); - - rsvspace_host.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) = - hid_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h); - - rsvspace_host.at(hid_shift + bacc * hy_stride + bs * hy_stride + h + - numlayer * batch_n * hy_h * bi) = - activfunc(hid_state[hid_shift + bacc * hy_stride + bs * hy_stride + h], - squash); - - hy_host.at(hx_shift + bs * uni_stride + h) = - hy_state.at(hx_shift + bs * uni_stride + h); - } - } - - if(bidirection) - { - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - wk_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + h) = - activfunc(hid_state[hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h], - squash); // squash_func - - hy_state.at(hx_shift + hy_n * hy_h + bs * uni_stride + h) = - wk_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + h); - - rsvspace_host.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + - h) = hid_state.at(hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h); - - rsvspace_host.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + - h + numlayer * batch_n * hy_h * bi) = - activfunc(hid_state[hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h], - squash); - - hy_host.at(hx_shift + hy_n * hy_h + bs * uni_stride + h) = - hy_state.at(hx_shift + hy_n * hy_h + bs * uni_stride + h); - } - } - } - - bacc += in_n.at(ti); - } - } - - // output - int prelayer_shift = (numlayer - 1) * batch_n * hy_h * bi; - - if(use_dropout) - { - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - rsvspace_host.at(numlayer * batch_n * hy_stride * 2 + i) = dropout_hid_state.at(i); - } - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - *(p_drop_rsv + i) = dropout_reservespace_host.at(i); - } - } - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - assert(!std::isnan(wk_state.at(prelayer_shift + bs * hy_stride + h))); - out_host.at(bs * out_stride + h) = wk_state.at(prelayer_shift + bs * hy_stride + h); - // printf("out_host[%d]: %f\n", bs * out_stride + h, out_host.at(bs * out_stride + h)); - } - } -} - -template -void RunRNNBackwardDataGEMMCPUVerify(std::vector& din_host, - std::vector& wei, // [ input_state_weight_trans - // hidden_state_weight0_trans input1_trans - // hidden1_trans ... output_weight; - // bidirectional reversed weights ] - std::vector& dhy, // current/final hidden state - std::vector& dhx_host, - std::vector& hx, // initial hidden state - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden layers) - // for unidirection, 2 by numlayer for bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 by - // hy_h related function for bidirection - int squash, - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - miopenDropoutDescriptor_t dropoutDesc, - bool dhy_is_null = false) -{ - /* - printf("BWD DATA CPU driver:\n"); - printf("seqLen: %d, in_h: %d, hy_d: %d, hy_n: %d, hy_h: %d, out_h: %d\n", seqLength, in_h, - hy_d, hy_n, hy_h, out_h); - printf("hx size: %d, dhx size: %d, dhy size: %d, reserveSpace: %d, workSpace: %d\n", - hx.size(), dhx_host.size(), dhy.size(), rsvspace_host.size(),wkspace_host.size()); - printf("dinput size: %d\n", din_host.size()); - */ - int batch_n = sumvc(in_n); - std::vector dh_state(hy_d * batch_n * hy_h, static_cast(0)); - - std::vector din_state(batch_n * in_h, static_cast(0)); - - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc, baccbi; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int hy_stride = hy_h * bi; - int out_stride = out_h; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - (void)hx; - - // initial dout - std::vector dout_state(batch_n * out_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state.at(h * out_h + w) = dout.at(h * out_h + w); - } - } - - // initial hidden states - std::vector dhx_state(hy_d * hy_n * hy_h, static_cast(0)); - std::vector dhy_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - dhy_state.at(h) = dhy.at(h); - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - // initial weights - int wei_len = (bi * (in_h + hy_h) + (numlayer - 1) * bi * (bi + 1) * hy_h) * hy_h; - if(biased) - { - int in_bias = 2; - wei_len += (bi * in_bias + (numlayer - 1) * bi * 2) * hy_h; - } - - std::vector wei_state(wei_len, static_cast(0)); - for(int h = 0; h < wei_len; h++) - { - wei_state.at(h) = wei.at(h); - } - - // initial dropoput - miopenTensorDescriptor_t dropout_inputTensor{}; - std::vector dropout_reservespace_host; - if(use_dropout) - { - std::array drop_in_len = {{batch_n, hy_h * bi}}; - std::array drop_in_str = {{hy_stride, 1}}; - miopenCreateTensorDescriptor(&dropout_inputTensor); - miopenSetTensorDescriptor( - dropout_inputTensor, miopenFloat, 2, drop_in_len.data(), drop_in_str.data()); - - size_t reserveSpaceSizeInBytes = 0; - miopenDropoutGetReserveSpaceSize(dropout_inputTensor, &reserveSpaceSizeInBytes); - size_t reserve_size = reserveSpaceSizeInBytes / sizeof(unsigned char); - dropout_reservespace_host = std::vector(reserve_size * (numlayer - 1), - static_cast(0)); - - auto p_drop_rsv = - reinterpret_cast(&rsvspace_host[numlayer * batch_n * hy_stride * 2 + - (numlayer - 1) * batch_n * hy_h * bi]); - for(int i = 0; i < (numlayer - 1) * batch_n * hy_h * bi; i++) - { - dropout_reservespace_host.at(i) = *(p_drop_rsv + i); - } - } - - // bwd data emulator - for(int li = numlayer - 1; li >= 0; li--) - { - int wei_shift = bi * (in_h + hy_h) * hy_h + li * bi * (bi * hy_h + hy_h) * hy_h; - int hid_shift = li * batch_n * hy_h * bi; - int hx_shift = li * bi * in_n.at(0) * hy_h; - - if(li == numlayer - 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < out_h; h++) - { - dh_state.at(hid_shift + bs * hy_stride + h) += - dout_state.at(bs * out_stride + h); - } - } - } - else - { - int prelayer_shift = (li + 1) * batch_n * hy_h * bi; - - ADNN_mm_cpu(&dh_state[prelayer_shift], - hy_h * bi, - batch_n, - hy_stride, - 0, - &wei_state[wei_shift], - hy_h * bi, - hy_h * bi, - bi_stride, - 0, - &dh_state[hid_shift], - hy_h * bi, - batch_n, - hy_stride, - 0, - 1, - 1); - - if(use_dropout) - { - RunDropoutBackwardEmulator(dropoutDesc, - dropout_inputTensor, - dh_state, - dropout_inputTensor, - dh_state, - dropout_reservespace_host, - hid_shift, - hid_shift, - li * batch_n * hy_h * bi); - } - } - - bacc = batch_n; - baccbi = 0; - for(int ti = seqLength - 1; ti >= 0; ti--) - { - bacc -= in_n.at(ti); - - // from post state - if(ti == seqLength - 1) - { - if(!dhy_is_null) - { - for(int bs = 0; bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - dhy_state.at(hx_shift + bs * uni_stride + h); - } - } - } - } - else - { - if(!dhy_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - for(int bs = in_n.at(ti + 1); bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - dhy_state.at(hx_shift + bs * uni_stride + h); - } - } - } - - for(int bs = 0; bs < in_n.at(ti + 1); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) += - dhx_state.at(hx_shift + bs * uni_stride + h); - } - } - } - - for(int bs = 0; bs < in_n.at(ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - dh_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) *= dervactivfunc( - rsvspace_host.at(hid_shift + bacc * hy_stride + bs * hy_stride + h), - squash); - wkspace_host.at(hid_shift + bacc * hy_stride + bs * hy_stride + h) = - dh_state.at(hid_shift + bacc * hy_stride + bs * hy_stride + h); - } - } - - if(ti < seqLength - 1) - { - for(int bs = 0; bs < in_n.at(ti + 1); bs++) - { - memset(&dhx_state[hx_shift + bs * uni_stride], 0, hy_h * sizeof(Tref)); - } - } - - wei_shift = li == 0 - ? (in_h * hy_stride) - : (bi * (in_h + hy_h) * hy_h + - (li - 1) * bi * (bi * hy_h + hy_h) * hy_h + bi * hy_h * hy_stride); - - ADNN_mm_cpu(&dh_state[hid_shift + bacc * hy_stride], - hy_h, - in_n.at(ti), - hy_stride, - 0, - &wei_state[wei_shift], - hy_h, - hy_h, - uni_stride, - 0, - &dhx_state[hx_shift], - hy_h, - in_n.at(ti), - uni_stride, - 0, - 1, - 1); - - if(bidirection) - { - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - for(int h = 0; h < hy_h; h++) - { - // from post state - if(ti == seqLength - 1) - { - if(!dhy_is_null) - { - dh_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + - h) += - dhy_state.at(hx_shift + hy_n * hy_h + bs * uni_stride + h); - } - } - else - { - dh_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + - h) += - dhx_state.at(hx_shift + hy_n * hy_h + bs * uni_stride + h); - } - - dh_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + h) *= - dervactivfunc(rsvspace_host.at(hid_shift + baccbi * hy_stride + hy_h + - bs * hy_stride + h), - squash); - wkspace_host.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + - h) = - dh_state.at(hid_shift + baccbi * hy_stride + hy_h + bs * hy_stride + h); - } - } - - if(ti < seqLength - 1) - { - for(int bs = 0; bs < in_n.at(seqLength - 1 - ti); bs++) - { - memset(&dhx_state[hx_shift + bs * uni_stride + hy_n * hy_h], - 0, - hy_h * sizeof(Tref)); - } - } - - ADNN_mm_cpu(&dh_state[hid_shift + baccbi * hy_stride + hy_h], - hy_h, - in_n.at(seqLength - 1 - ti), - hy_stride, - 0, - &wei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - 0, - &dhx_state[hx_shift + hy_n * hy_h], - hy_h, - in_n.at(seqLength - 1 - ti), - uni_stride, - 0, - 1, - 1); - } - - baccbi += in_n.at(seqLength - 1 - ti); - } - } - - // dinput - if(inputMode == 1) - { - for(int bs = 0; bs < batch_n; bs++) - { - for(int h = 0; h < hy_h; h++) - { - din_state.at(bs * in_stride + h) += dh_state.at(bs * hy_stride + h); - if(bidirection) - { - din_state.at(bs * in_stride + h) += dh_state.at(bs * hy_stride + hy_h + h); - } - } - } - } - else - { - ADNN_mm_cpu(dh_state.data(), - hy_h * bi, - batch_n, - hy_stride, - 0, - wei_state.data(), - in_h, - hy_h * bi, - in_stride, - 0, - din_state.data(), - in_h, - batch_n, - in_stride, - 0, - 1, - 1); - } - - for(int bs = 0; bs < batch_n; bs++) - { - for(int w = 0; w < in_stride; w++) - { - din_host.at(bs * in_stride + w) = din_state.at(bs * in_stride + w); - } - } - - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - dhx_host.at(h) = dhx_state.at(h); - // printf("dhx_host[%d]: %f\n", h, dhx_host.at(h)); - } -} - -template -void RunRNNBackwardWeightGEMMCPUVerify(std::vector& in, - std::vector& dwei_host, // [ input_state_weight_trans - // hidden_state_weight0_trans - // input1_trans hidden1_trans ... - // output_weight; bidirectional - // reversed weights ] - std::vector& hx, // initial hidden state - std::vector& dout, - std::vector& in_n, // input batch size - int in_h, // input data length - int seqLength, // Number of iterations to unroll over - bool bidirection, // whether using bidirectional net - bool biased, // whether using bias - int hy_d, // 1 by numlayer (number of stacks of hidden - // layers) for unidirection, 2 by numlayer for - // bidirection - int hy_n, // equal to input batch size in_n[0] - int hy_h, // hidden state number - int out_h, // 1 by hy_h related function for unidirection, 2 - // by hy_h related function for bidirection - int squash, - int inputMode, - std::vector& rsvspace_host, - std::vector& wkspace_host, - bool use_dropout, - bool hx_is_null = false) -{ - - // printf("BWD WEGIHTS CPU driver:\n"); - // printf("seqLen: %d, in_h: %d, hy_d: %d, hy_n: %d, hy_h: %d, out_h: %d\n", seqLength, in_h, - // hy_d, hy_n, hy_h, out_h); - // printf("dirmode: %d, hx size: %d, dout size: %d, reserveSpace: %d, workSpace: %d\n", - // bidirection ? 2 : 1, hx.size(), dout.size(), rsvspace_host.size(),wkspace_host.size()); - // printf("input size: %d\n", in.size()); - int batch_n = sumvc(in_n); - int numlayer = bidirection ? hy_d / 2 : hy_d; - int bacc; // accumulation of batch - int bi = bidirection ? 2 : 1; - - int in_stride = in_h; - int hy_stride = hy_h * bi; - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - (void)hy_n; - - // initial input - std::vector in_state(batch_n * in_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < in_h; w++) - { - in_state.at(h * in_h + w) = in.at(h * in_h + w); - } - } - - // initial output difference - std::vector dout_state(batch_n * out_h, static_cast(0)); - for(int h = 0; h < batch_n; h++) - { - for(int w = 0; w < out_h; w++) - { - dout_state.at(h * out_h + w) = dout.at(h * out_h + w); - } - } - - // initial saved data - std::vector wkspace_state(hy_d * batch_n * hy_h, static_cast(0)); - std::vector rsvspace_state(rsvspace_host.size(), static_cast(0)); - for(int h = 0; h < hy_d * batch_n * hy_h; h++) - { - rsvspace_state.at(h) = activfunc(rsvspace_host.at(h), squash); - wkspace_state.at(h) = wkspace_host.at(h); - } - if(use_dropout) - { - for(int h = hy_d * batch_n * hy_h; h < rsvspace_host.size(); h++) - { - rsvspace_state.at(h) = rsvspace_host.at(h); - } - } - - // initial hidden states - std::vector hx_state(hy_d * hy_n * hy_h, static_cast(0)); - for(int h = 0; h < hy_d * hy_n * hy_h; h++) - { - hx_state.at(h) = hx.at(h); - } - - if(inputMode == 1) - { - if(in_h != hy_h) - { - printf("Verification cannot be completed: The input tensor size must equal to the " - "hidden state size of the network in SKIP_INPUT mode!\n"); - return; - } - in_h = 0; - } - - int wei_len = (bi * (in_h + hy_h) + (numlayer - 1) * bi * (bi + 1) * hy_h) * hy_h; - int wei_shift_bias = wei_len; - if(biased) - { - int in_bias = 2; - wei_len += (bi * in_bias + (numlayer - 1) * bi * 2) * hy_h; - } - - // initial dwei - std::vector dwei_state(wei_len, static_cast(0)); - - // bwd weights emulator - for(int li = 0; li < numlayer; li++) - { - // between layers - if(li == 0) - { - if(inputMode != 1) - { - ADNN_mm_cpu(wkspace_state.data(), - hy_h * bi, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - in_state.data(), - in_h, - batch_n, - in_stride, - 0, - dwei_state.data(), - in_h, - hy_h * bi, - in_stride, - 0, - 1, - 1); - } - if(biased) - { - for(int h = 0; h < hy_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state.at(wei_shift_bias + h) += wkspace_host.at(w * hy_stride + h); - } - } - } - } - else - { - int prelayer_shift = (use_dropout ? 2 * numlayer * batch_n * hy_stride : 0) + - (li - 1) * batch_n * hy_h * bi; - int hid_shift = li * bi * batch_n * hy_h; - int wei_shift = bi * (in_h + hy_h) * hy_h + (li - 1) * bi * (bi * hy_h + hy_h) * hy_h; - - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h * bi, - batch_n, - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[prelayer_shift], - hy_h * bi, - batch_n, - hy_stride, - 0, - &dwei_state[wei_shift], - hy_h * bi, - hy_h * bi, - bi_stride, - 0, - 1, - 1); - - if(biased) - { - wei_shift = wei_shift_bias + li * bi * 2 * hy_h; - - for(int h = 0; h < hy_stride; h++) - { - for(int w = 0; w < batch_n; w++) - { - dwei_state.at(wei_shift + h) += - wkspace_host.at(hid_shift + w * hy_stride + h); - } - } - } - } - - bacc = 0; - for(int ti = 0; ti < seqLength; ti++) - { - int hid_shift = li * bi * batch_n * hy_h + bacc * hy_stride; - int hx_shift = li * bi * in_n.at(0) * hy_h; - int wei_shift; - int pretime_shift; - - wei_shift = li == 0 - ? (in_h * hy_stride) - : (bi * (in_h + hy_h) * hy_h + - (li - 1) * bi * (bi * hy_h + hy_h) * hy_h + bi * hy_h * hy_stride); - - // between time - if(ti == 0) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h, - in_n.at(ti), - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift], - hy_h, - in_n.at(ti), - uni_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * bi * 2 * hy_h + bi * hy_h; - - for(int h = 0; h < hy_h; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state.at(bias_shift + h) += - wkspace_host.at(hid_shift + w * hy_stride + h); - } - } - } - } - } - else - { - pretime_shift = li * bi * batch_n * hy_h + (bacc - in_n.at(ti - 1)) * hy_stride; - - ADNN_mm_cpu(&wkspace_state[hid_shift], - hy_h, - in_n.at(ti), - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[pretime_shift], - hy_h, - in_n.at(ti), - hy_stride, - 0, - &dwei_state[wei_shift], - hy_h, - hy_h, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * bi * 2 * hy_h + bi * hy_h; - - for(int h = 0; h < hy_h; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state.at(bias_shift + h) += - wkspace_host.at(hid_shift + w * hy_stride + h); - } - } - } - } - - if(bidirection) - { - if(ti == seqLength - 1) - { - if(!hx_is_null) - { - ADNN_mm_cpu(&wkspace_state[hid_shift + hy_h], - hy_h, - in_n.at(ti), - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift + hy_n * hy_h], - hy_h, - in_n.at(ti), - uni_stride, - 0, - &dwei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * bi * 2 * hy_h + bi * hy_h; - - for(int h = 0; h < hy_h; h++) - { - for(int w = 0; w < in_n.at(ti); w++) - { - dwei_state.at(bias_shift + hy_h + h) += - wkspace_host.at(hid_shift + w * hy_stride + hy_h + h); - } - } - } - } - } - else - { - if(!hx_is_null && in_n.at(ti) > in_n.at(ti + 1)) - { - ADNN_mm_cpu( - &wkspace_state[hid_shift + hy_h + in_n.at(ti + 1) * hy_stride], - hy_h, - (in_n.at(ti) - in_n.at(ti + 1)), - hy_stride, - ADNN_MM_TRANSPOSE, - &hx_state[hx_shift + hy_n * hy_h + in_n.at(ti + 1) * hy_h], - hy_h, - (in_n.at(ti) - in_n.at(ti + 1)), - uni_stride, - 0, - &dwei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * bi * 2 * hy_h + bi * hy_h; - - for(int h = 0; h < hy_h; h++) - { - for(int w = in_n.at(ti + 1); w < in_n.at(ti); w++) - { - dwei_state.at(bias_shift + hy_h + h) += - wkspace_host.at(hid_shift + w * hy_stride + hy_h + h); - } - } - } - } - - pretime_shift = li * bi * batch_n * hy_h + (bacc + in_n.at(ti)) * hy_stride; - - ADNN_mm_cpu(const_cast(&wkspace_state[hid_shift + hy_h]), - hy_h, - in_n.at(ti + 1), - hy_stride, - ADNN_MM_TRANSPOSE, - &rsvspace_state[pretime_shift + hy_h], - hy_h, - in_n.at(ti + 1), - hy_stride, - 0, - &dwei_state[wei_shift + hy_h * uni_stride], - hy_h, - hy_h, - uni_stride, - 0, - 1, - 1); - - if(biased) - { - int bias_shift = wei_shift_bias + li * bi * 2 * hy_h + bi * hy_h; - - for(int h = 0; h < hy_h; h++) - { - for(int w = 0; w < in_n.at(ti + 1); w++) - { - dwei_state.at(bias_shift + hy_h + h) += - wkspace_host.at(hid_shift + w * hy_stride + hy_h + h); - } - } - } - } - } - - bacc += in_n.at(ti); - } - } - - for(int i = 0; i < wei_len; i++) - { - dwei_host.at(i) = dwei_state.at(i); - } -} - -#endif // GUARD_MIOPEN_RNN_VERIFY_GEMM_HPP diff --git a/include/miopen/miopen.h b/include/miopen/miopen.h index 7ed36c72a4..e69de29bb2 100644 --- a/include/miopen/miopen.h +++ b/include/miopen/miopen.h @@ -1,7802 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef MIOPEN_GUARD_MIOPEN_H_ -#define MIOPEN_GUARD_MIOPEN_H_ - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wextern-c-compat" -#endif - -#include -#include -#include -#include - -#if MIOPEN_BACKEND_OPENCL -#define CL_TARGET_OPENCL_VERSION 120 -#if defined(__APPLE__) || defined(__MACOSX) -#include -#else -#define CL_USE_DEPRECATED_OPENCL_1_2_APIS -#include -#endif - -#elif MIOPEN_BACKEND_HIP -#include -#endif - -/* - * @defgroup convolutions - * @defgroup pooling - * @defgroup handle - * @defgroup layernorm - * @defgroup LRN - * @defgroup batchnorm - * @defgroup activation - * @defgroup tensor - * @defgroup softmax - * @defgroup RNN - * @defgroup fusion - * @defgroup LossFunction - * @defgroup TensorReduce - * @defgroup find2 - * @defgroup ReduceExtreme - * @defgroup groupnorm - * @defgroup cat - * @defgroup SGD - * @defgroup getitem - * @defgroup ReduceCalculation - * @defgroup RotaryPositionalEmbeddings - * @defgroup ReLU - * - */ - -/*! Constructs type name from a struct */ -#define MIOPEN_DECLARE_OBJECT(name) \ - struct name \ - { \ - }; \ - typedef struct name* name##_t; - -#ifdef __cplusplus -extern "C" { -#endif - -#if MIOPEN_BACKEND_OPENCL -typedef cl_command_queue miopenAcceleratorQueue_t; -#elif MIOPEN_BACKEND_HIP -typedef hipStream_t miopenAcceleratorQueue_t; -#endif - -/*! @ingroup handle - * @brief Creates the miopenHandle_t type - */ -MIOPEN_DECLARE_OBJECT(miopenHandle); - -/** @addtogroup handle - * - * @{ - */ - -/*! @enum miopenStatus_t - * Error codes that are returned by all MIOpen API calls. - */ -typedef enum -{ - miopenStatusSuccess = 0, /*!< No errors */ - miopenStatusNotInitialized = 1, /*!< Data not initialized. */ - miopenStatusInvalidValue = 2, /*!< Incorrect variable value. */ - miopenStatusBadParm = 3, /*!< Incorrect parameter detected. */ - miopenStatusAllocFailed = 4, /*!< Memory allocation error. */ - miopenStatusInternalError = 5, /*!< MIOpen failure. */ - miopenStatusNotImplemented = 6, /*!< Use of unimplemented feature. */ - miopenStatusUnknownError = 7, /*!< Unknown error occurred. */ - miopenStatusUnsupportedOp = 8, /*!< Unsupported operator for fusion. */ - miopenStatusGpuOperationsSkipped = 9, /*!< This is not an error. */ - miopenStatusVersionMismatch = 10, /*!< Version mismatch of the supplied binary data argment. */ -} miopenStatus_t; - -#ifdef MIOPEN_BETA_API -typedef enum -{ - miopenF8RoundingModeStandard = 0, - miopenF8RoundingModeStochastic = 1, -} miopenF8RoundingMode_t; -#endif - -/*! @brief Get character string for an error code. - * - * A function which returns a NULL terminated character string of the error code. - * - * @param error miopenStatus_t type error status (input) - * @return errorString - */ -MIOPEN_EXPORT const char* miopenGetErrorString(miopenStatus_t error); - -/*! @brief Custom allocator function - * - * This function allow for user-defined custom allocator - * - * @param context A pointer a context (input) - * @param sizeBytes Number of bytes to allocate (input) - * - */ -typedef void* (*miopenAllocatorFunction)(void* context, size_t sizeBytes); - -/*! @brief Custom deallocator function - * - * This function allow for user-defined custom deallocation function - * - * @param context A pointer context (input) - * @param memory A pointer allocated memory (input) - * - */ -typedef void (*miopenDeallocatorFunction)(void* context, void* memory); - -/*! @brief Method to return version of MIOpen - * - * The output values of this call follow from the versioning - * format major.minor.patch - * - * Pointers that are NULL will be ignored. - * - * @param major Major version number (output) - * @param minor Minor version number (output) - * @param patch Patch version number (output) - * - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetVersion(size_t* major, size_t* minor, size_t* patch); - -/*! @brief Method to create the MIOpen handle object. - * - * This function creates a MIOpen handle. This is called at the very start to initialize the MIOpen - * environment. - * @param handle A pointer to a MIOpen handle type (output) - * - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreate(miopenHandle_t* handle); - -/*! @brief Create a MIOpen handle with an accelerator stream. - * - * The HIP side uses a hipStream_t type for the stream, while OpenCL will use a - * cl_command_queue. - * - * Create a handle with a previously created accelerator command queue. - * @param handle A pointer to a MIOpen handle type (output) - * @param stream An accelerator queue type (input) - * - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateWithStream(miopenHandle_t* handle, - miopenAcceleratorQueue_t stream); - -/*! @brief Destroys the MIOpen handle. - * - * This is called when breaking down the MIOpen environment. - * @param handle MIOpen handle (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroy(miopenHandle_t handle); - -/*! @brief Set accelerator command queue previously created - * - * Set a command queue for an accelerator device - * @param handle MIOpen handle (input) - * @param streamID An accelerator queue type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetStream(miopenHandle_t handle, - miopenAcceleratorQueue_t streamID); - -/*! @brief Get the previously created accelerator command queue - * - * Creates a command queue for an accelerator device - * @param handle MIOpen handle (input) - * @param streamID Pointer to a accelerator queue type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetStream(miopenHandle_t handle, - miopenAcceleratorQueue_t* streamID); - -/*! @brief Set allocator for previously created miopenHandle - * - * Set a command queue for an accelerator device - * @param handle MIOpen handle - * @param allocator A callback function MIOpen will use for internal memory allocations. - * The provided callback function should allocate device memory with requested size - * and return a pointer to this memory. - * Passing 0 will restore the default MIOpen allocator and deallocator. - * @param deallocator A callback function MIOpen will use to for internal memory deallocation. - * The provided callback function should free the specified memory pointer - * @param allocatorContext User-specified pointer which is passed to \p allocator and \p - * deallocator - * This allows the callback function to access state set by the caller to this function, - * for example a stateful heap allocator or a c++ class. - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetAllocator(miopenHandle_t handle, - miopenAllocatorFunction allocator, - miopenDeallocatorFunction deallocator, - void* allocatorContext); - -/*! @brief Get time for last kernel launched - * - * This function is used only when profiling mode has been enabled. - * Kernel timings are based on the MIOpen handle and is not thread-safe. - * In order to use multi-threaded profiling, create an MIOpen handle for each - * concurrent thread. - * - * @param handle MIOpen handle (input) - * @param time Pointer to a float type to contain kernel time in milliseconds (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetKernelTime(miopenHandle_t handle, float* time); - -/*! @brief Enable profiling to retrieve kernel time - * - * Enable or disable kernel profiling. This profiling is only for kernel time. - * @param handle MIOpen handle (input) - * @param enable Boolean to toggle profiling (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenEnableProfiling(miopenHandle_t handle, bool enable); -/** @} */ -// CLOSEOUT HANDLE DOXYGEN GROUP - -/*! @ingroup fusion - * @brief Creates the miopenFusionOpDescriptor_t type - * - * Fusion Operator Descriptor contains the meta-data associated with an operator - * to be fused in a compute graph - * - */ -MIOPEN_DECLARE_OBJECT(miopenFusionOpDescriptor); - -/*! @ingroup tensor - * @brief Creates the miopenTensorDescriptor_t type - * - * Tensor descriptor is an object that allows the user to specify a layer's size for each - * dimension and dimension strides. - * - */ -MIOPEN_DECLARE_OBJECT(miopenTensorDescriptor); - -/*! @ingroup tensor - * @brief Creates the miopenSeqTensorDescriptor_t type - * - * SeqTensor descriptor is an object that allows the user to specify tensor with sequence dimension. - * - */ -MIOPEN_DECLARE_OBJECT(miopenSeqTensorDescriptor); - -/*! @ingroup convolutions - * @brief Creates the miopenConvolutionDescriptor_t type - * - * Convolution descriptor is an object that allows the user to specify a layer's padding, stride, - * and dilation of the convolutional filter. Parameters must all be non-negative. - * - */ -MIOPEN_DECLARE_OBJECT(miopenConvolutionDescriptor); - -/*! @ingroup pooling - * @brief Creates the miopenPoolingDescriptor_t type - * - * Pooling descriptor is an object that allows the user to specify the dimension sizes of the - * pooling windows, paddings, strides, and pooling mode. - * - */ -MIOPEN_DECLARE_OBJECT(miopenPoolingDescriptor); - -/*! @ingroup LRN - * @brief Creates the miopenLRNDescriptor_t type - * - * LRN descriptor is an object that allows the user to specify the LRN mode, the number of elements - * in the normalization window, and the LRN k-parameter. - * - */ -MIOPEN_DECLARE_OBJECT(miopenLRNDescriptor); - -/*! @ingroup activation - * @brief Creates the miopenActivationDescriptor_t type - * - * Activation descriptor is an object that allows the user to specify the activation mode. - * - */ -MIOPEN_DECLARE_OBJECT(miopenActivationDescriptor); - -/*! @ingroup RNN - * @brief Creates the miopenRNNDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenRNNDescriptor); - -/*! @ingroup LossFunction - * @brief Creates the miopenCTCLossDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenCTCLossDescriptor); - -/*! @ingroup Dropout - * @brief Creates the miopenDropoutDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenDropoutDescriptor); - -/*! @ingroup TensorReduce - * @brief Creates the miopenReduceTensorDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenReduceTensorDescriptor); - -/*! @ingroup mha - * @brief Creates the miopenMhaDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenMhaDescriptor); - -/*! @ingroup softmax - * @brief Creates the miopenSoftmaxDescriptor_t type - */ -MIOPEN_DECLARE_OBJECT(miopenSoftmaxDescriptor); - -/*! @ingroup tensor - * @enum miopenDataType_t - * MIOpen floating point datatypes. Both 32-bit and 16-bit floats are supported in MIOpen. - */ -typedef enum -{ - miopenHalf = 0, /*!< 16-bit floating point (Fully supported) */ - miopenFloat = 1, /*!< 32-bit floating point (Fully supported) */ - miopenInt32 = 2, /*!< 32-bit integer (Partially supported) */ - miopenInt8 = 3, /*!< 8-bit integer (Partially supported) */ - // miopenInt8x4 = 4, /*!< Pack of 4x Int8 in NCHW_VECT_C format (Support discontinued) */ - miopenBFloat16 = 5, /*!< 16-bit binary floating point (8-bit exponent, 7-bit fraction) - (Partially supported) */ - miopenDouble = 6, /*!< 64-bit floating point (Partially supported) */ -#ifdef MIOPEN_BETA_API - miopenFloat8 = 7, - miopenBFloat8 = 8, -#else -// miopenReserved1 = 7, -// miopenReserved2 = 8, -#endif - miopenInt64 = 9, -} miopenDataType_t; - -/*! @ingroup tensor - * @enum miopenTensorLayout_t - * Tensor layouts supported by MIOpen. - * miopenTensorCHWNc4 and miopenTensorCHWNc8 layout only support weight tensor. - */ -typedef enum -{ - miopenTensorNCHW = 0, /*!< NCHW memory layout (Fully supported) */ - miopenTensorNHWC = 1, /*!< NHWC memory layout (Fully supported) */ - miopenTensorCHWN = 2, /*!< CHWN memory layout (Not supported) */ - miopenTensorNCHWc4 = 3, /*!< NCHWc4 memory layout (Partially supported) */ - miopenTensorNCHWc8 = 4, /*!< NCHWc8 memory layout (Partially supported) */ - miopenTensorCHWNc4 = 5, /*!< CHWNc4 memory layout (Partially supported) */ - miopenTensorCHWNc8 = 6, /*!< CHWNc8 memory layout (Partially supported) */ - miopenTensorNCDHW = 7, /*!< NCDHW memory layout (Fully supported) */ - miopenTensorNDHWC = 8, /*!< NCDHW memory layout (Fully supported) */ -} miopenTensorLayout_t; - -/*! @ingroup pooling - * @enum miopenIndexType_t - * MIOpen index datatypes. - */ -typedef enum -{ - miopenIndexUint8 = 0, /*!< 8-bit unsigned */ - miopenIndexUint16 = 1, /*!< 16-bit unsigned */ - miopenIndexUint32 = 2, /*!< 32-bit unsigned */ - miopenIndexUint64 = 3, /*!< 64-bit unsigned */ -} miopenIndexType_t; - -/*! @ingroup tensor - * @enum miopenTensorOp_t - * Element-wise tensor operation modes - */ -typedef enum -{ - miopenTensorOpAdd = 0, /*!< Add tensors element-wise */ - miopenTensorOpMul = 1, /*!< Multiply two tensors element-wise */ - miopenTensorOpMin = 2, /*!< Minimum of tensor element pairs */ - miopenTensorOpMax = 3, /*!< Maximum of tensor element pairs */ -} miopenTensorOp_t; - -/*! @ingroup convolutions - * @enum miopenConvolutionMode_t - * Convolution mode selection for convolution layer preference. - */ -typedef enum -{ - miopenConvolution = 0, /*!< Cross-Correlation convolution */ - miopenTranspose = 1, /*!< Transpose convolutions -- deconvolution */ - miopenGroupConv = 2, /*!< Deprecated Group convolution legacy, ToBe Removed */ - miopenDepthwise = 3, /*!< Deprecated Depthwise convolution legacy, ToBe Removed */ -} miopenConvolutionMode_t; - -/*! @ingroup padding - * @enum miopenPaddingMode_t - * Padding mode selection for convolution/Pooling layer preference - */ -typedef enum -{ - miopenPaddingDefault = 0, /*!< MIOPEN Default Padding */ - miopenPaddingSame = 1, /*!< Tensorflow SAME Padding */ - miopenPaddingValid = 2, /*!< Tensorflow VALID Padding */ -} miopenPaddingMode_t; - -/*! @ingroup pooling - * @enum miopenPoolingMode_t - * Pooling layer mode - */ -typedef enum -{ - miopenPoolingMax = 0, /*!< Maximum pooling */ - miopenPoolingAverage = 1, /*!< Average pooling */ - miopenPoolingAverageInclusive = 2, /*!< Inclusive Average pooling */ -} miopenPoolingMode_t; - -/*! @ingroup pooling - * @enum miopenPoolingWorkspaceIndexMode_t - * Pooling layer workspace index mode. miopenPoolingWorkspaceIndexMask mode records indices - * indicating the max values' positions in the filter/mask. miopenPoolingWorkspaceIndexImage mode - * records indices indicating the max values' positions in the image. - */ -typedef enum -{ - miopenPoolingWorkspaceIndexMask = 0, /*!< Use mask indices, 2D pooling only */ - miopenPoolingWorkspaceIndexImage = 1, /*!< Use image indices */ -} miopenPoolingWorkspaceIndexMode_t; - -/*! @ingroup LRN - * @enum miopenLRNMode_t - * Local Response Normalization layer mode - */ -typedef enum -{ - miopenLRNWithinChannel = 0, /*!< Channel independent */ - miopenLRNCrossChannel = 1, /*!< Cross Channel */ -} miopenLRNMode_t; -#ifdef MIOPEN_BETA_API -/*! @ingroup layernorm - * @enum miopenNormMode_t - * LayerNorm mode - */ -typedef enum -{ - MIOPEN_ELEMENTWISE_AFFINE = 0, /*!< initialized to ones for weights and zeros for biases */ - MIOPEN_WEIGHT_BIAS = - 1, /*!< learnable weights and biases of the module of shape normalized_shape */ - MIOPEN_ELEMENTWISE_AFFINE_FUSED_ADD = - 2, /*!< initialized to ones for weights and zeros for biases in addlayernorm */ - MIOPEN_WEIGHT_BIAS_FUSED_ADD = 3, /*!< learnable weights and biases of the module of shape - normalized_shape in addlayernorm */ - MIOPEN_ELEMENTWISE_AFFINE_T5 = - 4, /*!< initialized to ones for weights and zeros for biases in t5layernorm */ - MIOPEN_WEIGHT_BIAS_T5 = 5, /*!< learnable weights and biases of the module of shape - normalized_shape in t5layernorm */ -} miopenNormMode_t; -#endif -/*! @ingroup batchnorm - * @enum miopenBatchNormMode_t - * Batch Normalization layer mode - */ -typedef enum -{ - miopenBNPerActivation = 0, /*!< Element-wise normalization for fully connected layer */ - miopenBNSpatial = 1, /*!< Mini-batch spatial normalization for convolutional layers */ -} miopenBatchNormMode_t; - -/*! @ingroup activation - * @enum miopenActivationMode_t - * Activation layer modes - */ -typedef enum -{ - miopenActivationPASTHRU = 0, /*!< No activation, pass through the data */ - miopenActivationLOGISTIC = 1, /*!< Sigmoid function: \f$1 / (1 + e^{-x})\f$ */ - miopenActivationTANH = 2, /*!< Tanh activation \f$ \beta * tanh( \alpha * x) \f$ */ - miopenActivationRELU = 3, /*!< Rectified Linear Unit \f$ max(0, x) \f$ */ - miopenActivationSOFTRELU = 4, /*!< \f$log(1 + e^x)\f$ */ - miopenActivationABS = 5, /*!< Absolute value \f$abs(x)\f$ */ - miopenActivationPOWER = 6, /*!< Scaled and shifted power \f$(\alpha + \beta * x)^{gamma}\f$ */ - miopenActivationCLIPPEDRELU = - 7, /*!< Clipped Rectified Linear Unit \f$ min(\alpha, max(0,x)) \f$ */ - miopenActivationLEAKYRELU = - 8, /*!< Leaky Rectified Linear Unit \f$ \alpha * x | x <= 0; x | x > 0 \f$ */ - miopenActivationELU = - 9, /*!< Exponential Rectified Linear Unit \f$ \alpha * (e^{x} - 1) | x <= 0; x | x > 0 \f$ - */ -} miopenActivationMode_t; - -/*! @ingroup softmax - * @enum miopenSoftmaxAlgorithm_t - * Softmax implementation algorithms - */ -typedef enum -{ - MIOPEN_SOFTMAX_FAST = 0, /*!< straightforward softmax */ - MIOPEN_SOFTMAX_ACCURATE = 1, /*!< scaled softmax by maximum value in input domain */ - MIOPEN_SOFTMAX_LOG = 2, /*!< log softmax */ -} miopenSoftmaxAlgorithm_t; - -/*! @ingroup softmax - * @enum miopenSoftmaxMode_t - * Softmax modes - */ -typedef enum -{ - MIOPEN_SOFTMAX_MODE_INSTANCE = 0, /*!< compute per image (N) across C, H, W */ - MIOPEN_SOFTMAX_MODE_CHANNEL = - 1, /*!< compute per spatial location (H, W) per image (N) across C */ -} miopenSoftmaxMode_t; - -/*! @ingroup TensorReduce - * @brief Version of TensorReduce API. Applications may use it to ensure - * backward compatibility with older library versions. - * - * - 0 or undefined - Initial API. Supported operations: ADD, MIN, MIN, MAX. - * - 1 - Added AMAX, AVG, NORM1, NORM2 ops. - */ -#define MIOPEN_API_VERSION_REDUCE_TENSOR 1 - -/*! @ingroup TensorReduce - * @enum miopenReduceTensorOp_t - * Tensor Reduction operation types - */ -typedef enum -{ - MIOPEN_REDUCE_TENSOR_ADD = 0, /*!< the operation is adding the values of the reduced elements */ - MIOPEN_REDUCE_TENSOR_MUL = - 1, /*!< the operation is multiplying the values of the reduced elements */ - MIOPEN_REDUCE_TENSOR_MIN = - 2, /*!< the operation is getting the minimum value of the reduced elements */ - MIOPEN_REDUCE_TENSOR_MAX = - 3, /*!< the operation is getting the maximum value of the reduced elements */ - MIOPEN_REDUCE_TENSOR_AMAX = - 4, /*!< the operation is getting the maximum absolute value of the reduced elements */ - MIOPEN_REDUCE_TENSOR_AVG = - 5, /*!< the operation is getting the averaged value of the reduced elements */ - MIOPEN_REDUCE_TENSOR_NORM1 = - 6, /*!< the operation is adding the absolute values of the reduced elements */ - MIOPEN_REDUCE_TENSOR_NORM2 = 7, /*!< the operation is getting the square root of the sum of - squares of the reduced elements */ - // MIOPEN_REDUCE_TENSOR_MUL_NO_ZEROS = - // 8, /*!< the operation is same as MUL, but does not have the zero values considered */ -} miopenReduceTensorOp_t; - -/*! @ingroup TensorReduce - * @enum miopenReduceTensorOp_t - * Nan numbers propagation modes - */ -typedef enum -{ - MIOPEN_NOT_PROPAGATE_NAN = 0, /*!< does not propagate Nan number */ - MIOPEN_PROPAGATE_NAN = 1, /*!< propagate the Nan number by the Reduction operation */ -} miopenNanPropagation_t; - -/*! @ingroup TensorReduce - * @enum miopenReduceTensorIndices_t - * Reduction Indices computation modes - */ -typedef enum -{ - MIOPEN_REDUCE_TENSOR_NO_INDICES = 0, /*!< Does not compuate indices */ - MIOPEN_REDUCE_TENSOR_FLATTENED_INDICES = 1, /*!< Compute the relative, flatted indices */ -} miopenReduceTensorIndices_t; - -/*! @ingroup TensorReduce - * @enum miopenIndicesType_t - * Reduction Indices types - */ -typedef enum -{ - MIOPEN_32BIT_INDICES = 0, /*!< 32-bit unsigned integer indices */ - MIOPEN_64BIT_INDICES = 1, /*!< 64-bit unsigned integer indices */ - MIOPEN_16BIT_INDICES = 2, /*!< 16-bit unsigned integer indices */ - MIOPEN_8BIT_INDICES = 3, /*!< 8-bit unsigned integer indices */ -} miopenIndicesType_t; - -/*! @ingroup convolutions - * @enum miopenConvolutionAttrib_t - * Attribute for convolution descriptor, used for alternating the convolution behavior - */ -typedef enum -{ - MIOPEN_CONVOLUTION_ATTRIB_FP16_ALT_IMPL = - 0, /*!< Use alternative fp16 implementation. - Only supported for gfx90a; has no effect for other targets. - 0 - disabled, 1 - enabled, -1 or unset - default (F0B1W1) >*/ - MIOPEN_CONVOLUTION_ATTRIB_DETERMINISTIC = - 1, /*!< Restrict MIOpen convolutions to kernels which produce numerically deterministic - results. 0 - disabled (default), 1 - enabled >*/ -#ifdef MIOPEN_BETA_API - MIOPEN_CONVOLUTION_ATTRIB_FP8_ROUNDING_MODE = - 2, /*!*/ -#else -// miopenReserved1 = 2, -#endif -} miopenConvolutionAttrib_t; - -/** @addtogroup tensor - * - * @{ - */ - -/*! @brief Create a Tensor Descriptor - * - * API for creating an uninitialized tensor descriptor. - * @param tensorDesc Pointer to a tensor descriptor type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateTensorDescriptor(miopenTensorDescriptor_t* tensorDesc); - -/*! @brief Set shape of 4D tensor - * - * Interface for setting 4-D tensor shape. MIOpen currently implements NCHW and NHWC layout. - * - * @param tensorDesc Tensor descriptor (input/output) - * @param dataType MIOpen datatype (input) - * @param n Mini-batch size (input) - * @param c Number of channels (input) - * @param h Data height dimension size (input) - * @param w Data width dimension size (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSet4dTensorDescriptor( - miopenTensorDescriptor_t tensorDesc, miopenDataType_t dataType, int n, int c, int h, int w); - -/*! @brief Set shape of ND tensor with specific layout - * - * Interface for setting N-D packed tensor shape. This interface support NHWC, NCHW, NCHWc*, CHWNc* - * @param tensorDesc Tensor descriptor (input/output) - * @param dataType MIOpen datatype (input) - * @param tensorLayout Tensor layout (input) - * @param lens Tensor dimensions (input) - * @param num_lens Tensor dimension size (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetNdTensorDescriptorWithLayout(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t dataType, - miopenTensorLayout_t tensorLayout, - const int* lens, - int num_lens); -/*! @brief Set shape and stride of 4D tensor - * - * Interface for setting 4-D tensor shape and stride. It allows to create the non-packed tensor. - * A non-packed tensor refers to the tensor where the elements are not compressed or packed in any - * specific way. Each element in the tensor is stored individually, and there is no special - * compression applied to the storage. - * - * @param tensorDesc Tensor descriptor (input/output) - * @param dataType MIOpen datatype (input) - * @param n Mini-batch size (input) - * @param c Number of channels (input) - * @param h Data height dimension size (input) - * @param w Data width dimension size (input) - * @param nStride Mini-batch dimension stride (input) - * @param cStride Channel dimension stride (input) - * @param hStride Height dimension stride (input) - * @param wStride Width dimension stride (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSet4dTensorDescriptorEx(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t dataType, - int n, - int c, - int h, - int w, - int nStride, - int cStride, - int hStride, - int wStride); - -/*! @brief Get the details of the tensor descriptor - * - * Interface to query the 4-D tensor shape. - * - * @param tensorDesc Tensor descriptor (input) - * @param dataType MIOpen datatype (output) - * @param n Mini-batch size (output) - * @param c Number of channels (output) - * @param h Data height dimension size (output) - * @param w Data width dimension size (output) - * @param nStride Mini-batch dimension stride (output) - * @param cStride Channel dimension stride (output) - * @param hStride Height dimension stride (output) - * @param wStride Width dimension stride (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGet4dTensorDescriptor(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t* dataType, - int* n, - int* c, - int* h, - int* w, - int* nStride, - int* cStride, - int* hStride, - int* wStride); - -/*! @brief Set shape of N-dimensional tensor - * - * Interface for setting non-packed tensor shape. - * @param tensorDesc Tensor descriptor (input/output) - * @param dataType MIOpen datatype (input) - * @param nbDims Number of dimensions in the dimsA array (input) - * @param dimsA Array containing the size of dimensions (input) - * @param stridesA Array containing the size of stride (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetTensorDescriptor(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t dataType, - int nbDims, - const int* dimsA, - const int* stridesA); - -#ifdef MIOPEN_BETA_API -/*! @copydoc miopenSetTensorDescriptor() - */ -MIOPEN_EXPORT miopenStatus_t miopenSetTensorDescriptorV2(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t dataType, - int nbDims, - const size_t* dimsA, - const size_t* stridesA); -#endif - -#ifdef MIOPEN_BETA_API -/*! @brief Set the tensor cast type - * - * For tensors where the cast_type attribute is set, the tensor elements would be converted to the - * target type before the target operation is applied. Currently, only supported for convolution - * operations targeting the FP8 datatype - * - * @param tensorDesc Tensor descriptor type (input) - * @param cast_type MIOpen datatype (input) - */ -MIOPEN_EXPORT miopenStatus_t miopenSetTensorCastType(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t cast_type); -#endif - -/*! @brief Set shape of N-dimensional tensor - * - * Interface for querying tensor size. MIOpen has support for 1, 2, 3, 4, 5 dimensional tensor of - * layout. - * @param tensorDesc Tensor descriptor (input) - * @param size number of elements in tensor described by the descriptor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetTensorDescriptorSize(miopenTensorDescriptor_t tensorDesc, - int* size); - -/*! @brief Get the details of the N-dimensional tensor descriptor. - * - * @param tensorDesc Tensor descriptor (input) - * @param dataType MIOpen datatype (output) - * @param dimsA Array containing the size of dimensions (output) - * @param stridesA Array containing the size of stride (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetTensorDescriptor(miopenTensorDescriptor_t tensorDesc, - miopenDataType_t* dataType, - int* dimsA, - int* stridesA); - -/*! @brief Destroys the tensor descriptor - * - * @param tensorDesc Tensor descriptor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyTensorDescriptor(miopenTensorDescriptor_t tensorDesc); - -/*! @brief Create a Tensor Descriptor for sequence data - * - * API for creating an uninitialized sequence data tensor descriptor. - * @param tensorDesc Pointer to a sequence data tensor descriptor type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateSeqTensorDescriptor(miopenSeqTensorDescriptor_t* tensorDesc); - -/*! @brief Destroys the sequence data tensor descriptor - * - * @param tensorDesc Tensor descriptor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenDestroySeqTensorDescriptor(miopenSeqTensorDescriptor_t tensorDesc); - -/*! @brief Execute element-wise tensor operations - * - * This function implements: \f$ C = op ( alpha1[0] * A, alpha2[0] * B ) + beta[0] * C \f$ - * - * For Forward Bias one can also use, miopenConvolutionForwardBias() - * - * @param handle MIOpen handle (input) - * @param tensorOp Operation from miopenTensorOp_t (input) - * @param alpha1 Tensor A's floating point scaling factor, allocated on the host (input) - * @param aDesc Tensor descriptor for tensor A (input) - * @param A Tensor A (input) - * @param alpha2 Tensor B's floating point scaling factor, allocated on the host (input) - * @param bDesc Tensor descriptor for tensor B (input) - * @param B Tensor B (input) - * @param beta Tensor C's floating point scaling factor, allocated on the host (input) - * @param cDesc Tensor descriptor for tensor C (input) - * @param C Tensor C (input and output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenOpTensor(miopenHandle_t handle, - miopenTensorOp_t tensorOp, - const void* alpha1, - const miopenTensorDescriptor_t aDesc, - const void* A, - const void* alpha2, - const miopenTensorDescriptor_t bDesc, - const void* B, - const void* beta, - const miopenTensorDescriptor_t cDesc, - void* C); - -/*! @brief Fills a tensor with a single value. - * - * Supported datatypes are fp32, fp16, and bfp16 - * - * @param handle MIOpen handle (input) - * @param yDesc Tensor descriptor for tensor y (input) - * @param y Tensor y (input) - * @param alpha Pointer to fill value (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetTensor(miopenHandle_t handle, - const miopenTensorDescriptor_t yDesc, - void* y, - const void* alpha); - -/*! @brief Scales all elements in a tensor by a single value. - * - * Supported datatypes are fp32 and fp16 - * - * @param handle MIOpen handle (input) - * @param yDesc Tensor descriptor for tensor y (input) - * @param y Tensor y (input and output) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenScaleTensor(miopenHandle_t handle, - const miopenTensorDescriptor_t yDesc, - void* y, - const void* alpha); - -/*! @brief Returns number of bytes associated with tensor descriptor - * - * @param tensorDesc Tensor descriptor (input) - * @param numBytes Number of bytes associated with tensor descriptor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetTensorNumBytes(miopenTensorDescriptor_t tensorDesc, - size_t* numBytes); - -/*! @brief Copies one tensor to another tensor with a different layout/scale. - * - * This function implements: - * 1. \f$ Y = alpha * X + beta * Y \f$ for fp32 and fp16 datatype - * 2. Vectorize/de-vectorize along channel dimension C for int8 datatype - * - * Currently this is used for transforming from int8 to int8x4 vector datatypes - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Source Tensor descriptor for tensor x (input) - * @param x Source Tensor x (input) - * @param beta Floating point scaling factor, allocated on the host (input) - * @param yDesc Destination Tensor descriptor for tensor y (input) - * @param y Destination Tensor y (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenTransformTensor(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y); - -/** @} */ -// CLOSEOUT TENSOR DOXYGEN GROUP - -/** @addtogroup convolutions - * - * @{ - */ - -/*! @brief Creates a convolution layer descriptor - * - * @param convDesc Convolution layer descriptor - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateConvolutionDescriptor(miopenConvolutionDescriptor_t* convDesc); - -/*! @brief Creates a 2-D convolution layer descriptor - * - * For group/depthwise convolution dilation height and width, only a dilation value of 1 is - * supported. - * - * @param convDesc Convolution layer descriptor (output) - * @param c_mode Convolutional mode (input) - * @param pad_h Height input data padding (input) - * @param pad_w Width input data padding (input) - * @param stride_h Stride for the height of input data (input) - * @param stride_w Stride for the width of input data (input) - * @param dilation_h Dilation height (input) - * @param dilation_w Dilation width (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenInitConvolutionDescriptor(miopenConvolutionDescriptor_t convDesc, - miopenConvolutionMode_t c_mode, - int pad_h, - int pad_w, - int stride_h, - int stride_w, - int dilation_h, - int dilation_w); - -/*! @brief Creates a N-dimensional convolution layer descriptor - * - * @param convDesc Convolution layer descriptor (output) - * @param spatialDim Convolutional spatial dimension (input) - * @param padA Array of input data padding (input) - * @param strideA Array of convolution stride (input) - * @param dilationA Array of convolution dilation (input) - * @param c_mode Convolutional mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenInitConvolutionNdDescriptor(miopenConvolutionDescriptor_t convDesc, - int spatialDim, - const int* padA, - const int* strideA, - const int* dilationA, - miopenConvolutionMode_t c_mode); - -/*! @brief Retrieves the spatial dimension of a convolution layer descriptor - * - * @param convDesc Convolution layer descriptor (input) - * @param spatialDim Spatial dimension of convolution descriptor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetConvolutionSpatialDim(miopenConvolutionDescriptor_t convDesc, - int* spatialDim); - -/*! @brief Retrieves a 2-D convolution layer descriptor's details - * - * For group/depthwise convolution dilation height and width, only a dilation value of 1 is - * supported. - * - * @param convDesc Convolution layer descriptor (input) - * @param c_mode Convolutional mode (output) - * @param pad_h Height input data padding (output) - * @param pad_w Width input data padding (output) - * @param stride_h Stride for the height of input data (output) - * @param stride_w Stride for the width of input data (output) - * @param dilation_h Dilation height (output) - * @param dilation_w Dilation width (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetConvolutionDescriptor(miopenConvolutionDescriptor_t convDesc, - miopenConvolutionMode_t* c_mode, - int* pad_h, - int* pad_w, - int* stride_h, - int* stride_w, - int* dilation_h, - int* dilation_w); - -/*! @brief Retrieves a N-dimensional convolution layer descriptor's details - * - * @param convDesc Convolution layer descriptor (input) - * @param requestedSpatialDim Expected convolution spatial dimension (intput) - * @param spatialDim Convolutional spatial dimension (output) - * @param padA Array of input data padding (output) - * @param strideA Array of convolution stride (output) - * @param dilationA Array of convolution dilation (output) - * @param c_mode Convolutional mode (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetConvolutionNdDescriptor(miopenConvolutionDescriptor_t convDesc, - int requestedSpatialDim, - int* spatialDim, - int* padA, - int* strideA, - int* dilationA, - miopenConvolutionMode_t* c_mode); - -/*! @brief Get the number of groups to be used in Group/Depthwise convolution - * - * @param convDesc Convolution layer descriptor (input) - * @param groupCount Pointer to number of groups in group/depthwise convolution (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetConvolutionGroupCount(miopenConvolutionDescriptor_t convDesc, - int* groupCount); - -/*! @brief Set the number of groups to be used in Group/Depthwise convolution - * - * Must be called before all computational APIs of group/depthwise convolution, it is preferable to - * call miopenInitConvolutionDescriptor() first, then miopenSetConvolutionGroupCount() to fully - * initialize group convolutions. Both Convolution Mode and Transpose Convolution Mode support - * group/depthwise convolution. To run depthwise convolution, set groupCount value equal to number - * of channels. - * - * @param convDesc Convolution layer descriptor (output) - * @param groupCount number of groups, in depthwise conv using filter_number/channel_multiplier - * (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetConvolutionGroupCount(miopenConvolutionDescriptor_t convDesc, - int groupCount); - -/*! @brief Set the output padding to be used in 2-D Transpose convolution - * - * This function is optional for initialization of Transpose convolution. If applicable, it must be - * called before all computational APIs of Transpose convolution. It is preferable to call - * miopenInitConvolutionDescriptor() first, then miopenSetTransposeConvOutputPadding() to fully - * initialize transpose convolutions. - * - * @param convDesc Convolution layer descriptor (output) - * @param adj_h output padding for the height of output data (input) - * @param adj_w output padding for the width of output data (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetTransposeConvOutputPadding(miopenConvolutionDescriptor_t convDesc, int adj_h, int adj_w); - -/*! @brief Set the output padding to be used in N-dimensional Transpose convolution - * - * This function is optional for initialization of Transpose convolution. If applicable, it must be - * called before all computational APIs of Transpose convolution. It is preferable to call - * miopenInitConvolutionNdDescriptor() first, then miopenSetTransposeConvNdOutputPadding() to fully - * initialize transpose convolutions. Currently, 2-D and 3-D convolutions are supported. - * - * @param convDesc Convolution layer descriptor (output) - * @param spatialDim Convolutional spatial dimension (input) - * @param adjA array of output padding for output data (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetTransposeConvNdOutputPadding( - miopenConvolutionDescriptor_t convDesc, int spatialDim, const int* adjA); - -/*! @brief Get the shape of a resulting 4-D tensor from a 2-D convolution - * - * This function returns the dimensions of the resulting 4D tensor of a 2D - * convolution, given the convolution descriptor, the input tensor descriptor - * and the filter descriptor. This function can help to setup the output tensor - * and allocate the proper amount of memory prior to launch the actual - * convolution. - * - * @param convDesc Convolution layer descriptor (input) - * @param inputTensorDesc Input data tensor descriptor (input) - * @param filterDesc Weight descriptor (input) - * @param n Mini-batch size (output) - * @param c Number of channels (output) - * @param h Data height dimension size (output) - * @param w Data width dimension size (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetConvolutionForwardOutputDim(miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t inputTensorDesc, - const miopenTensorDescriptor_t filterDesc, - int* n, - int* c, - int* h, - int* w); - -/*! @brief Get the shape of a resulting N-dimensional tensor from a (N-2)-dimensional convolution - * - * This function returns the dimensions of the resulting N-dimensional tensor of a (N-2)-dimensional - * convolution, given the convolution descriptor, the input tensor descriptor - * and the filter descriptor. It is used to setup the output tensor descriptor prior to executing - * the convolution layer. - * - * @param convDesc Convolution layer descriptor (input) - * @param inputTensorDesc Input data tensor descriptor (input) - * @param filterDesc Weight descriptor (input) - * @param nDim Pointer to Output data tensor dimension (output) - * @param outputTensorDimA Array of Output data tensor length (output) - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetConvolutionNdForwardOutputDim(miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t inputTensorDesc, - const miopenTensorDescriptor_t filterDesc, - int* nDim, - int* outputTensorDimA); - -/*! @brief Destroys the tensor descriptor object - * - * @param convDesc Convolution tensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenDestroyConvolutionDescriptor(miopenConvolutionDescriptor_t convDesc); - -/*! @brief Set the attribute of the convolution descriptor - * - * @param convDesc Convolution layer descriptor (input) - * @param attr Attribute of this convolution to set (input) - * @param value Value of this attribute (input) - */ -MIOPEN_EXPORT miopenStatus_t miopenSetConvolutionAttribute(miopenConvolutionDescriptor_t convDesc, - const miopenConvolutionAttrib_t attr, - int value); - -/*! @brief Get the attribute of the convolution descriptor - * - * @param convDesc Convolution layer descriptor (input) - * @param attr Attribute of this convolution to get (input) - * @param value Value of this attribute (output) - */ -MIOPEN_EXPORT miopenStatus_t miopenGetConvolutionAttribute(miopenConvolutionDescriptor_t convDesc, - const miopenConvolutionAttrib_t attr, - int* value); - -/*! @enum miopenConvFwdAlgorithm_t - * Convolutional algorithm mode for forward propagation. MIOpen use cross-correlation for its - * convolution implementation. - */ -typedef enum -{ - miopenConvolutionFwdAlgoGEMM = 0, /*!< GEMM variant */ - miopenConvolutionFwdAlgoDirect = 1, /*!< Direct convolutions */ - miopenConvolutionFwdAlgoFFT = 2, /*!< Fast Fourier Transform indirect convolutions */ - miopenConvolutionFwdAlgoWinograd = 3, /*!< Winograd indirect convolutions */ - miopenConvolutionFwdAlgoImplicitGEMM = 5, /*!< Implicit GEMM convolutions */ -} miopenConvFwdAlgorithm_t; - -/*! @enum miopenConvBwdWeightsAlgorithm_t - * Convolutional algorithm mode for back propagation on weights. - */ -typedef enum -{ - miopenConvolutionBwdWeightsAlgoGEMM = 0, /*!< GEMM variant */ - miopenConvolutionBwdWeightsAlgoDirect = 1, /*!< Direct convolution algorithm */ - miopenConvolutionBwdWeightsAlgoWinograd = 3, /*!< Winograd convolutions */ - miopenConvolutionBwdWeightsAlgoImplicitGEMM = 5, /*!< Implicit GEMM convolutions */ -} miopenConvBwdWeightsAlgorithm_t; - -/*! @enum miopenConvBwdDataAlgorithm_t - * Convolutional algorithm mode for back propagation on data. - */ -typedef enum -{ - miopenConvolutionBwdDataAlgoGEMM = 0, /*!< GEMM variant */ - miopenConvolutionBwdDataAlgoDirect = 1, /*!< Direct convolutions */ - miopenConvolutionBwdDataAlgoFFT = 2, /*!< Fast Fourier Transform indirect convolutions */ - miopenConvolutionBwdDataAlgoWinograd = 3, /*!< Winograd indirect convolutions */ - miopenTransposeBwdDataAlgoGEMM = - 4, /*!< Deprecated Transpose GEMM variant legacy, ToBe Removed */ - miopenConvolutionBwdDataAlgoImplicitGEMM = 5, /*!< Implicit GEMM convolutions */ -} miopenConvBwdDataAlgorithm_t; - -/*! @enum miopenConvAlgorithm_t - * Top-level convolutional algorithm mode - */ -typedef enum -{ - miopenConvolutionAlgoGEMM = 0, /*!< GEMM variant */ - miopenConvolutionAlgoDirect = 1, /*!< Direct convolutions */ - miopenConvolutionAlgoFFT = 2, /*!< Fast Fourier Transform indirect convolutions */ - miopenConvolutionAlgoWinograd = 3, /*!< Winograd indirect convolutions */ - miopenConvolutionAlgoImplicitGEMM = 5, /*!< Implicit GEMM convolutions */ -} miopenConvAlgorithm_t; - -/*! @brief Perf struct for forward, backward filter, or backward data algorithms - * - * Contains the union to hold the selected convolution algorithm for forward, or backwards layers, - * and also contains the time it took to run the algorithm and the workspace required to run the - * algorithm. The workspace in this structure can be used when executing the convolution layer. - */ -typedef struct -{ - union - { - miopenConvFwdAlgorithm_t fwd_algo; /*!< Forward convolution algorithm enum selection */ - miopenConvBwdWeightsAlgorithm_t bwd_weights_algo; /*!< Back propagation on weights - convolution algorithm enum selection */ - miopenConvBwdDataAlgorithm_t - bwd_data_algo; /*!< Back propagation on data convolution algorithm enum selection */ - }; - - float time; /*!< Time to exectued the selected algorithm represented in the union */ - size_t memory; /*!< Workspace required to run the selected algorithm represented in the union */ - -} miopenConvAlgoPerf_t; - -/*! @brief Performance struct for forward, backward filter, or backward data algorithms in - * immediate mode - * - * Contains a 64-bit integer identifying the solution and the algorithm for the solution, - * as well as the runtime, workspace size and a boolean flag indicating whether the returned - * solution is a heuristic or resulting from an actual run - * - */ -typedef struct -{ - float time; /*!< Represents the approximate time required to execute this solution on the GPU, - in milliseconds. This value may either be based on an acutal kernel run or an - estimate based on a heuristic.*/ - size_t workspace_size; /*!< Workspace required to run the selected algorithm represented in the - union */ - uint64_t solution_id; /*!< Identifier for the returned solution */ - miopenConvAlgorithm_t algorithm; /*!< The algorithm used to compute the solution */ - -} miopenConvSolution_t; - -/*! @brief Query the maximum number of solutions applicable for the given input/output and weights - * tensor descriptor for Convolution in the Forward direction. - * - * This call returns the maximum number of applicable solutions for a forward convolution problem. - * The \c solutionCount returned may be used to allocate the memory required for the - * \c miopenConvAlgoPerf_t which is required by miopenConvolutionGetSolution API calls. - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param solutionCount Pointer to memory to return number of applicable solutions (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardGetSolutionCount(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - size_t* solutionCount); - -/*! @brief Query the applicable solutions for a convolution configuration described by - * input, output and convolution descriptors. - * - * The returned solutions array is sorted in the order of decreasing performance. The returned - * solutions - * might be based - * on heuristics and for more consistent performance results the user the advised to run the Find - * step. - * The maximum length of the solutions array may be queried using - * miopenConvolutionForwardGetSolutionCount - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param maxSolutionCount The size of the solutions array passed in below (input) - * @param solutionCount The size of the solutions array returned (output) - * @param solutions A pointer to an array of type miopenConvSolution_t allocated by the user, - * filled in by MIOpen with applicable solutions. (output) - * @return miopenStatus_t - * - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardGetSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - const size_t maxSolutionCount, - size_t* solutionCount, - miopenConvSolution_t* solutions); - -/*! @brief Returns the workspace size required for a particular solution id. - * - * This is an optional call for users who may have serialized the solution id and just need the - * workspace - * size for it. The same information is returned by the miopenConvolutionForwardGetSolution as part - * of the - * miopenConvSolution_t struct. - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param solution_id ID of the solution for which workspace size is required (input) - * @param workSpaceSize The size of the workspace (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardGetSolutionWorkspaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - const uint64_t solution_id, - size_t* workSpaceSize); - -/*! @brief Compiles the solution provided by the user, this solution may be acquired by the - * miopenConvolutionForwardGetSolution API call above. - * Compiling the solution ensures that the first API call to miopenConvolutionForwardImmediate - * does - * not cause a compile. - * - * This is an optional step and may be skipped if a slow first miopenConvolutionForwardImmediate - * invocation is acceptable. - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardCompileSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - const uint64_t solution_id); - -/*! @brief Executes the Forward convolution operation based on the provided solution ID. - * - * Supported datatypes are fp32, fp16, bfp16, and int8 - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param x Data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param workSpace Workspace tensor (input) - * @param workSpaceSize Size of the memory in bytes pointed to by workSpace above - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardImmediate(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - void* y, - void* workSpace, - size_t workSpaceSize, - const uint64_t solution_id); - -/*! @brief Query the maximum number of solutions applicable for the given input/output and weights - * tensor descriptor for backward Convolution w-r-t Data. - * - * This call returns the maximum number of applicable solutions for a the convolution problem, the - * number - * returned may be used to allocate the memory required for the miopenConvAlgoPert2_t which is - * required - * by miopenConvolutionBackwardDataGetSolution API calls. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param solutionCount Pointer to memory to return number of applicable solutions (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataGetSolutionCount(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - size_t* solutionCount); - -/*! @brief Query the applicable solutions for a backward convolution w-r-t data as described by - * input, output and convolution descriptors. - * - * The returned solutions array is sorted in the order of decreasing performance. The returned - * solutions - * ns - * might be based - * on heuristics and for more consistent performance results the user the advised to run the Find - * step. - * The maximum length of the solutions array may be queried using - * miopenConvolutionBackwardDataGetSolutionCount - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param maxSolutionCount The size of the solutions array passed in below (input) - * @param solutionCount The size of the solutions array returned (output) - * @param solutions A pointer to an array of type miopenConvSolution_t allocated by the user, - * filled in by MIOpen with applicable solutions. (output) - * @return miopenStatus_t - * - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataGetSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - const size_t maxSolutionCount, - size_t* solutionCount, - miopenConvSolution_t* solutions); - -/*! @brief Returns the workspace size required for a particular solution id. - * - * This is an optional call for users who may have serialized the solution id and just need the - * workspace - * size for it. The same information is returned by the miopenConvolutionBackwardDataGetSolution as - * part of the - * miopenConvSolution_t struct. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param solution_id ID of the solution for which workspace size is required (input) - * @param workSpaceSize The size of the workspace (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataGetSolutionWorkspaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - const uint64_t solution_id, - size_t* workSpaceSize); - -/*! @brief Compiles the solution provided by the user, this solution may be acquired by the - * miopenConvolutionBackwardDataGetSolution API call above. - * Compiling the solution ensures that the first API call to - * miopenConvolutionBackwardDataImmediate - * does not cause a compile. - * - * This is an optional step and may be skipped if a slow first - * miopenConvolutionBackwardDataImmediate - * invocation is acceptable. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataCompileSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - const uint64_t solution_id); - -/*! @brief Executes the Backward convolution w-r-t data operation based on the provided solution - * ID. - * - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dx Data delta tensor dx (output) - * @param workSpace Workspace tensor (input) - * @param workSpaceSize Size in bytes of the workspace memory pointed to by workSpace - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataImmediate(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - void* dx, - void* workSpace, - size_t workSpaceSize, - const uint64_t solution_id); - -/*! @brief Query the maximum number of solutions applicable for the given input/output and weights - * tensor descriptor for backward Convolution w-r-t Weights. - * - * This call returns the maximum number of applicable solutions for a the convolution problem, the - * number - * returned may be used to allocate the memory required for the miopenConvAlgoPert2_t which is - * required - * by miopenConvolutionBackwardWeightsGetSolution API calls. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param solutionCount Pointer to memory to return number of applicable solutions (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeightsGetSolutionCount(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - size_t* solutionCount); - -/*! @brief Query the applicable solutions for a backward convolution w-r-t weights as described by - * input, output and convolution descriptors. - * - * The returned solutions array is sorted in the order of decreasing performance. The returned - * solutions - * might be based - * on heuristics and for more consistent performance results the user the advised to run the Find - * step. - * The maximum length of the solutions array may be queried using - * miopenConvolutionBackwardWeightsGetSolutionCount - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param maxSolutionCount The size of the solutions array passed in below (input) - * @param solutionCount The size of the solutions array returned (output) - * @param solutions A pointer to an array of type miopenConvSolution_t allocated by the user, - * filled in by MIOpen with applicable solutions. (output) - * @return miopenStatus_t - * - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeightsGetSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - const size_t maxSolutionCount, - size_t* solutionCount, - miopenConvSolution_t* solutions); - -/*! @brief Returns the workspace size required for a particular solution id. - * - * This is an optional call for users who may have serialized the solution id and just need the - * workspace - * size for it. The same information is returned by the miopenConvolutionBackwardWeightsGetSolution - * as part of the - * miopenConvSolution_t struct. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param solution_id ID of the solution for which workspace size is required (input) - * @param workSpaceSize The size of the workspace (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenConvolutionBackwardWeightsGetSolutionWorkspaceSize( - miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - const uint64_t solution_id, - size_t* workSpaceSize); - -/*! @brief Compiles the solution provided by the user, this solution may be acquired by the - * miopenConvolutionBackwardWeightsGetSolution API call above. - * Compiling the solution ensures that the first API call to - * miopenConvolutionBackwardWeightsImmediate - * does not cause a compile. - * - * This is an optional step and may be skipped if a slow first - * miopenConvolutionBackwardWeightsImmediate invocation is acceptable. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeightsCompileSolution(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - const uint64_t solution_id); - -/*! @brief Executes the Backward convolution w-r-t weights operation based on the provided solution - * ID. - * - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param x Data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param dw Weights delta tensor dw (output) - * @param workSpace Workspace tensor (input) - * @param workSpaceSize Size in bytes of the memory passed in, pointed to by workSpace pointer - * above - * @param solution_id ID of the solution to be compiled, as chosen by the user - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeightsImmediate(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - void* dw, - void* workSpace, - size_t workSpaceSize, - const uint64_t solution_id); - -/*! @brief Query the workspace size required for a forward convolution algorithm. - * - * For given tensor and convolution descriptors, this function calculates and returns the minimum - * size of the workspace that must be provided to miopenFindConvolutionForwardAlgorithm() in order - * for the latter to find the best candidate from the available forward data convolution algorithms. - * - * WARNING: Providing smaller workspace may result in the selection of a slow convolution - * algorithm, and therefore affect library performance. - * - * It should be assumed that the required workspace size is different for each convolution - * configuration. Therefore, typically this function should be called at least once for each - * convolution configuration used. - * - * Since the convolution configuration is determined by tensor and convolution descriptors, the user - * should ensure that all descriptors contain complete information. For example, if Group/Depthwise - * convolution mode is used, then miopenSetConvolutionGroupCount() should be called before running - * this, and so on. - * - * @param handle MIOpen handle (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param workSpaceSize Pointer to memory to return size in bytes (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionForwardGetWorkSpaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t wDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - size_t* workSpaceSize); - -/*! @brief Search and run the forward convolutional algorithms and return a list of kernel times. - * - * This function attempts all MIOpen forward convolution algorithms based on - * the input configuration, and outputs performance metrics to a - * user-allocated array of type miopenConvAlgoPerf_t. These metrics are written - * in a sorted fashion where the first element has the lowest compute time. - * Users can chose the top-most algorithm if they only care about the fastest - * algorithm. - * - * This function is mandatory before using miopenConvolutionForward(). In order - * to execute this function, miopenConvolutionForwardGetWorkSpaceSize() must be - * run to determine the required memory for this search. - * - * * If exhaustiveSearch == 0, MIOpen will look for the first kernel with a configuration match. If - * a configuration match is not found, a default configuration will be returned. - * - * * If exhaustiveSearch == 1, MIOpen will look for the best kernel for the provided configuration. - * If a match is not found, an exhaustive search is performed by running individual algorithms. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param requestAlgoCount Number of algorithms to return kernel times (input) - * @param returnedAlgoCount Pointer to number of algorithms returned (output) - * @param perfResults Pointer to union of best algorithm for forward and backwards (input) - * @param workSpace Pointer to workspace buffer (input). - * @param workSpaceSize Size in bytes of the workspace buffer (input). - * The buffer must be allocated on the device by the caller. - * The size of the buffer should be determined by calling - * miopenConvolutionForwardGetWorkSpaceSize(), see its - * documentation for details. - * @param exhaustiveSearch A boolean to toggle a full search of all algorithms - * and configurations (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFindConvolutionForwardAlgorithm(miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t yDesc, - void* y, - const int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - void* workSpace, - size_t workSpaceSize, - bool exhaustiveSearch); - -/*! @brief Execute a forward convolution layer - * - * Runs the forward convolution layer based on the selected algorithm. The function - * miopenFindConvolutionForwardAlgorithm() must have been executed previously to - * determine the required memory needed for the workspace and the best convolutional algorithm. - * The scaling parameter alpha (float) and shift parameter beta (float) are only supported for - * alpha = 1 and beta = 0 in 2D. In 3D, these parameters can take other values. - * - * The forward convolution is designed to accommodate both packed and non-packed tensor strides for - * multiple data types and dimensions across various platforms. This flexibility ensures optimal - * performance in handling diverse computational scenarios. To configure tensor parameters, - * including strides, users can utilize the APIs miopenSetTensorDescriptor() and - * miopenGetTensorDescriptor(). These APIs empower developers to seamlessly set and retrieve tensor - * information, facilitating a more intuitive and efficient workflow. The tensor strides are - * non-packed by default. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (inputs) - * @param convDesc Convolution layer descriptor (inputs) - * @param algo Algorithm selected (inputs) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param workSpace Pointer to workspace required (input) - * @param workSpaceSize Size in bytes of the memory determined by the find step (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenConvolutionForward(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - miopenConvFwdAlgorithm_t algo, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y, - void* workSpace, - size_t workSpaceSize); - -/*! @brief Calculate element-wise scale and shift of a tensor via a bias tensor - * - * This function applies an element-wise bias to a data tensor from an input bias tensor. - * The scaling parameter alpha (float) and shift parameter beta (float) are only supported for - * alpha = 1 and beta = 0. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param bDesc Tensor descriptor for bias tensor b (input) - * @param b Bias tensor b (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for data tensor y (input) - * @param y Data tensor y (input and output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenConvolutionForwardBias(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t bDesc, - const void* b, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y); - -/*! @brief Query the workspace size required for a backward data convolution algorithm. - * - * For given tensor and convolution descriptors, this function calculates and returns the minimum - * size of the workspace that must be provided to miopenFindConvolutionBackwardDataAlgorithm() in - * order for the latter to find the best candidate from the available backward data convolution - * algorithms. - * - * WARNING: Providing smaller workspace may result in the selection of a slow convolution - * algorithm, and therefore affect library performance. - * - * It should be assumed that the required workspace size is different for each convolution - * configuration. Therefore, typically this function should be called at least once for each - * convolution configuration used. - * - * Since the convolution configuration is determined by tensor and convolution descriptors, the user - * should ensure that all descriptors contain complete information. For example, if Group/Depthwise - * convolution mode is used, then miopenSetConvolutionGroupCount() should be called before running - * this, and so on. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param workSpaceSize Size in bytes of the memory required (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardDataGetWorkSpaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t wDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - size_t* workSpaceSize); - -/*! @brief Search and run the backwards data convolution algorithms and return a list of kernel - * times. - * - * This function attempts all MIOpen backward data convolution algorithms, and outputs the - * performance metrics to a user-allocated array of type miopenConvAlgoPerf_t. - * These metrics are written in sorted fashion where the first element has the lowest compute time. - * This function is mandatory before using backwards convolutions. Users can chose the top-most - * algorithm if they only care about the fastest algorithm. - * - * This function is mandatory before using miopenConvolutionBackwardData(). In order to - * execute this function, miopenConvolutionBackwardsDataGetWorkSpaceSize() must be run to determine - * the required memory for this search. - * - * * If exhaustiveSearch == 0, MIOpen will look for the first kernel with a configuration match. If - * a configuration match is not found, a default configuration will be returned. - * - * * If exhaustiveSearch == 1, MIOpen will look for the best kernel for the provided configuration. - * If a match is not found, an exhaustive search is performed by running individual algorithms. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dx Data delta tensor dx (input) - * @param requestAlgoCount Number of algorithms to return kernel times (input) - * @param returnedAlgoCount Pointer to number of algorithms returned (output) - * @param perfResults Pointer to union of best algorithm for forward and backwards (output) - * @param workSpace Pointer to workspace buffer (input). - * @param workSpaceSize Size in bytes of the workspace buffer (input). - * The buffer must be allocated on the device by the caller. - * The size of the buffer should be determined by calling - * miopenConvolutionBackwardDataGetWorkSpaceSize(), see its - * documentation for details. - * @param exhaustiveSearch A boolean to toggle a full search of all algorithms - * and configurations (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFindConvolutionBackwardDataAlgorithm(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - void* workSpace, - size_t workSpaceSize, - bool exhaustiveSearch); - -/*! @brief Execute a backward data convolution layer - * - * Runs the backward data convolution layer based on the selected algorithm. The function - * miopenFindConvolutionBackwardDataAlgorithm() must have been executed previously to - * determine the required memory needed for the workspace and the best convolutional - * algorithm. - * - * The backward data convolution is designed to accommodate both packed and non-packed tensor - * strides for multiple data types and dimensions across various platforms. This flexibility ensures - * optimal performance in handling diverse computational scenarios. To configure tensor parameters, - * including strides, users can utilize the APIs miopenSetTensorDescriptor() and - * miopenGetTensorDescriptor(). These APIs empower developers to seamlessly set and retrieve tensor - * information, facilitating a more intuitive and efficient workflow. The tensor strides are - * non-packed by default. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param algo Algorithm selected (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dx Data delta tensor dx (output) - * @param workSpace Pointer to workspace required for the search (input) - * @param workSpaceSize Size in bytes of the memory needed for find (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardData(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - miopenConvBwdDataAlgorithm_t algo, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx, - void* workSpace, - size_t workSpaceSize); - -/*! @brief Get the GPU memory required for the backward weights convolution algorithm. - * - * For given tensor and convolution descriptors, this function calculates and returns the minimum - * size of the workspace that must be provided to miopenFindConvolutionBackwardWeightsAlgorithm() in - * order for the latter to find the best candidate from the available backward weights convolution - * algorithms. - * - * WARNING: Providing smaller workspace may result in the selection of a slow convolution - * algorithm, and therefore affect library performance. - * - * It should be assumed that the required workspace size is different for each convolution - * configuration. Therefore, typically this function should be called at least once for each - * convolution configuration used. - * - * Since the convolution configuration is determined by tensor and convolution descriptors, the user - * should ensure that all descriptors contain complete information. For example, if Group/Depthwise - * convolution mode is used, then miopenSetConvolutionGroupCount() should be called before running - * this, and so on. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for output weights tensor dw (input) - * @param workSpaceSize Size in bytes of the memory required (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeightsGetWorkSpaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - size_t* workSpaceSize); - -/*! @brief Search and run the backwards weights convolutional algorithms and return a list of kernel - * times. - * - * This function attempts all MIOpen backward weights convolution algorithms, and outputs - * the performance metrics to a user-allocated array of type miopenConvAlgoPerf_t. These metrics are - * written in sorted fashion where the first element has the lowest compute time. - * This function is mandatory before using backwards weight convolutions. Users can chose the - * top-most algorithm if they only care about the fastest algorithm. - * - * This function is mandatory before using miopenConvolutionBackwardWeights(). In order to - * execute this function, miopenConvolutionBackwardsWeightsGetWorkSpaceSize() must be run to - * determine the required memory for this search. - * - * * If exhaustiveSearch == 0, MIOpen will look for the first kernel with a configuration match. If - * a configuration match is not found, a default configuration will be returned. - * - * * If exhaustiveSearch == 1, MIOpen will look for the best kernel for the provided configuration. - * If a match is not found, an exhaustive search is performed by running individual algorithms. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for output data tensor x (input) - * @param x Data delta tensor dx (input) - * @param convDesc Convolution layer descriptor (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param dw Weights delta tensor dw (input) - * @param requestAlgoCount Number of algorithms to return kernel times (input) - * @param returnedAlgoCount Pointer to number of algorithms returned (output) - * @param perfResults Pointer to union of best algorithm for forward and backwards (output) - * @param workSpace Pointer to workspace buffer (input). - * @param workSpaceSize Size in bytes of the workspace buffer (input). - * The buffer must be allocated on the device by the caller. - * The size of the buffer should be determined by calling - * miopenConvolutionBackwardWeightsGetWorkSpaceSize(), see its - * documentation for details. - * @param exhaustiveSearch A boolean to toggle a full search of all algorithms - * and configurations (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFindConvolutionBackwardWeightsAlgorithm(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t dwDesc, - void* dw, - const int requestAlgoCount, - int* returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - void* workSpace, - size_t workSpaceSize, - bool exhaustiveSearch); - -/*! @brief Execute a backward weights convolution layer - * - * Runs the backward weights convolution layer based on the selected algorithm. The function - * miopenFindConvolutionBackwardWeightsAlgorithm() must have - * been executed previously to determine the required memory needed for the workspace and the - * best convolutional algorithm. - * - * The backward weights convolution is designed to accommodate both packed and non-packed tensor - * strides for multiple data types and dimensions across various platforms. This flexibility ensures - * optimal performance in handling diverse computational scenarios. To configure tensor parameters, - * including strides, users can utilize the APIs miopenSetTensorDescriptor() and - * miopenGetTensorDescriptor(). These APIs empower developers to seamlessly set and retrieve tensor - * information, facilitating a more intuitive and efficient workflow. The tensor strides are - * non-packed by default. - * - * If using Group/Depthwise convolution mode, call miopenSetConvolutionGroupCount() before running - * this. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param dyDesc Tensor descriptor for data tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param x Data tensor x (input) - * @param convDesc Convolution layer descriptor (input) - * @param algo Algorithm selected (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dwDesc Tensor descriptor for weight tensor dw (input) - * @param dw Weights delta tensor dw (output) - * @param workSpace Pointer to workspace required for the search (input) - * @param workSpaceSize Size in bytes of the memory needed for find (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBackwardWeights(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenConvolutionDescriptor_t convDesc, - miopenConvBwdWeightsAlgorithm_t algo, - const void* beta, - const miopenTensorDescriptor_t dwDesc, - void* dw, - void* workSpace, - size_t workSpaceSize); - -/*! @brief Calculates the gradient with respect to the bias. - * - * Compute the convolution backwards gradient with respect to the bias tensor. - * The scaling parameter alpha (float) and shift parameter beta (float) are only supported for - * alpha = 1 and beta = 0. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dbDesc Tensor descriptor for input bias tensor db (input) - * @param db Bias delta tensor db (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenConvolutionBackwardBias(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const void* beta, - const miopenTensorDescriptor_t dbDesc, - void* db); - -/** @} */ -// CLOSEOUT CONVOLUTIONS DOXYGEN GROUP - -// Pooling APIs -/** @addtogroup pooling - * - * @{ - */ - -/*! @brief Creates a pooling layer descriptor - * - * @param poolDesc Pointer to a pooling layer descriptor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreatePoolingDescriptor(miopenPoolingDescriptor_t* poolDesc); - -/*! @brief Set index data type for pooling layer. The default indexing type is uint8_t. - * Users can set the index type to any of the miopenIndexType_t sizes; 8, 16, 32, or 64 bit - * unsigned integers. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param index_type Index type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetPoolingIndexType(miopenPoolingDescriptor_t poolDesc, - miopenIndexType_t index_type); - -/*! @brief Get the index data type for pooling layer. The index type to any of the - * miopenIndexType_t sizes; 8, 16, 32, or 64 bit unsigned integers. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param index_type Index type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetPoolingIndexType(miopenPoolingDescriptor_t poolDesc, - miopenIndexType_t* index_type); - -/*! @brief Set workspace index mode for pooling layer. The default mode is - * miopenPoolingWorkSpaceIndexMask. - * - * @param poolDesc Pointer to a pooling layer descriptor (input/output) - * @param workspace_index Workspace index mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetPoolingWorkSpaceIndexMode( - miopenPoolingDescriptor_t poolDesc, miopenPoolingWorkspaceIndexMode_t workspace_index); - -/*! @brief Get workspace index mode for pooling layer. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param workspace_index Workspace index mode (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetPoolingWorkSpaceIndexMode( - miopenPoolingDescriptor_t poolDesc, miopenPoolingWorkspaceIndexMode_t* workspace_index); - -/*! @brief Sets a 2-D pooling layer descriptor details. - * - * Sets the window shape, padding, and stride for a previously created 2-D pooling descriptor. - * - * @param poolDesc Pointer to a pooling layer descriptor (output) - * @param mode Pooling mode enum (input) - * @param windowHeight Input window height dimension (input) - * @param windowWidth Input window width dimension (input) - * @param pad_h Number of elements to pad height (input) - * @param pad_w Number of elements to pad width (input) - * @param stride_h Vertical stride (input) - * @param stride_w Horizontal stride (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSet2dPoolingDescriptor(miopenPoolingDescriptor_t poolDesc, - miopenPoolingMode_t mode, - int windowHeight, - int windowWidth, - int pad_h, - int pad_w, - int stride_h, - int stride_w); - -/*! @brief Gets a 2-D pooling layer descriptor details - * - * Gets the window shape, padding, and stride for a previously created 2-D pooling descriptor. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param mode Pooling mode enum (output) - * @param windowHeight Input window height dimension (output) - * @param windowWidth Input window width dimension (output) - * @param pad_h Number of elements to pad height (output) - * @param pad_w Number of elements to pad width (output) - * @param stride_h Vertical stride (output) - * @param stride_w Horizontal stride (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGet2dPoolingDescriptor(const miopenPoolingDescriptor_t poolDesc, - miopenPoolingMode_t* mode, - int* windowHeight, - int* windowWidth, - int* pad_h, - int* pad_w, - int* stride_h, - int* stride_w); - -/*! @brief Gets the shape of the output tensor for 2-D pooling - * - * Retrieve the tensor dimensions for the forward 2-D pooling. This call is required for - * the forward if the output dimensions are different than the input tensor - * dimensions. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param tensorDesc Input tensor descriptor (input) - * @param n Mini-batch dim (output) - * @param c Number of channels (output) - * @param h Heights of input map (output) - * @param w Width of input map (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetPoolingForwardOutputDim(const miopenPoolingDescriptor_t poolDesc, - const miopenTensorDescriptor_t tensorDesc, - int* n, - int* c, - int* h, - int* w); - -/*! @brief Set details of a N-D pooling layer descriptor - * - * Set the window shape, padding, and stride for a previously created N-D pooling descriptor. - * - * @param poolDesc Pointer to a pooling layer descriptor (input/output) - * @param mode Pooling mode enum (input) - * @param nbDims Dimension of the pooling (input) - * @param windowDimA Array of input window dimensions with length equal to or larger than - * dimsRequested (input) - * @param padA Array of number of elements to padding with length equal to or larger than - * dimsRequested (input) - * @param stridesA Array of stride parameter with length equal to or larger than dimsRequested - * (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetNdPoolingDescriptor(miopenPoolingDescriptor_t poolDesc, - const miopenPoolingMode_t mode, - int nbDims, - const int* windowDimA, - const int* padA, - const int* stridesA); - -/*! @brief Get details of a N-D pooling layer descriptor - * - * Get the window shape, padding, and stride for a previously created N-D pooling descriptor. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param nbDimsRequested Dimension of the expected pooling descriptor (input) - * @param mode Pooling mode enum (output) - * @param nbDims Actual dimension of the pooling descriptor (output) - * @param windowDimA Array of input window dimensions with length equal to or larger than - * dimsRequested (output) - * @param padA Array of number of elements to padding with length equal to or larger - * than dimsRequested (output) - * @param stridesA Array of stride parameter with length equal to or larger than - * dimsRequested (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetNdPoolingDescriptor(const miopenPoolingDescriptor_t poolDesc, - int nbDimsRequested, - miopenPoolingMode_t* mode, - int* nbDims, - int* windowDimA, - int* padA, - int* stridesA); - -/*! @brief Gets the shape of the output tensor for N-D pooling - * - * Retrieve the tensor dimensions for the forward N-D pooling. This call is required for - * the forward if the output dimensions are different than the input tensor - * dimensions. - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param tensorDesc Input tensor descriptor (input) - * @param dims Dimension of the pooling (input) - * @param tensorDimArr Array of tensor dimension (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetPoolingNdForwardOutputDim(const miopenPoolingDescriptor_t poolDesc, - const miopenTensorDescriptor_t tensorDesc, - int dims, - int* tensorDimArr); - -/*! @brief Get the amount of GPU memory required for pooling - * - * Retrieves the amount of workspace in bytes require for pooling. This call is required to - * determine the amount of GPU memory needed for the backwards pooling algorithms. For max- - * pooling, an assumption is that index data type is uint8_t, therefore the returned - * workspace size will be based on this assumption even if the user sets the index type with - * miopenSetPoolingIndexType(). - * - * @param yDesc Descriptor for pooling layer (input) - * @param workSpaceSize Pointer to workSpaceSize (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenPoolingGetWorkSpaceSize(const miopenTensorDescriptor_t yDesc, - size_t* workSpaceSize); - -/*! @brief Get the amount of GPU memory required for pooling - * - * Retrieves the amount of workspace in bytes require for pooling. This call is required to - * determine the amount of GPU memory needed for the backwards pooling algorithms. For max- - * pooling, there is no assumption on index data type. As the user can set the index datatype - * size using miopenSetPoolingIndexType(). - * - * @param poolDesc Pointer to a pooling layer descriptor (input) - * @param yDesc Descriptor for pooling layer (input) - * @param workSpaceSize Pointer to workSpaceSize (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenPoolingGetWorkSpaceSizeV2(const miopenPoolingDescriptor_t poolDesc, - const miopenTensorDescriptor_t yDesc, - size_t* workSpaceSize); - -/*! @brief Execute a forward pooling layer - * - * Runs forward pooling. miopenGetPoolingForwardOutputDim() should be called before - * miopenPoolingForward(). - * If the parameter do_backward == 0, then set workSpace = nullptr and workSpaceSize = 0. However, - * for back-propagation do_backwards must be set to 1 in miopenPoolingForward(). - * - * @param handle MIOpen handle (input) - * @param poolDesc Descriptor for pooling layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param do_backward Boolean to toggle save data in workspace for backwards pass (input) - * @param workSpace Pointer user allocated memory (input) - * @param workSpaceSize Size in bytes of the memory needed (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenPoolingForward(miopenHandle_t handle, - const miopenPoolingDescriptor_t poolDesc, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y, - bool do_backward, - void* workSpace, - size_t workSpaceSize); - -/*! @brief Execute a backward pooling layer - * - * Runs backward pooling. miopenPoolingGetWorkSpaceSize() must be called before - * miopenPoolingBackward() to determine the amount of workSpace to be allocated. - * - * @param handle MIOpen handle (input) - * @param poolDesc Descriptor for pooling layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for output data tensor x (input) - * @param x Data tensor x (output) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for tensor dx (input) - * @param dx Weights delta tensor dx (output) - * @param workSpace Pointer to user allocated workspace (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenPoolingBackward(miopenHandle_t handle, - const miopenPoolingDescriptor_t poolDesc, - const void* alpha, - const miopenTensorDescriptor_t yDesc, - const void* y, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx, - void* workSpace); - -/*! @brief Destroys the pooling descriptor object - * - * @param poolDesc Pooling tensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyPoolingDescriptor(miopenPoolingDescriptor_t poolDesc); - -/** @} */ -// CLOSEOUT POOLING DOXYGEN GROUP - -// LRN APIs -/** @addtogroup LRN - * - * @{ - */ -/*! @brief Creates a local response normalization (LRN) layer descriptor - * - * @param lrnDesc Pointer to a local response normalization layer descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateLRNDescriptor(miopenLRNDescriptor_t* lrnDesc); - -/*! @brief Sets a LRN layer descriptor details - * - * Sets all of the descriptor details for the LRN layer. The number of window elements lrnN is - * a diameter and always odd. - * - * @param lrnDesc Pointer to a LRN layer descriptor (output) - * @param mode LRN mode enum (input) - * @param lrnN Number of normalization window elements (input) - * @param lrnAlpha Scaling factor (input) - * @param lrnBeta Shift factor (input) - * @param lrnK K factor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetLRNDescriptor(const miopenLRNDescriptor_t lrnDesc, - miopenLRNMode_t mode, - unsigned int lrnN, - double lrnAlpha, - double lrnBeta, - double lrnK); - -/*! @brief Gets a LRN layer descriptor details - * - * Retrieve the LRN descriptor details. - * - * @param lrnDesc Pointer to a LRN layer descriptor (input) - * @param mode LRN mode enum (output) - * @param lrnN Number of normalization window elements (output) - * @param lrnAlpha Scaling factor (output) - * @param lrnBeta Shift factor (output) - * @param lrnK K factor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetLRNDescriptor(const miopenLRNDescriptor_t lrnDesc, - miopenLRNMode_t* mode, - unsigned int* lrnN, - double* lrnAlpha, - double* lrnBeta, - double* lrnK); - -/*! @brief Determine the workspace requirements. - * - * This function determines the GPU memory allocation required to execute the LRN layer based on the - * LRN descriptor. - * - * @param yDesc Pointer to a LRN layer descriptor (input) - * @param workSpaceSize Output variable for workspace size (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenLRNGetWorkSpaceSize(const miopenTensorDescriptor_t yDesc, - size_t* workSpaceSize); - -/*! @brief Execute a LRN forward layer - * - * Runs the forward layer normalization in the forward direction. If do_backward == 0, then - * set workSpace = nullptr and workSpaceSize = 0. However, if the user wishes to execute backwards, - * then they must set do_backwards = 1 in miopenLRNForward(). - * - * @param handle MIOpen handle (input) - * @param lrnDesc Descriptor for LRN layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param do_backward Boolean to toggle save data in workspace for backwards pass (input) - * @param workSpace Pointer user allocated memory (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenLRNForward(miopenHandle_t handle, - const miopenLRNDescriptor_t lrnDesc, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y, - bool do_backward, - void* workSpace); - -/*! @brief Execute a LRN backward layer - * - * @param handle MIOpen handle (input) - * @param lrnDesc Descriptor for LRN layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param yDesc Tensor descriptor for data input tensor y (input) - * @param y Data tensor y (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for output data tensor dx(input) - * @param dx Data delta tensor x (output) - * @param workSpace Pointer user allocated memory (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenLRNBackward(miopenHandle_t handle, - const miopenLRNDescriptor_t lrnDesc, - const void* alpha, - const miopenTensorDescriptor_t yDesc, - const void* y, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const void* workSpace); - -/*! @brief Destroys the LRN descriptor object - * - * @param lrnDesc LRN tensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyLRNDescriptor(miopenLRNDescriptor_t lrnDesc); - -/** @} */ -// CLOSEOUT LRN DOXYGEN GROUP - -#ifdef MIOPEN_BETA_API -// LayerNorm APIs -/** @addtogroup layernorm - * - * @{ - */ -/*! @brief Execute a layernorm forward layer - * - * @param handle MIOpen handle (input) - * @param mode LayerNorm mode (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param weight Data tensor weight (input) - * @param biasDesc Tensor descriptor for data input tensor bias (input) - * @param bias Data tensor bias (input) - * @param epsilon Value to stablize inverse variance calculation (input) - * @param normalized_dim Nomalized dimensions in the input array (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param meanDesc Tensor descriptor for output data tensor mean (input) - * @param mean Data tensor mean (output) - * @param rstdDesc Tensor descriptor for output data tensor rstd (input) - * @param rstd Data tensor rstd (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenLayerNormForward(miopenHandle_t handle, - miopenNormMode_t mode, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t weightDesc, - const void* weight, - const miopenTensorDescriptor_t biasDesc, - const void* bias, - const float epsilon, - const int32_t normalized_dim, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t meanDesc, - void* mean, - const miopenTensorDescriptor_t rstdDesc, - void* rstd); - -/** @} */ -// CLOSEOUT LAYERNORM DOXYGEN GROUP -#endif - -#ifdef MIOPEN_BETA_API -// Cat APIs -/** @addtogroup cat - * - * @{ - */ -/*! @brief Execute a cat forward layer - * - * @param handle MIOpen handle (input) - * @param xCount Number of input tensor x (input) - * @param xDescs Tensor descriptor of input tensor x (input) - * @param xs Source data tensor x (input) - * @param yDesc Tensor descriptor of output tensor y (input) - * @param y Data tensor y (output) - * @param dim Concatenation dimension (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCatForward(miopenHandle_t handle, - const int32_t xCount, - const miopenTensorDescriptor_t* xDescs, - const void* const* xs, - const miopenTensorDescriptor_t yDesc, - void* y, - const int32_t dim); - -/** @} */ -// CLOSEOUT CAT DOXYGEN GROUP -#endif - -// Batch-Normalization APIs -/** @addtogroup batchnorm - * - * @{ - */ - -/*! @brief Derive tensor for gamma and beta from input tensor descriptor - * - * This function takes the input tensor descriptor and outputs a derived tensor for the - * normalization scale (gamma) and shift (beta) tensors. - * - * For an input tensor NCHW and spatial mode, the output derived tensor is 1C11, while for - * per-activation the derived tensor is 1CHW. - * - * For an input tensor NCDHW and spatial mode, the output derived tensor is 1C111, while for - * per-activation the derived tensor is 1CDHW. - * - * @param derivedBnDesc Output derived tensor descriptor (output) - * @param xDesc Input tensor descriptor (input) - * @param bn_mode Batch Normalization mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDeriveBNTensorDescriptor(miopenTensorDescriptor_t derivedBnDesc, - const miopenTensorDescriptor_t xDesc, - miopenBatchNormMode_t bn_mode); - -/*! @brief Execute forward training layer for batch normalization - * - * Batch normalization pass for forward training pass. - * Takes in batch normalization mode bn_mode and input tensor x, output tensor y, bnBias and bnScale - * with their descriptor. - * - * If either resultSaveMean, or resultSaveInvVariance are null pointers then the values for the mean - * and inverse variance will not be used. - * - * Likewise, if either resultRunningMean, or resultRunningVariance are null pointers then the values - * for the running mean and variance will not be saved. - * Running averages and variances are scaled using an exponential averaging factor: \f[ - * \mu_{old} = \mu_{new}*factor + \mu_{old}*(1-factor) - * \f] - * where \f[ - * factor=1/(1+iteration) - * \f] - * - * @param handle MIOpen handle (input) - * @param bn_mode Batch normalization mode (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param bnScaleBiasMeanVarDesc Tensor descriptor for BN scaling, shifting, saved variance and - * mean (input) - * @param bnScale Batch norm scaling, gamma, tensor (input) - * @param bnBias Batch norm bias, beta, tensor (input) - * @param expAvgFactor Exponential averaging factor (input) - * @param resultRunningMean Running average saved for inference (output) - * @param resultRunningVariance Running variance saved for inference (output) - * @param epsilon Value to stablize inverse variance calculation (input) - * @param resultSaveMean Saved mini-batch mean for backwards pass (output) - * @param resultSaveInvVariance Saved mini-batch inverse variance for backwards pass (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenBatchNormalizationForwardTraining(miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - void* alpha, - void* beta, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc, - void* bnScale, - void* bnBias, - double expAvgFactor, - void* resultRunningMean, - void* resultRunningVariance, - double epsilon, - void* resultSaveMean, - void* resultSaveInvVariance); - -/*! @brief Execute forward inference layer for batch normalization - * - * Batch normalization pass for forward inference pass. - * Takes in batch normalization mode bn_mode and input tensor x, output tensor y, bnBias and bnScale - * with their descriptor. - * - * If either estimatedMean, or estimatedVariance are null pointers then the values for the mean and - * variance will be calculated from input data and this calculated mean and variance will be used - * to update input values. - * If variance is zero and epsilon is also zero, this function outputs NAN values. Input espilon - * value should always be non zero positive value. - * - * @param handle MIOpen handle (input) - * @param bn_mode Batch normalization mode (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param bnScaleBiasMeanVarDesc Tensor descriptor for BN scaling, shifting, saved variance and - * mean (input) - * @param bnScale Batch norm scaling, gamma, tensor (input) - * @param bnBias Batch norm bias, beta, tensor (input) - * @param estimatedMean Running average saved during forward training (input) - * @param estimatedVariance Running variance saved during forward training (input) - * @param epsilon Value to stabilize inverse variance calculation (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenBatchNormalizationForwardInference(miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - void* alpha, - void* beta, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc, - void* bnScale, - void* bnBias, - void* estimatedMean, - void* estimatedVariance, - double epsilon); - -/*! @brief Execute backwards propagation layer for batch normalization - * - * Batch normalization pass for backwards propagation training pass. - * The method for backwards propagation batch normalization. - * - * Takes in batch normalization mode bn_mode and input tensor data x, input activation tensor dy, - * output tensor dx, the learned tensors resultBNBiasDiff and resultBNScaleDiff with their - * descriptor. - * - * If BOTH savedMean, and savedVariance are not null pointers then the method will use the saved - * mean and variance calculated by the forward training phase. - * - * @param handle MIOpen handle (input) - * @param bn_mode Batch normalization mode (input) - * @param alphaDataDiff Floating point scaling factor, allocated on the host (input) - * @param betaDataDiff Floating point shift factor, allocated on the host (input) - * @param alphaParamDiff Floating point scaling factor, allocated on the host (input) - * @param betaParamDiff Floating point shift factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param dyDesc Tensor descriptor for output data tensor y (input) - * @param dy Data tensor y (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dx Data delta tensor dx (output) - * @param bnScaleBiasDiffDesc Tensor descriptor for BN scaling, shifting, saved variance and - * mean (input) - * @param bnScale Batch norm scaling, gamma, tensor (input) - * @param resultBnScaleDiff Tensor for dscale (output) - * @param resultBnBiasDiff Tensor for dbias (output) - * @param epsilon Value to stabilize inverse variance calculation (input) - * @param savedMean Saved mini-batch mean for backwards pass (input) - * @param savedInvVariance Saved mini-bathc inverse variance for backwards pass (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenBatchNormalizationBackward(miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - const void* alphaDataDiff, - const void* betaDataDiff, - const void* alphaParamDiff, - const void* betaParamDiff, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const miopenTensorDescriptor_t bnScaleBiasDiffDesc, - const void* bnScale, - void* resultBnScaleDiff, - void* resultBnBiasDiff, - double epsilon, - const void* savedMean, - const void* savedInvVariance); - -/** @} */ -// CLOSEOUT BATCHNORM DOXYGEN GROUP - -// Activation APIs -/** @addtogroup activation - * - * @{ - */ -/*! @brief Creates the Activation descriptor object - * - * @param activDesc Pointer to an activation tensor descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateActivationDescriptor(miopenActivationDescriptor_t* activDesc); - -/*! @brief Sets the activation layer descriptor details - * - * Sets all of the descriptor details for the activation layer - * - * @param activDesc Pointer to a activation layer descriptor (output) - * @param mode Activation mode enum (input) - * @param activAlpha Alpha value for some activation modes (input) - * @param activBeta Beta value for some activation modes (input) - * @param activGamma Gamma value for some activation modes (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetActivationDescriptor(const miopenActivationDescriptor_t activDesc, - miopenActivationMode_t mode, - double activAlpha, - double activBeta, - double activGamma); - -/*! @brief Gets the activation layer descriptor details - * - * Retrieves all of the descriptor details for the activation layer. - * - * @param activDesc Pointer to a activation layer descriptor (input) - * @param mode Activation mode enum (output) - * @param activAlpha Alpha value for some activation modes (output) - * @param activBeta Beta value for some activation modes (output) - * @param activGamma Gamma value for some activation modes (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetActivationDescriptor(const miopenActivationDescriptor_t activDesc, - miopenActivationMode_t* mode, - double* activAlpha, - double* activBeta, - double* activGamma); - -/*! @brief Execute an activation forward layer - * - * @param handle MIOpen handle (input) - * @param activDesc Descriptor for activation layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenActivationForward(miopenHandle_t handle, - const miopenActivationDescriptor_t activDesc, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y); - -/*! @brief Execute a activation backwards layer - * - * @param handle MIOpen handle (input) - * @param activDesc Descriptor for activation layer (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param yDesc Tensor descriptor for input data tensor y (input) - * @param y Data tensor y (input) - * @param dyDesc Tensor descriptor for input data tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for data output tensor dx (input) - * @param dx Output data delta tensor dx (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenActivationBackward(miopenHandle_t handle, - const miopenActivationDescriptor_t activDesc, - const void* alpha, - const miopenTensorDescriptor_t yDesc, - const void* y, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx); - -/*! @brief Destroys the activation descriptor object - * - * @param activDesc Activation tensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenDestroyActivationDescriptor(miopenActivationDescriptor_t activDesc); - -/** @} */ -// CLOSEOUT ACTIVATION DOXYGEN GROUP - -#ifdef MIOPEN_BETA_API -/** @addtogroup activation - * - * @{ - */ - -/*! @brief Execute a GLU forward layer - * - * @param handle MIOpen handle (input) - * @param inputDesc Tensor descriptor for input tensor (input) - * @param input Input tensor (input) - * @param outputDesc Tensor descriptor for output tensor (input) - * @param output Output tensor (output) - * @param dim Dimension to split the input (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGLUForward(miopenHandle_t handle, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputDesc, - void* output, - const uint32_t dim); - -/*! @brief Execute a GLU backward layer - * - * @param handle MIOpen handle (input) - * @param inputDesc Tensor descriptor for input tensor (input) - * @param input Input tensor (input) - * @param outputGradDesc Tensor descriptor for delta output tensor (input) - * @param outputGrad Delta output tensor (input) - * @param inputGradDesc Tensor descriptor for delta input tensor (input) - * @param inputGrad Delta input tensor (output) - * @param dim Dimension to split the input (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGLUBackward(miopenHandle_t handle, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputGradDesc, - const void* outputGrad, - const miopenTensorDescriptor_t inputGradDesc, - void* inputGrad, - const uint32_t dim); - -/** @} */ -// CLOSEOUT ACTIVATION DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -// Softmax APIs -/** @addtogroup softmax - * - * @{ - */ -/*! @brief Execute a softmax forward layer - * - * This API only implements the SOFTMAX_MODE_CHANNEL in SOFTMAX_ACCURATE path. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSoftmaxForward(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y); - -/*! @brief Execute a softmax backwards layer - * - * This API only implements the SOFTMAX_MODE_CHANNEL in SOFTMAX_ACCURATE path. - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param yDesc Tensor descriptor for input data tensor y (input) - * @param y Data tensor y (input) - * @param dyDesc Tensor descriptor for input data tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for data output tensor dx (input) - * @param dx Output data delta tensor dx (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSoftmaxBackward(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t yDesc, - const void* y, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx); - -/*! @brief Execute a softmax forward layer with expanded modes and algorithms - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param algorithm Softmax implementation algorithm (input) - * @param mode Softmax mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSoftmaxForward_V2(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t xDesc, - const void* x, - const void* beta, - const miopenTensorDescriptor_t yDesc, - void* y, - miopenSoftmaxAlgorithm_t algorithm, - miopenSoftmaxMode_t mode); - -/*! @brief Execute a softmax backwards layer with expanded modes and algorithms - * - * @param handle MIOpen handle (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param yDesc Tensor descriptor for input data tensor y (input) - * @param y Data tensor y (input) - * @param dyDesc Tensor descriptor for input data tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param dxDesc Tensor descriptor for data output tensor dx (input) - * @param dx Output data delta tensor dx (output) - * @param algorithm Softmax implementation algorithm (input) - * @param mode Softmax mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSoftmaxBackward_V2(miopenHandle_t handle, - const void* alpha, - const miopenTensorDescriptor_t yDesc, - const void* y, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const void* beta, - const miopenTensorDescriptor_t dxDesc, - void* dx, - miopenSoftmaxAlgorithm_t algorithm, - miopenSoftmaxMode_t mode); - -/** @} */ -// CLOSEOUT SOFTMAX DOXYGEN GROUP - -/*! @ingroup FUSION - * @brief MIOpen fusion interface - */ -MIOPEN_DECLARE_OBJECT(miopenFusionPlanDescriptor); -MIOPEN_DECLARE_OBJECT(miopenOperatorDescriptor); -MIOPEN_DECLARE_OBJECT(miopenOperatorArgs); - -/** @addtogroup FUSION - * - * @{ - */ - -/*! @enum miopenFusionDirection_t - * @brief Kernel fusion direction in the network - */ -typedef enum -{ - miopenVerticalFusion = 0, /*!< fuses layers vertically, current the only supported mode */ - miopenHorizontalFusion = 1, /*!< fuses layers horizontally, this is unimplemented */ -} miopenFusionDirection_t; - -/*! @brief Creates the kenrel fusion plan descriptor object - * - * @param fusePlanDesc Pointer to a fusion plan (output) - * @param fuseDirection Horizontal or Vertical fusion (input) - * @param inputDesc Descriptor to tensor for the input (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateFusionPlan(miopenFusionPlanDescriptor_t* fusePlanDesc, - const miopenFusionDirection_t fuseDirection, - const miopenTensorDescriptor_t inputDesc); - -/*! @brief Destroy the fusion plan descriptor object - * - * @param fusePlanDesc A fusion plan descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyFusionPlan(miopenFusionPlanDescriptor_t fusePlanDesc); - -/*! @brief Compiles the fusion plan - * - * @param handle MIOpen handle (input) - * @param fusePlanDesc A fusion plan descriptor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCompileFusionPlan(miopenHandle_t handle, - miopenFusionPlanDescriptor_t fusePlanDesc); - -/*! - * @brief Allows access to the operators in a fusion plan - * @details This api call does bounds checking on the supplied op_idx and would - * return miopenStatusError if the index is out of bounds - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param op_idx Index of the required operator in the fusion plan, in the order of insertion - * @param op returned pointer to the operator - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenFusionPlanGetOp(miopenFusionPlanDescriptor_t fusePlanDesc, - const int op_idx, - miopenFusionOpDescriptor_t* op); - -/*! @brief Query the workspace size required for the fusion plan - * @param handle MIOpen handle (input) - * @param fusePlanDesc A fusion plan descriptor (input) - * @param workSpaceSize Pointer to memory to return size in bytes (output) - * @param algo Algorithm selected (inputs) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFusionPlanGetWorkSpaceSize(miopenHandle_t handle, - miopenFusionPlanDescriptor_t fusePlanDesc, - size_t* workSpaceSize, - miopenConvFwdAlgorithm_t algo); - -/*! - * @brief Returns the supported algorithms for the convolution operator in the Fusion Plan - * - * @details A Convolution operator in a fusion plan may be implemented by different algorithms - * representing different tradeoffs of memory and performance. The returned list of algorithms - * is sorted in decreasing order of priority. Therefore, if the user does not request an - * algorithm to be set using the miopenFusionPlanConvolutionSetAlgo call, the first algorithm - * in the list would be used to execute the convolution in the fusion plan. Moreover this call - * must be immediately preceded by the miopenCreateOpConvForward call for the op in question. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param requestAlgoCount Number of algorithms to return (input) - * @param returnedAlgoCount The actual number of returned algorithms; always be less than - * equal to requestAlgoCount (output) - * @param returnedAlgos Pointer to the list of supported algorithms - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFusionPlanConvolutionGetAlgo(miopenFusionPlanDescriptor_t fusePlanDesc, - const int requestAlgoCount, - int* returnedAlgoCount, - miopenConvFwdAlgorithm_t* returnedAlgos); - -/*! @brief Requests the fusion runtime to choose a particular algorithm for the added convolution - * operation - * - * @details Please see the description for miopenFusionPlanConvolutionGetAlgo - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param algo Requested algorithm for the convolution operator (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenFusionPlanConvolutionSetAlgo( - miopenFusionPlanDescriptor_t fusePlanDesc, miopenConvFwdAlgorithm_t algo); - -/*! @brief Creates forward convolution operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param convOp Pointer to an operator type (output) - * @param convDesc Convolution layer descriptor (input) - * @param wDesc Descriptor for the weights tensor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateOpConvForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* convOp, - miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t wDesc); - -//--- - -// Activation forward create ops --- -/*! @brief Creates a forward activation operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param activFwdOp Pointer to an operator type (output) - * @param mode Activation version (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateOpActivationForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* activFwdOp, - miopenActivationMode_t mode); - -// Activation backward create ops --- -/*! @brief Creates a backward activation operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param activBwdOp Pointer to an operator type (output) - * @param mode Activation version (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateOpActivationBackward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* activBwdOp, - miopenActivationMode_t mode); - -// Bias create ops --- -/*! @brief Creates a forward bias operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param biasOp Pointer to an operator type (output) - * @param bDesc bias tensor descriptor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateOpBiasForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* biasOp, - const miopenTensorDescriptor_t bDesc); - -// Batch normalization create ops --- -/*! @brief Creates a forward inference batch normalization operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param bnOp Pointer to an operator type (output) - * @param bn_mode Batch normalization layer mode (input) - * @param bnScaleBiasMeanVarDesc Gamma, beta, mean, variance tensor descriptor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateOpBatchNormInference(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnOp, - const miopenBatchNormMode_t bn_mode, - const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc); - -/*! @brief Creates a forward training batch normalization operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param bnFwdOp Pointer to an operator type (output) - * @param bn_mode Batch normalization layer mode (input) - * @param runningMeanVariance Toggles whether or not to save population statistics for inference; - * batch statistic are required (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateOpBatchNormForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnFwdOp, - const miopenBatchNormMode_t bn_mode, - bool runningMeanVariance); - -/*! @brief Creates a back propagation batch normalization operator. - * - * @param fusePlanDesc A fusion plan descriptor (input) - * @param bnBwdOp Pointer to an operator type (output) - * @param bn_mode Batch normalization layer mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateOpBatchNormBackward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnBwdOp, - const miopenBatchNormMode_t bn_mode); - -//--- -/*! @brief Creates an operator argument object - * - * @param args Pointer to an operator argument type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateOperatorArgs(miopenOperatorArgs_t* args); - -/*! @brief Destroys an operator argument object - * - * @param args An operator argument type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyOperatorArgs(miopenOperatorArgs_t args); - -// Convolution set arguments --- -/*! @brief Sets the arguments for forward convolution op - * - * @param args An arguments object type (output) - * @param convOp Forward convolution operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param w Pointer to tensor memory (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetOpArgsConvForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t convOp, - const void* alpha, - const void* beta, - const void* w); -// Activation set arguments --- -/*! @brief Sets the arguments for forward activation op - * - * @param args An arguments object type (output) - * @param activFwdOp Activation backwards operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param activAlpha Double precision activation parameter which depends on activation mode (input) - * @param activBeta Double precision activation parameter which depends on activation mode (input) - * @param activGamma Double precision activation parameter which depends on activation mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetOpArgsActivForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t activFwdOp, - const void* alpha, - const void* beta, - double activAlpha, - double activBeta, - double activGamma); - -/*! @brief Sets the arguments for backward activation op - * - * @param args An arguments object type (output) - * @param activBwdOp Activation backwards operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param y Data tensor y, output of activations in the forward direction (input) - * @param reserved Data tensor reserved memory space; currently should be nullptr (input) - * @param activAlpha Double precision activation parameter which depends on activation mode (input) - * @param activBeta Double precision activation parameter which depends on activation mode (input) - * @param activGamma Double precision activation parameter which depends on activation mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetOpArgsActivBackward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t activBwdOp, - const void* alpha, - const void* beta, - const void* y, - const void* reserved, - double activAlpha, - double activBeta, - double activGamma); - -// Batch Normalization set arguments --- -/*! @brief Sets the arguments for inference batch normalization op - * - * @param args An arguments object type (output) - * @param bnOp Batch normalization inference operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param bnScale Pointer to the gamma tensor memory (input) - * @param bnBias Pointer to the beta tensor memory (input) - * @param estimatedMean Pointer to population mean memory (input) - * @param estimatedVariance Pointer to population variance memory (input) - * @param epsilon Scalar value for numerical stability (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetOpArgsBatchNormInference(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnOp, - const void* alpha, - const void* beta, - const void* bnScale, - const void* bnBias, - const void* estimatedMean, - const void* estimatedVariance, - double epsilon); - -/*! @brief Sets the arguments for forward batch normalization op - * - * @param args An arguments object type (output) - * @param bnOp Batch normalization forward operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param bnScale Pointer to the gamma tensor memory (input) - * @param bnBias Pointer to the beta tensor memory (input) - * @param savedMean Pointer to batch mean memory (input) - * @param savedInvVariance Pointer to batch inverse variance memory (input) - * @param runningMean Pointer to population mean memory (input) - * @param runningVariance Pointer to population variance memory (input) - * @param expAvgFactor Scalar value for control of population statistics (input) - * @param epsilon Scalar value for numerical stability (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetOpArgsBatchNormForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnOp, - const void* alpha, - const void* beta, - const void* bnScale, - const void* bnBias, - void* savedMean, - void* savedInvVariance, - void* runningMean, - void* runningVariance, - double expAvgFactor, - double epsilon); - -/*! @brief Sets the arguments for backward batch normalization op - * - * @param args An arguments object type (output) - * @param bnOp Batch normalization forward operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param x Pointer to the forward input tensor memory (input) - * @param bnScale Pointer to the gamma tensor memory (input) - * @param bnBias Pointer to the beta tensor memory (input) - * @param resultBnScaleDiff Pointer to the gamma gradient tensor memory (output) - * @param resultBnBiasDiff Pointer to the beta gradient tensor memory (output) - * @param savedMean Pointer to batch mean memory (input) - * @param savedInvVariance Pointer to batch inverse variance memory (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetOpArgsBatchNormBackward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnOp, - const void* alpha, - const void* beta, - const void* x, - const void* bnScale, - const void* bnBias, - void* resultBnScaleDiff, - void* resultBnBiasDiff, - const void* savedMean, - const void* savedInvVariance); - -// Bias forward set arguments --- -/*! @brief Sets the arguments for forward bias op - * - * @param args An arguments object type (output) - * @param biasOp Forward bias operator (input) - * @param alpha Floating point scaling factor, allocated on the host (input) - * @param beta Floating point shift factor, allocated on the host (input) - * @param bias Pointer to the forward bias input tensor memory (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetOpArgsBiasForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t biasOp, - const void* alpha, - const void* beta, - const void* bias); -/*! @brief Executes the fusion plan - * - * - * @param handle MIOpen handle (input) - * @param fusePlanDesc fused plan descriptor (input) - * @param inputDesc Descriptor of the input tensor (input) - * @param input Source data tensor (input) - * @param outputDesc Decriptor of the output tensor (input) - * @param output Destination data tensor (output) - * @param args An argument object of the fused kernel (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenExecuteFusionPlan(const miopenHandle_t handle, - const miopenFusionPlanDescriptor_t fusePlanDesc, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputDesc, - void* output, - miopenOperatorArgs_t args); - -/*! @brief Prepares and executes the Convlution+Bias+Activation Fusion. - * - * - * @param handle MIOpen handle (input) - * @param alpha1 floating point scaling factor, allocated on the host (input) - * @param xDesc Tensor descriptor for input data tensor x (input) - * @param x Data tensor x (input) - * @param wDesc Tensor descriptor for weight tensor w (input) - * @param w Weights tensor w (input) - * @param convDesc Convolution layer descriptor (input) - * @param algo Algorithm selected (inputs) - * @param workspace Pointer to workspace required (input) - * @param workspaceSizeInBytes Size of the memory in bytes pointed to by workSpace above - * @param alpha2 floating point scaling factor, allocated on the host (input) - * @param zDesc Tensor descriptor for tensor z (input) - * @param z Data tensor z (input) - * @param biasDesc Tensor descriptor for input data tensor x (input) - * @param bias Data tensor bias (input) - * @param activationDesc Activation descriptor that specifies the activation mode - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Output data tensor - */ - -MIOPEN_EXPORT miopenStatus_t -miopenConvolutionBiasActivationForward(miopenHandle_t handle, - const void* alpha1, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t convDesc, - miopenConvFwdAlgorithm_t algo, - void* workspace, - size_t workspaceSizeInBytes, - const void* alpha2, - const miopenTensorDescriptor_t zDesc, - const void* z, - const miopenTensorDescriptor_t biasDesc, - const void* bias, - const miopenActivationDescriptor_t activationDesc, - const miopenTensorDescriptor_t yDesc, - void* y); -/** @} */ -// CLOSEOUT FUSION DOXYGEN GROUP - -/** @addtogroup RNN - * - * @{ - */ - -/*! @enum miopenRNNMode_t - * RNN mode selection for rnn layer preference - */ -typedef enum -{ - miopenRNNRELU = 0, /*!< RNN with ReLU activation */ - miopenRNNTANH = 1, /*!< RNN with tanh activation */ - miopenLSTM = 2, /*!< LSTM */ - miopenGRU = 3, /*!< GRU */ -} miopenRNNMode_t; - -/*! @enum miopenRNNInputMode_t - * Recurrent Neural Network layer initial input mode - */ -typedef enum -{ - miopenRNNlinear = 0, /*!< Matrix multiplication at the input of the first layer */ - miopenRNNskip = 1, /*!< No operation is performed at the input of the first layer. */ -} miopenRNNInputMode_t; - -/*! @enum miopenRNNAlgo_t - * Recurrent Neural Network algorithm mode - */ -typedef enum -{ - miopenRNNdefault = 0, /*!< Use dedicated gate-operation kernel for LSTM and fundamental - algorithm for vanilla RNN & GRU */ - miopenRNNfundamental = - 1, /*!< Function by basic tesnsor operations, supported for vanilla RNN, LSTM, GRU */ -} miopenRNNAlgo_t; - -/*! @enum miopenRNNDirectionMode_t - * Recurrent Neural Network bi-directional behavior - */ -typedef enum -{ - miopenRNNunidirection = 0, /*!< Forward in time only. */ - miopenRNNbidirection = 1, /*!< Forward and backwards in time. */ -} miopenRNNDirectionMode_t; - -/*! @enum miopenRNNBiasMode_t - * Recurrent Neural Network add on bias - */ -typedef enum -{ - miopenRNNNoBias = 0, /*!< No Biases will be applied to GEMM operations */ - miopenRNNwithBias = 1, /*!< Biases will be applied to GEMM operations */ -} miopenRNNBiasMode_t; - -/*! @enum miopenRNNGEMMalgoMode_t - * Recurrent Neural Network add on bias - */ -typedef enum -{ - miopenRNNAlgoGEMM = 0, -} miopenRNNGEMMalgoMode_t; - -/*! @enum miopenRNNPaddingMode_t - * Recurrent Neural Network input/output data padding mode - */ -typedef enum -{ - miopenRNNIONotPadded = 0, /*!< Not padded data at RNN input/output */ - miopenRNNIOWithPadding = 1, /*!< Padded data at RNN input/output */ -} miopenRNNPaddingMode_t; - -/*! @enum miopenRNNFWDMode_t - * Recurrent Neural Network Training/Inference mode - */ -typedef enum -{ - miopenRNNTraining = 0, /*!< FWD, BWD, WRW */ - miopenRNNInference = 1, /*!< Only FWD-inference no back-propagation */ -} miopenRNNFWDMode_t; - -/*! @enum miopenRNNBaseLayout_t - * Data layouts for RNN operations - */ -typedef enum -{ - miopenRNNDataUnknownLayout = 0, - miopenRNNDataSeqMajorNotPadded = 1, - miopenRNNDataSeqMajorPadded = 2, - miopenRNNDataBatchMajorPadded = 3, -} miopenRNNBaseLayout_t; - -/*! @brief Create a RNN layer Descriptor - * - * API for creating an uninitialized RNN layer descriptor. - * @param rnnDesc Pointer to a tensor descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateRNNDescriptor(miopenRNNDescriptor_t* rnnDesc); - -/*! @brief Retrieves a RNN layer descriptor's details - * - * @param rnnDesc RNN layer descriptor (input) - * @param rnnMode RNN mode (output) - * @param algoMode RNN algorithm mode (output) - * @param inputMode RNN data input mode (output) - * @param dirMode Uni or bi direction mode (output) - * @param biasMode Bias used (output) - * @param hiddenSize Size of hidden state (output) - * @param layer Number of stacked layers (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNDescriptor(miopenRNNDescriptor_t rnnDesc, - miopenRNNMode_t* rnnMode, - miopenRNNAlgo_t* algoMode, - miopenRNNInputMode_t* inputMode, - miopenRNNDirectionMode_t* dirMode, - miopenRNNBiasMode_t* biasMode, - int* hiddenSize, - int* layer); - -/*! @brief Retrieves a RNN layer descriptor's details version 2. This version enables retrieving - * information of the dropout descriptor of the rnn descriptor. - * - * @param rnnDesc RNN layer descriptor (input) - * @param hiddenSize Size of hidden state (output) - * @param layer Number of stacked layers (output) - * @param dropoutDesc Pre-configured dropout descriptor for dropout layer in between RNN layers - * (output) - * @param inputMode RNN data input mode (output) - * @param dirMode Uni or bi direction mode (output) - * @param rnnMode RNN mode (output) - * @param biasMode Bias used (output) - * @param algoMode RNN algorithm mode (output) - * @param dataType Data type of RNN (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNDescriptor_V2(miopenRNNDescriptor_t rnnDesc, - int* hiddenSize, - int* layer, - miopenDropoutDescriptor_t* dropoutDesc, - miopenRNNInputMode_t* inputMode, - miopenRNNDirectionMode_t* dirMode, - miopenRNNMode_t* rnnMode, - miopenRNNBiasMode_t* biasMode, - miopenRNNAlgo_t* algoMode, - miopenDataType_t* dataType); - -/*! @brief Destroys the tensor descriptor object - * - * @param rnnDesc RNN tensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyRNNDescriptor(miopenRNNDescriptor_t rnnDesc); - -/*! @brief Set the details of the RNN descriptor - * - * Interface for setting the values of the RNN descriptor object. This function requires specific - * algorithm selection. - * @param rnnDesc RNN layer descriptor type (input) - * @param hsize Hidden layer size (input) - * @param nlayers Number of layers (input) - * @param inMode RNN first layer input mode (input) - * @param direction RNN direction (input) - * @param rnnMode RNN model type (input) - * @param biasMode RNN bias included (input) - * @param algo RNN algorithm selected (input) - * @param dataType MIOpen datatype (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetRNNDescriptor(miopenRNNDescriptor_t rnnDesc, - const int hsize, - const int nlayers, - miopenRNNInputMode_t inMode, - miopenRNNDirectionMode_t direction, - miopenRNNMode_t rnnMode, - miopenRNNBiasMode_t biasMode, - miopenRNNAlgo_t algo, - miopenDataType_t dataType); - -/*! @brief Set the details of the RNN descriptor version 2. This version enables the use of dropout - * in rnn. - * - * Interface for setting the values of the RNN descriptor object. This function requires specific - * algorithm selection. - * @param rnnDesc RNN layer descriptor type (input/output) - * @param hsize Hidden layer size (input) - * @param nlayers Number of layers (input) - * @param dropoutDesc Pre-initialized dropout descriptor for dropout layer in between RNN layers - * (input) - * @param inMode RNN first layer input mode (input) - * @param direction RNN direction (input) - * @param rnnMode RNN model type (input) - * @param biasMode RNN bias included (input) - * @param algo RNN algorithm selected (input) - * @param dataType MIOpen datatype (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetRNNDescriptor_V2(miopenRNNDescriptor_t rnnDesc, - const int hsize, - const int nlayers, - miopenDropoutDescriptor_t dropoutDesc, - miopenRNNInputMode_t inMode, - miopenRNNDirectionMode_t direction, - miopenRNNMode_t rnnMode, - miopenRNNBiasMode_t biasMode, - miopenRNNAlgo_t algo, - miopenDataType_t dataType); - -/*! @brief Set shape of RNN seqData tensor - * - * Interface for setting tensor shape to be used as RNN input data - * - * @param seqTensorDesc Tensor descriptor (input/output) - * @param dataType MIOpen datatype (input) - * @param layout One of the main supported layouts for RNN data(input) - * @param maxSequenceLen Sequence length limit within this SeqTensor(input) - * @param batchSize Number of sequences within this SeqTensor (input) - * @param vectorSize Vector size (input) - * @param sequenceLenArray Array containing the length of each sequence in the SeqTensor(input) - * @param paddingMarker Not used, should be NULL (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetRNNDataSeqTensorDescriptor(miopenSeqTensorDescriptor_t seqTensorDesc, - miopenDataType_t dataType, - miopenRNNBaseLayout_t layout, - int maxSequenceLen, - int batchSize, - int vectorSize, - const int* sequenceLenArray, - void* paddingMarker); - -/*! @brief Get shape of RNN seqData tensor - * - * Interface for setting tensor shape to be used as RNN input data - * - * @param seqTensorDesc Tensor descriptor (input) - * @param dataType MIOpen datatype (output) - * @param layout One of the main supported layouts for RNN data(output) - * @param maxSequenceLen Sequence length limit within this SeqTensor(output) - * @param batchSize Number of sequences within this SeqTensor (output) - * @param vectorSize Vector size (output) - * @param sequenceLenArrayLimit Limit for number of elements that can be returned to user - * by sequenceLenArray (input) - * @param sequenceLenArray Array containing the length of each sequence in the - * SeqTensor. This is allowed to be a NULL pointer if sequenceLenArrayLimit is 0 (output) - * @param paddingMarker Not used, should be NULL (input) - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t -miopenGetRNNDataSeqTensorDescriptor(miopenSeqTensorDescriptor_t seqTensorDesc, - miopenDataType_t* dataType, - miopenRNNBaseLayout_t* layout, - int* maxSequenceLen, - int* batchSize, - int* vectorSize, - int sequenceLenArrayLimit, - int* sequenceLenArray, - void* paddingMarker); - -/*! @brief Query the amount of memory required to execute the RNN layer - * - * This function calculates the amount of memory required to run the RNN layer given an RNN - * descriptor and a tensor descriptor. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Number of iteration unrolls (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param numBytes Number of bytes required for RNN layer execution (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNWorkspaceSize(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* xDesc, - size_t* numBytes); - -/*! @brief Query the amount of memory required for RNN training - * - * This function calculates the amount of memory required to train the RNN layer given an - * RNN descriptor and a tensor descriptor. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Number of iteration unrolls (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param numBytes Number of bytes required for RNN layer execution (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNTrainingReserveSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* xDesc, - size_t* numBytes); - -/*! @brief Query the amount of additional memory required for this RNN layer execution. - * - * This function calculates the size of extra buffers, depending on the layer configuration, which - * is determined by: RNN descriptor, isInference, and data descriptor. If isInference is True, - * reserve_space_size is always zero, because the reserve_space buffer is not used in Inference - * computation. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param xDesc Sequence data tensor descriptor (input) - * @param fwdMode Specifies in which mode the buffers will be used. - * @param workSpaceSize Minimum WorkSpace buffer size required for RNN layer execution (output) - * @param reserveSpaceSize Minimum ReserveSpaceSize buffer size required for RNN layer execution - * (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNTempSpaceSizes(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - miopenSeqTensorDescriptor_t xDesc, - miopenRNNFWDMode_t fwdMode, - size_t* workSpaceSize, - size_t* reserveSpaceSize); - -/*! @brief Query the amount of parameter memory required for RNN training - * - * This function calculates the amount of parameter memory required to train the RNN layer given an - * RNN descriptor and a tensor descriptor. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param xDesc A tensor descriptor (input) - * @param numBytes Number of bytes required for RNN layer execution (output) - * @param dtype MIOpen data type enum (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNParamsSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - miopenTensorDescriptor_t xDesc, - size_t* numBytes, - miopenDataType_t dtype); - -/*! @brief Obtain a weight tensor descriptor for RNNs - * - * This function populates a weight descriptor that describes the memory layout of the - * weight matrix. - * - * @param handle MIOpen handle (input) - * @param rnnDesc Fully populated RNN layer descriptor type (input) - * @param xDesc A previously populated tensor descriptor (input) - * @param wDesc A previously allocated tensor descriptor (output) - * @param dtype MIOpen data type enum (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNParamsDescriptor(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - miopenTensorDescriptor_t xDesc, - miopenTensorDescriptor_t wDesc, - miopenDataType_t dtype); - -/*! @brief Obtain a the size in bytes of the RNN input tensor - * - * This function determines the size in bytes of the allocation needed for the input data - * tensor for an RNN layer. The number of bytes is derived from the array of - * tensor descriptors. - * - * @param handle MIOpen handle (input) - * @param rnnDesc Fully populated RNN layer descriptor (input) - * @param seqLen Number of iteration unrolls (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param numBytes Number of bytes required for input tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNInputTensorSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int seqLen, - miopenTensorDescriptor_t* xDesc, - size_t* numBytes); - -/*! @brief Obtain a the size in bytes of the RNN hidden tensor - * - * This function determines the size in bytes of the allocation needed for the - * hidden tensor over all layers - * - * @param handle MIOpen handle (input) - * @param rnnDesc Fully populated RNN layer descriptor type (input) - * @param seqLen Number of iteration unrolls (input) - * @param xDesc An array of previously populated tensor descriptors (input) - * @param numBytes Number of bytes required for input tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNHiddenTensorSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int seqLen, - miopenTensorDescriptor_t* xDesc, - size_t* numBytes); - -/*! @brief Gets the number of bytes of a parameter matrix - * - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, paramID == 0 retrieves the - * weight matrix associated with the in input GEMM, while paramID == 1 retrieves - * the weight matrix associated with the hidden state GEMM. - * - * For miopenLSTM paramID 0 to 3 refer to the weight matrices associated - * with the input GEMM, 4-7 are associated with matrices associated with the - * hidden state GEMM. - * - * * paramID 0 and 4 are for the input gate. - * - * * paramID 1 and 5 are for the forget gate. - * - * * paramID 2 and 6 are for the output gate. - * - * * paramID 3 and 7 are for the new memory gate. - * - * For miopenGRU paramID 0 to 2 refer to the weight matrix offset associated - * with the input GEMM, while 3 through 5 are associated with the hidden state - * GEMM. - * - * * paramID 0 and 3 are for the update gate. - * - * * paramID 1 and 4 are for the reset gate. - * - * * paramID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param paramID ID of the internal parameter tensor (input) - * @param numBytes The number of bytes of the layer's parameter matrix (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerParamSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - const int paramID, - size_t* numBytes); - -/*! @brief Gets the number of bytes of a bias - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, biasID == 0 retrieves the - * weight matrix associated with the in input GEMM, while biasID == 1 retrieves - * the bias associated with the hidden state GEMM. - * - * For miopenLSTM biasID 0 to 3 refer to the biases associated - * with the input GEMM, 4-7 are associated with biases associated with the - * hidden state GEMM. - * - * * biasID 0 and 4 are for the input gate. - * - * * biasID 1 and 5 are for the forget gate. - * - * * biasID 2 and 6 are for the output gate. - * - * * biasID 3 and 7 are for the new memory gate. - * - * For miopenGRU biasID 0 to 2 refer to the biases associated with the input GEMM, - * while 3 through 5 are associated with the hidden state GEMM. - * - * * biasID 0 and 3 are for the update gate. - * - * * biasID 1 and 4 are for the reset gate. - * - * * biasID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param biasID ID of the internal parameter tensor (input) - * @param numBytes The number of bytes of the layer's bias (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerBiasSize(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - const int biasID, - size_t* numBytes); - -/*! @brief Gets a weight matrix for a specific layer in an RNN stack - * - * This function retrieves the weight matrix data for a specific layer and parameter ID - * and copies the data into previously allocated device memory. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, paramID == 0 retrieves the - * weight matrix associated with the in input GEMM, while paramID == 1 retrieves - * the weight matrix associated with the hidden state GEMM. - * - * For miopenLSTM paramID 0 to 3 refer to the weight matrices associated - * with the input GEMM, 4-7 are associated with matrices associated with the - * hidden state GEMM. - * - * * paramID 0 and 4 are for the input gate. - * - * * paramID 1 and 5 are for the forget gate. - * - * * paramID 2 and 6 are for the output gate. - * - * * paramID 3 and 7 are for the new memory gate. - * - * For miopenGRU paramID 0 to 2 refer to the weight matrix offset associated - * with the input GEMM, while 3 through 5 are associated with the hidden state - * GEMM. - * - * * paramID 0 and 3 are for the update gate. - * - * * paramID 1 and 4 are for the reset gate. - * - * * paramID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The output argument paramDesc is a previously created tensor descriptor that is populated - * to describe the memory layout of the parameter matrix. It is full packed and is used when - * calling to miopenSetRNNLayerParam() - * - * The argument layerParam should either be nullptr, or have device memory allocated - * to allow copying of the entire layer parameter matrix into it. If layerParam is - * nullptr then only the paramDesc is populated and returned. The size in bytes of the - * layer parameter matrix can be determined by using miopenGetRNNLayerParamSize(). - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenGetRNNLayerParam() will return - * a error status miopenStatusBadParm for input paramID associated with the input GEMM. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param wDesc A tensor descriptor to the parameter tensor (input) - * @param w Pointer to memory containing parameter tensor (input) - * @param paramID ID of the internal parameter tensor (input) - * @param paramDesc Tensor descriptor for the fully packed output parameter tensor (output) - * @param layerParam Pointer to the memory location of the parameter tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerParam(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - miopenTensorDescriptor_t wDesc, - const void* w, - const int paramID, - miopenTensorDescriptor_t paramDesc, - void* layerParam); - -/*! @brief Gets a bias for a specific layer in an RNN stack - * - * This function retrieves the bias data for a specific layer and bias ID and copies - * the data into previously allocated device memory. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, biasID == 0 retrieves the - * bias associated with the in input GEMM, while biasID == 1 retrieves - * the bias associated with the hidden state GEMM. - * - * For miopenLSTM biasID 0 to 3 refer to the biases associated - * with the input GEMM, 4-7 are associated with biases associated with the - * hidden state GEMM. - * - * * biasID 0 and 4 are for the input gate. - * - * * biasID 1 and 5 are for the forget gate. - * - * * biasID 2 and 6 are for the output gate. - * - * * biasID 3 and 7 are for the new memory gate. - * - * For miopenGRU biasID 0 to 2 refer to the biases associated with the input GEMM, - * while 3 through 5 are associated with the hidden state GEMM. - * - * * biasID 0 and 3 are for the update gate. - * - * * biasID 1 and 4 are for the reset gate. - * - * * biasID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The output argument biasDesc is a previously created tensor descriptor that is populated - * to describe the memory layout of the bias. It is full packed and is used when - * calling to miopenSetRNNLayerBias() - * - * The argument layerBias should either be nullptr, or have device memory allocated - * to allow copying of the entire layer bias into it. If layerBias is - * nullptr then only the biasDesc is populated and returned. The size in bytes of the - * layer bias can be determined by using miopenGetRNNLayerBiasSize(). - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenGetRNNLayerBias() will return - * a error status miopenStatusBadParm for input biasID associated with the input GEMM. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param wDesc A tensor descriptor to the parameter tensor (input) - * @param w Pointer to memory containing parameter tensor (input) - * @param biasID ID of the internal parameter tensor (input) - * @param biasDesc Descriptor of the parameter tensor (output) - * @param layerBias Pointer to the memory location of the bias tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerBias(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - miopenTensorDescriptor_t wDesc, - const void* w, - const int biasID, - miopenTensorDescriptor_t biasDesc, - void* layerBias); - -/*! @brief Gets an index offset for a specific weight matrix for a layer in the - * RNN stack - * - * This function retrieves the index offset for a weight matrix in a layer. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, paramID == 0 retrieves the - * weight matrix offset associated with the in input GEMM, while paramID == 1 - * retrieves the weight matrix offset associated with the hidden state GEMM. - * - * For miopenLSTM paramID 0 to 3 refer to the weight matrix offsets associated - * with the input GEMM, 4-7 are associated with matrix offset associated with the - * hidden state GEMM. - * - * * paramID 0 and 4 are for the input gate. - * - * * paramID 1 and 5 are for the forget gate. - * - * * paramID 2 and 6 are for the output gate. - * - * * paramID 3 and 7 are for the new memory gate. - * - * For miopenGRU paramID 0 to 2 refer to the weight matrix offset associated - * with the input GEMM, while 3 through 5 are associated with the hidden state - * GEMM. - * - * * paramID 0 and 3 are for the update gate. - * - * * paramID 1 and 4 are for the reset gate. - * - * * paramID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The output argument paramDesc is a previously created tensor descriptor that is populated - * to describe the memory layout of the parameter matrix. It is full packed and is used when - * calling to miopenSetRNNLayerParam(). - * - * The argument layerParamOffset should either be nullptr, or an address to place the - * offset. If layerParamOffset is nullptr then only the paramDesc is populated and returned. - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenGetRNNLayerParamOffset() will return - * a error status miopenStatusBadParm for input paramID associated with the input GEMM. - * - * - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param paramID ID of the internal parameter tensor (input) - * @param paramDesc Tensor descriptor for the fully packed output parameter tensor (output) - * @param layerParamOffset Location for the parameter offset (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerParamOffset(miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - const int paramID, - miopenTensorDescriptor_t paramDesc, - size_t* layerParamOffset); - -/*! @brief Gets a bias index offset for a specific layer in an RNN stack - * - * This function retrieves the bias index offset for a specific layer and bias ID. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, biasID == 0 retrieves the - * bias associated with the in input GEMM, while biasID == 1 retrieves - * the weight matrix associated with the hidden state GEMM. - * - * For miopenLSTM biasID 0 to 3 refer to the bias offset associated - * with the input GEMM, 4-7 are the bias offsets associated with the hidden state GEMM. - * - * * biasID 0 and 4 are for the input gate. - * - * * biasID 1 and 5 are for the forget gate. - * - * * biasID 2 and 6 are for the output gate. - * - * * biasID 3 and 7 are for the new memory gate. - * - * For miopenGRU biasID 0 to 2 refer to the biases associated with the input GEMM, - * while 3 through 5 are associated with the hidden state GEMM. - * - * * biasID 0 and 3 are for the update gate. - * - * * biasID 1 and 4 are for the reset gate. - * - * * biasID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The output argument biasDesc is a previously created tensor descriptor that is populated - * to describe the memory layout of the bias. It is full packed and is used when - * calling to miopenSetRNNLayerBias() - * - * The argument layerBiasOffset should either be nullptr, or point to an output address. - * If layerBias is nullptr then only the biasDesc is populated and returned. - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenGetRNNLayerBiasOffset() will return - * a error status miopenStatusBadParm for input biasID associated with the input GEMM. - * - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param biasID ID of the internal parameter tensor (input) - * @param biasDesc Descriptor of the parameter tensor (output) - * @param layerBiasOffset Pointer to the memory location of the bias tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetRNNLayerBiasOffset(miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - const int biasID, - miopenTensorDescriptor_t biasDesc, - size_t* layerBiasOffset); - -/*! @brief Sets a weight matrix for a specific layer in an RNN stack - * - * This function sets the weight matrix data for a specific layer and parameter ID. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, paramID == 0 sets the - * weight matrix associated with the in input GEMM, while paramID == 1 sets - * the weight matrix associated with the hidden state GEMM. - * - * - * For miopenLSTM paramID 0 to 3 refer to the weight matrices associated - * with the input GEMM, 4-7 are associated with matrices associated with the - * hidden state GEMM. - * - * * paramID 0 and 4 are for the input gate. - * - * * paramID 1 and 5 are for the forget gate. - * - * * paramID 2 and 6 are for the output gate. - * - * * paramID 3 and 7 are for the new memory gate. - * - * For miopenGRU paramID 0 to 2 refer to the weight matrix offset associated - * with the input GEMM, while 3 through 5 are associated with the hidden state - * GEMM. - * - * * paramID 0 and 3 are for the update gate. - * - * * paramID 1 and 4 are for the reset gate. - * - * * paramID 2 and 5 are for the new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The input argument paramDesc is a previously populated tensor descriptor typically - * by first calling miopenGetRNNLayerParam(). - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenSetRNNLayerParam() will return - * a error status miopenStatusBadParm for input paramID associated with the input GEMM. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param wDesc A tensor descriptor to the parameter tensor (input) - * @param w Pointer to memory containing parameter tensor (input) - * @param paramID ID of the internal parameter tensor (input) - * @param paramDesc Descriptor of the parameter tensor (input) - * @param layerParam Pointer to the memory location of the parameter tensor (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetRNNLayerParam(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - miopenTensorDescriptor_t wDesc, - void* w, - const int paramID, - miopenTensorDescriptor_t paramDesc, - const void* layerParam); - -/*! @brief Sets a bias for a specific layer in an RNN stack - * - * This function sets the bias data for a specific layer and bias ID. - * - * For RNN vanilla miopenRNNRELU and miopenRNNTANH, biasID == 0 retrieves the - * weight matrix associated with the in input GEMM, while biasID == 1 retrieves - * the bias associated with the hidden state GEMM. - * - * For miopenLSTM biasID 0 to 3 refer to the biases associated - * with the input GEMM, 4-7 are associated with the biases associated with the - * hidden state GEMM. - * - * * biasID 0 and 4 are for the input gate. - * - * * biasID 1 and 5 are for the forget gate. - * - * * biasID 2 and 6 are for the output gate. - * - * * biasID 3 and 7 are for the new memory gate. - * - * For miopenGRU biasID 0 to 2 refer to the biases associated with the input GEMM, - * while 3 through 5 are associated with the hidden state GEMM. - * - * * biasID 0 and 3 are for the update gate. - * - * * biasID 1 and 4 are for the reset gate. - * - * * biasID 2 and 5 are for the new new memory gate. - * - * For bi-directional RNNs the backwards in time direction is numbered as the layer - * directly after the forward in time direction. - * - * The input argument biasDesc is a previously populated tensor descriptor typically - * by first calling miopenGetRNNLayeBias(). - * - * Note: When inputSkip mode is selected there is no input layer matrix operation, - * and therefore no associated memory. In this case miopenSetRNNLayerBias will return - * a error status miopenStatusBadParm for input biasID associated with the input GEMM. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param layer The layer number in the RNN stack (input) - * @param xDesc A tensor descriptor to input (input) - * @param wDesc A tensor descriptor to the bias tensor (input) - * @param w Pointer to memory containing bias tensor (input) - * @param biasID ID of the internal bias tensor (input) - * @param biasDesc Descriptor of the bias tensor (output) - * @param layerBias Pointer to the memory location of the bias tensor (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetRNNLayerBias(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int layer, - miopenTensorDescriptor_t xDesc, - miopenTensorDescriptor_t wDesc, - void* w, - const int biasID, - miopenTensorDescriptor_t biasDesc, - const void* layerBias); - -/*! @brief Sets a bias for a specific layer in an RNN stack - * - * This function changes padidng mode at previously created and initialized RNN descriptor. - * This function must be called before using miopenGetRNNWorkspaceSize() - * and miopenGetRNNTrainingReserveSize() functions. - * By default, not padded data is expected at the RNN input/output. - * - * @param rnnDesc RNN layer descriptor type (input/output) - * @param paddingMode RNN input/output data padding mode (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetRNNPaddingMode(miopenRNNDescriptor_t rnnDesc, - miopenRNNPaddingMode_t paddingMode); - -/*! @brief This function retrieves the RNN padding mode from the RNN descriptor. - * - * @param rnnDesc RNN layer descriptor type (input) - * @param paddingMode Pointer to the RNN padding mode (output) - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenGetRNNPaddingMode(miopenRNNDescriptor_t rnnDesc, - miopenRNNPaddingMode_t* paddingMode); - -/*! @brief Execute forward training for recurrent layer. - * - * Interface for executing the forward training / inference pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param fwdMode Specifies in which mode the buffers will be used. - * @param xDesc An input tensor descriptor for sequenced RNN data. This - * miopenSeqTensorDescriptor_t should be initialyzed by `miopenSetRNNDataSeqTensorDescriptor` - * function.(input) - * @param x Pointer to input tensor (input) - * - * @param hDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param hy Pointer to the hidden layer output tensor. If hy is NULL, - * then the final hidden state will not be saved. (output) - * - * @param cDesc A cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cx Pointer to the cell layer input tensor. If cx is NULL, - * then the initial cell state will be zero initialized. (input) - * @param cy Pointer to the cell layer output tensor. If hy is NULL, - * then the final cell state will not be saved. (output) - * - * @param yDesc An array of fully packed tensor descriptors associated - * with the output from each time step. The first dimension of the tensor descriptors - * must equal the first dimension of the first descriptor (batch size) in the xDesc - * tensor array. The second dimension of the element of the descriptor array - * depends on the direction mode selected. If the direction mode is unidirectional, - * the second dimension is the hiddenSize. If direction mode is bidirectional - * the second dimension is twice the hiddenSize. (input) - * @param y Pointer to output tensor (output) - * - * @param w Pointer to input weights tensor (input) - * @param weightSpaceSize Number of allocated bytes in memory for the weights tensor - * @param workSpace Pointer to memory allocated for forward (input / output) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for hidden states used durning training - * (input / output) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNForward(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - miopenRNNFWDMode_t fwdMode, - const miopenSeqTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t hDesc, - const void* hx, - void* hy, - const miopenTensorDescriptor_t cDesc, - const void* cx, - void* cy, - const miopenSeqTensorDescriptor_t yDesc, - void* y, - const void* w, - size_t weightSpaceSize, - void* workSpace, - size_t workSpaceNumBytes, - void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute backward data for recurrent layer - * - * Interface for executing the backward data pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - - * @param yDesc An output tensor descriptor for sequenced RNN data. This - * miopenSeqTensorDescriptor_t should be initialyzed by `miopenSetRNNDataSeqTensorDescriptor` - function.(input) - * @param y Pointer to input tensor (input) - * @param dy Pointer to the hidden layer input tensor (input) - * - * @param hDesc An input hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param dhy Pointer to the cell layer input tensor (input) - * @param dhx Pointer to the delta hidden layer output tensor. If dhx is NULL - * the hidden gradient will not ouput. (output) - * - * @param cDesc A input cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cx Pointer to the hidden layer input tensor. If cx is NULL, - * then the initial cell state will be zero initialized. (input) - * @param dcy Pointer to the cell layer input tensor. If dcy is NULL, - * then the initial delta cell state will be zero initialized. (input) - * @param dcx Pointer to the cell layer output tensor. If dcx is NULL - * the cell gradient will not ouput. (output) - - * @param xDesc An input tensor descriptor for sequenced RNN data. This - * miopenSeqTensorDescriptor_t should be initialyzed by `miopenSetRNNDataSeqTensorDescriptor` - function.(input) - * @param dx Pointer to the cell layer output tensor (output) - * - * @param w Pointer to input weights tensor (input) - * @param weightSpaceSize Number of allocated bytes in memory for the weights tensor - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for random states (input / output) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNBackwardSeqData(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const miopenSeqTensorDescriptor_t yDesc, - const void* y, - const void* dy, - const miopenTensorDescriptor_t hDesc, - const void* hx, - const void* dhy, - void* dhx, - const miopenTensorDescriptor_t cDesc, - const void* cx, - const void* dcy, - void* dcx, - const miopenSeqTensorDescriptor_t xDesc, - void* dx, - const void* w, - size_t weightSpaceSize, - void* workSpace, - size_t workSpaceNumBytes, - void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute backward weights for recurrent layer - * - * Interface for executing the backward weights pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - - * @param xDesc An input tensor descriptor for sequenced RNN data. This - * miopenSeqTensorDescriptor_t should be initialyzed by `miopenSetRNNDataSeqTensorDescriptor` - function.(input) - * @param x Pointer to input tensor (input) - * - * @param hDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * - * @param yDesc An output tensor descriptor for sequenced RNN data. This - * miopenSeqTensorDescriptor_t should be initialyzed by `miopenSetRNNDataSeqTensorDescriptor` - function.(input) - * @param y Pointer to the output tensor (input) - * - * @param dw Pointer to input weights tensor (input / output) - * @param weightSpaceSize Number of allocated bytes in memory for the weights tensor - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for random states (input) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenRNNBackwardWeightsSeqTensor(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const miopenSeqTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t hDesc, - const void* hx, - const miopenSeqTensorDescriptor_t yDesc, - const void* y, - void* dw, - size_t weightSpaceSize, - void* workSpace, - size_t workSpaceNumBytes, - const void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute forward training for recurrent layer - * - * Interface for executing the forward training pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Temporal iterations to unroll (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param x Pointer to input tensor (input) - * @param hxDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param cxDesc A cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cx Pointer to the cell layer input tensor. If cx is NULL, - * then the initial cell state will be zero initialized. (input) - * @param wDesc A weights tensor descriptor (input) - * @param w Pointer to input weights tensor (input) - * @param yDesc An array of fully packed tensor descriptors associated - * with the output from each time step. The first dimension of the tensor descriptors - * must equal the first dimension of the first descriptor (batch size) in the xDesc - * tensor array. The second dimension of the element of the descriptor array - * depends on the direction mode selected. If the direction mode is unidirectional, - * the second dimension is the hiddenSize. If direction mode is bidirectional - * the second dimension is twice the hiddenSize. (input) - * @param y Pointer to output tensor (output) - * @param hyDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hy Pointer to the hidden layer output tensor. If hy is NULL, - * then the final hidden state will not be saved. (output) - * @param cyDesc A cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cy Pointer to the cell layer output tensor. If hy is NULL, - * then the final cell state will not be saved. (output) - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for random states (input / output) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNForwardTraining(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* xDesc, - const void* x, - const miopenTensorDescriptor_t hxDesc, - const void* hx, - const miopenTensorDescriptor_t cxDesc, - const void* cx, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenTensorDescriptor_t* yDesc, - void* y, - const miopenTensorDescriptor_t hyDesc, - void* hy, - const miopenTensorDescriptor_t cyDesc, - void* cy, - void* workSpace, - size_t workSpaceNumBytes, - void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute backward data for recurrent layer - * - * Interface for executing the backward data pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Temporal iterations to unroll (input) - * @param yDesc An array of tensor descriptors (input) - * @param y Pointer to input tensor (input) - * @param dyDesc An array of fully packed tensor descriptors associated - * with the output from each time step. The first dimension of the tensor descriptors - * must equal the first dimension of the first descriptor (batch size) in the xDesc - * tensor array. The second dimension of the element of the descriptor array - * depends on the direction mode selected. If the direction mode is unidirectional, - * the second dimension is the hiddenSize. If direction mode is bidirectional - * the second dimension is twice the hiddenSize. (input) - * @param dy Pointer to the hidden layer input tensor (input) - * @param dhyDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param dhy Pointer to the cell layer input tensor (input) - * @param dcyDesc A cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param dcy Pointer to the cell layer input tensor. If dcy is NULL, - * then the initial delta cell state will be zero initialized. (input) - * @param wDesc A weights tensor descriptor (input) - * @param w Pointer to input weights tensor (input) - * @param hxDesc An input hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param cxDesc A input cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cx Pointer to the hidden layer input tensor. If cx is NULL, - * then the initial cell state will be zero initialized. (input) - * @param dxDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param dx Pointer to the cell layer output tensor (output) - * @param dhxDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param dhx Pointer to the delta hidden layer output tensor. If dhx is NULL - * the hidden gradient will not ouput. (output) - * @param dcxDesc A tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param dcx Pointer to the cell layer output tensor. If dcx is NULL - * the cell gradient will not ouput. (output) - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for random states (input / output) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNBackwardData(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* yDesc, - const void* y, - const miopenTensorDescriptor_t* dyDesc, - const void* dy, - const miopenTensorDescriptor_t dhyDesc, - const void* dhy, - const miopenTensorDescriptor_t dcyDesc, - const void* dcy, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenTensorDescriptor_t hxDesc, - const void* hx, - const miopenTensorDescriptor_t cxDesc, - const void* cx, - const miopenTensorDescriptor_t* dxDesc, - void* dx, - const miopenTensorDescriptor_t dhxDesc, - void* dhx, - const miopenTensorDescriptor_t dcxDesc, - void* dcx, - void* workSpace, - size_t workSpaceNumBytes, - void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute backward weights for recurrent layer - * - * Interface for executing the backward weights pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Temporal iterations to unroll (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param x Pointer to input tensor (input) - * @param hxDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param yDesc An array of fully packed tensor descriptors associated - * with the output from each time step. The first dimension of the tensor descriptors - * must equal the first dimension of the first descriptor (batch size) in the xDesc - * tensor array. The second dimension of the element of the descriptor array - * depends on the direction mode selected. If the direction mode is unidirectional, - * the second dimension is the hiddenSize. If direction mode is bidirectional - * the second dimension is twice the hiddenSize. (input) - * @param y Pointer to the output tensor (input) - * @param dwDesc A weights tensor descriptor (input) - * @param dw Pointer to input weights tensor (input / output) - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @param reserveSpace Pointer to memory allocated for random states (input) - * @param reserveSpaceNumBytes Number of allocated bytes in memory for use in the forward (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNBackwardWeights(miopenHandle_t handle, - const miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* xDesc, - const void* x, - const miopenTensorDescriptor_t hxDesc, - const void* hx, - const miopenTensorDescriptor_t* yDesc, - const void* y, - const miopenTensorDescriptor_t dwDesc, - void* dw, - void* workSpace, - size_t workSpaceNumBytes, - const void* reserveSpace, - size_t reserveSpaceNumBytes); - -/*! @brief Execute forward inference for RNN layer - * - * Interface for executing the forward inference pass on a RNN. - * - * @param handle MIOpen handle (input) - * @param rnnDesc RNN layer descriptor type (input) - * @param sequenceLen Temporal iterations to unroll (input) - * @param xDesc An array of tensor descriptors. These are the - * input descriptors to each time step. The first dimension of each descriptor is the - * batch size and may decrease from element n to element n+1 and not increase in size. - * The second dimension is the same for all descriptors in the array and is the input - * vector length. (input) - * @param x Pointer to input tensor (input) - * @param hxDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hx Pointer to the hidden layer input tensor. If hx is NULL, - * then the initial hidden state will be zero initialized. (input) - * @param cxDesc A cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cx Pointer to the cell layer input tensor. If cx is NULL, - * then the initial cell state will be zero initialized. (input) - * @param wDesc A weights tensor descriptor (input) - * @param w Pointer to input weights tensor (input) - * @param yDesc An array of fully packed tensor descriptors associated - * with the output from each time step. The first dimension of the tensor descriptors - * must equal the first dimension of the first descriptor (batch size) in the xDesc - * tensor array. The second dimension of the element of the descriptor array - * depends on the direction mode selected. If the direction mode is unidirectional, - * the second dimension is the hiddenSize. If direction mode is bidirectional - * the second dimension is twice the hiddenSize. (input) - * @param y Pointer to output tensor (output) - * @param hyDesc A hidden tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param hy Pointer to the hidden layer output tensor. If hy is NULL, - * then the final hidden state will not be saved. (output) - * @param cyDesc A output cell tensor descriptor that has as its first dimension - * of the number of layers if the direction mode is unidirectional and twice the - * number of layers if the direction mode is bidirectional. The second dimension of - * the descriptor must equal the largest first dimension of the xDesc tensor descriptor - * array. The third dimension equals the hiddenSize. (input) - * @param cy Pointer to the cell layer output tensor. If cy is NULL, - * then the final cell state will not be saved. (output) - * @param workSpace Pointer to memory allocated for forward training (input) - * @param workSpaceNumBytes Number of allocated bytes in memory for the workspace (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRNNForwardInference(miopenHandle_t handle, - miopenRNNDescriptor_t rnnDesc, - const int sequenceLen, - const miopenTensorDescriptor_t* xDesc, - const void* x, - const miopenTensorDescriptor_t hxDesc, - const void* hx, - const miopenTensorDescriptor_t cxDesc, - const void* cx, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenTensorDescriptor_t* yDesc, - void* y, - const miopenTensorDescriptor_t hyDesc, - void* hy, - const miopenTensorDescriptor_t cyDesc, - void* cy, - void* workSpace, - size_t workSpaceNumBytes); - -/** @} */ -// CLOSEOUT RNN DOXYGEN GROUP - -/** @addtogroup LossFunction - * - * @{ - */ - -/*! @enum miopenCTCLossAlgo_t - * Algorithms available to execute the CTC loss operation - */ -typedef enum -{ - MIOPEN_CTC_LOSS_ALGO_DETERMINISTIC = 0, /*!< Results are guaranteed to be reproducible */ -} miopenCTCLossAlgo_t; - -/*! @brief Create a CTC loss function Descriptor - * - * API for creating an uninitialized CTC loss function descriptor. - * @param ctcLossDesc Pointer to the CTC loss function descriptor type (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateCTCLossDescriptor(miopenCTCLossDescriptor_t* ctcLossDesc); - -/*! @brief Retrieves a CTC loss function descriptor's details - * - * @param ctcLossDesc CTC loss function descriptor (input) - * @param dataType Data type used in this CTC loss operation, only fp32 currently - * supported (output) - * @param blank_label_id User defined index for blank label (output) - * @param apply_softmax_layer Boolean to toggle input layer property (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetCTCLossDescriptor(miopenCTCLossDescriptor_t ctcLossDesc, - miopenDataType_t* dataType, - int* blank_label_id, - bool* apply_softmax_layer); - -/*! @brief Destroys a CTC loss function descriptor object - * - * @param ctcLossDesc CTC loss function descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyCTCLossDescriptor(miopenCTCLossDescriptor_t ctcLossDesc); - -/*! @brief Set the details of a CTC loss function descriptor - * - * @param ctcLossDesc CTC loss function descriptor type (input) - * @param dataType Data type used in this CTC loss operation, only fp32 currently - * supported (input) - * @param blank_label_id User defined index for blank label, default 0 (input) - * @param apply_softmax_layer Boolean to toggle input layer property (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetCTCLossDescriptor(miopenCTCLossDescriptor_t ctcLossDesc, - miopenDataType_t dataType, - const int blank_label_id, - bool apply_softmax_layer); - -/*! @brief Query the amount of memory required to execute miopenCTCLoss - * - * This function calculates the amount of memory required to run the CTC loss function given a CTC - * loss function descriptor with the specified algorithm. - * @param handle MIOpen handle (input) - * @param probsDesc Tensor descriptor for probabilities (input) - * @param gradientsDesc Tensor descriptor for gradients (input) - * @param labels Pointer to the flattened labels list (input) - * @param labelLengths Pointer to the lengths list for "labels" (input) - * @param inputLengths Pointer to the list of the time steps in each batch (input) - * @param algo CTC loss algorithm selected (input) - * @param ctcLossDesc CTC loss function descriptor type (input) - * @param workSpaceSize Number of bytes of workspace required for CTC loss operation with selected - * algorithm (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetCTCLossWorkspaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t probsDesc, - const miopenTensorDescriptor_t gradientsDesc, - const int* labels, - const int* labelLengths, - const int* inputLengths, - miopenCTCLossAlgo_t algo, - const miopenCTCLossDescriptor_t ctcLossDesc, - size_t* workSpaceSize); - -/*! @brief Execute forward inference for CTCLoss layer - * - * Interface for executing the forward inference pass on a CTCLoss. - * @param handle MIOpen handle (input) - * @param probsDesc Tensor descriptor for probabilities (input) - * @param probs Pointer to the probabilities tensor (input) - * @param labels Pointer to the flattened labels list (input) - * @param labelLengths Pointer to the lengths list for "labels" (input) - * @param inputLengths Pointer to the list of the time steps in each batch (input) - * @param losses Pointer to the computed losses of CTC (Output) - * @param gradientsDesc Tensor descriptor for gradients (input) - * @param gradients Pointer to the computed gradients of CTC (Output) - * @param algo CTC loss algorithm selected (input) - * @param ctcLossDesc CTC loss function descriptor type (input) - * @param workSpace Pointer to memory allocated for execute CTC loss operation (input) - * @param workSpaceSize Number of bytes of workspace required for CTC loss operation with selected - * algorithm (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCTCLoss(miopenHandle_t handle, - const miopenTensorDescriptor_t probsDesc, - const void* probs, - const int* labels, - const int* labelLengths, - const int* inputLengths, - void* losses, - const miopenTensorDescriptor_t gradientsDesc, - void* gradients, - miopenCTCLossAlgo_t algo, - const miopenCTCLossDescriptor_t ctcLossDesc, - void* workSpace, - size_t workSpaceSize); - -/** @} */ -// CLOSEOUT LossFunction DOXYGEN GROUP - -// Dropout APIs -/** @addtogroup dropout - * - * @{ - */ - -/*! @enum miopenRNGType_t - * random number generator type - */ -typedef enum -{ - MIOPEN_RNG_PSEUDO_XORWOW = 0, /*!< XORWOW pseudorandom generator */ -} miopenRNGType_t; - -/*! @brief Creates the dropout descriptor object - * - * @param dropoutDesc Pointer to a dropout descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateDropoutDescriptor(miopenDropoutDescriptor_t* dropoutDesc); - -/*! @brief Destroys the dropout descriptor object - * - * @param dropoutDesc Dropout descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc); - -/*! @brief Query the amount of memory required to run dropout - * - * This function calculates the amount of memory required to run dropout. - * @param xDesc Tensor descriptor for data tensor x (input) - * @param reserveSpaceSizeInBytes Number of bytes of reservespace required for executing dropout - * (Output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDropoutGetReserveSpaceSize(const miopenTensorDescriptor_t xDesc, - size_t* reserveSpaceSizeInBytes); - -/*! @brief Query the amount of memory required to store the states of the random number generators - * - * This function calculates the amount of memory required to store the states of the random number - * generators used by miopenDropoutForward. - * @param handle MIOpen handle (input) - * @param stateSizeInBytes Number of bytes required to store random generator states (Output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDropoutGetStatesSize(miopenHandle_t handle, - size_t* stateSizeInBytes); - -/*! @brief Get the details of the dropout descriptor - * - * Interface for querying the dropout descriptor - * @param dropoutDesc Dropout layer descriptor (input) - * @param handle MIOpen handle (input) - * @param dropout The probability by which the input is set to 0 in the dropout layer (Output) - * @param states Pointer to memory that holds random number generator states (Output) - * @param seed Seed used to initialize random number generator states (Output) - * @param use_mask Boolean flag indicating whether to use a saved mask (an existing or - * user-defined dropout layout) in reserveSpace (Output) - * @param state_evo Boolean flag indicating whether to adopt state evolution strategy to update - * the PRNG states by the end of each implementation (Output placeholder, currently not enabled) - * @param rng_mode Random number generator used to generate parallel random number sequences - * (Output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t handle, - float* dropout, - void** states, - unsigned long long* seed, - bool* use_mask, - bool* state_evo, - miopenRNGType_t* rng_mode); - -/*! @brief Restore the dropout descriptor to a saved state - * - * This function restores the state of dropout descriptor using the address of a state buffer with - * previously saved PRNG state pattern, without launching the expensive PRNG initialization process. - * - * Interface for restoring the dropout descriptor - * @param dropoutDesc Dropout layer descriptor (input/Output) - * @param handle MIOpen handle (input) - * @param dropout The probability by which the input is set to 0 in the dropout layer - * (input) - * @param states Pointer to memory that holds random number generator states (input) - * @param stateSizeInBytes Number of bytes holding random generator states (input) - * @param seed Seed used to initialize random number generator states (input) - * @param use_mask Boolean flag indicating whether to use a saved mask (an existing or - * user-defined dropout layout) in reserveSpace (input) - * @param state_evo Boolean flag indicating whether to adopt state evolution strategy to - * update the PRNG states by the end of each implementation (input placeholder, currently not - * enabled) - * @param rng_mode Random number generator used to generate parallel random number - * sequences (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRestoreDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t handle, - float dropout, - void* states, - size_t stateSizeInBytes, - unsigned long long seed, - bool use_mask, - bool state_evo, - miopenRNGType_t rng_mode); - -/*! @brief Initialize the dropout descriptor - * - * Interface for setting up the dropout descriptor - * @param dropoutDesc Dropout layer descriptor (input/Output) - * @param handle MIOpen handle (input) - * @param dropout The probability by which the input is set to 0 in the dropout layer - * (input) - * @param states Pointer to memory that holds random number generator states (input) - * @param stateSizeInBytes Number of bytes provided for random generator states (input) - * @param seed Seed used to initialize random number generator states (input) - * @param use_mask Boolean flag indicating whether to use a saved mask (an existing or - * user-defined dropout layout) in reserveSpace (input) - * @param state_evo Boolean flag indicating whether to adopt state evolution strategy to - * update the PRNG states by the end of each implementation (input placeholder, currently not - * enabled) - * @param rng_mode Random number generator used to generate parallel random number - * sequences (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t handle, - float dropout, - void* states, - size_t stateSizeInBytes, - unsigned long long seed, - bool use_mask, - bool state_evo, - miopenRNGType_t rng_mode); - -/*! @brief Execute forward dropout operation - * - * Interface for executing the forward pass on a Dropout. - * @param handle MIOpen handle (input) - * @param dropoutDesc Dropout layer descriptor (input) - * @param noise_shape Tensor descriptor for noise shape (input placeholder, currently - * not enabled) - * @param xDesc Tensor descriptor for data tensor x (input) - * @param x Data tensor x (input) - * @param yDesc Tensor descriptor for data tensor y (input) - * @param y Data tensor y (Output) - * @param reserveSpace Pointer to memory allocated for executing forward dropout, - * expecting reserveSpace unchanged before next call of miopenDropoutBackward (Output) - * @param reserveSpaceSizeInBytes Number of bytes of reservespace required for executing forward - * dropout (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDropoutForward(miopenHandle_t handle, - const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t noise_shape, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t yDesc, - void* y, - void* reserveSpace, - size_t reserveSpaceSizeInBytes); - -/*! @brief Execute backward dropout operation - * - * Interface for executing the backward pass on a Dropout. - * @param handle MIOpen handle (input) - * @param dropoutDesc Dropout layer descriptor (input) - * @param noise_shape Tensor descriptor for noise shape (input placeholder, currently - * not enabled) - * @param dyDesc Tensor descriptor for data delta tensor dy (input) - * @param dy Data delta tensor dy (input) - * @param dxDesc Tensor descriptor for data delta tensor dx (input) - * @param dx Data delta tensor dx (Output) - * @param reserveSpace Pointer to memory allocated for executing backward dropout, - * expecting reserveSpace unchanged after previous call of miopenDropoutForward (input) - * @param reserveSpaceSizeInBytes Number of bytes of reservespace required for executing backward - * dropout (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDropoutBackward(miopenHandle_t handle, - const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t noise_shape, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t dxDesc, - void* dx, - void* reserveSpace, - size_t reserveSpaceSizeInBytes); - -/** @} */ -// CLOSEOUT DROPOUT DOXYGEN GROUP - -// TensorReduce APIs -/** @addtogroup TensorReduce - * - * @{ - */ - -/*! @brief Creates the ReduceTensor descriptor object - * - * @param reduceTensorDesc Pointer to a ReduceTensor descriptor type - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateReduceTensorDescriptor(miopenReduceTensorDescriptor_t* reduceTensorDesc); - -/*! @brief Destroy the ReduceTensor descriptor object - * - * @param reduceTensorDesc ReduceTensor descriptor type (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenDestroyReduceTensorDescriptor(miopenReduceTensorDescriptor_t reduceTensorDesc); - -/*! @brief Initialize a ReduceTensor descriptor object - * - * @param reduceTensorDesc Pointer to the ReduceTensor descriptor object (output) - * @param reduceTensorOp Enumerant specifying the operation used by ReduceTensor (input) - * @param reduceTensorCompType Enumerant specifying the data type used with ReduceTensor - * operation (input) - * @param reduceTensorNanOpt Enumerant specifying the Nan number propagation mode (input) - * @param reduceTensorIndices Enumerant specifying the indices modes used by ReduceTensor - * (input) - * @param reduceTensorIndicesType Enumerant specifying the data type of the indices (input) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetReduceTensorDescriptor(miopenReduceTensorDescriptor_t reduceTensorDesc, - miopenReduceTensorOp_t reduceTensorOp, - miopenDataType_t reduceTensorCompType, - miopenNanPropagation_t reduceTensorNanOpt, - miopenReduceTensorIndices_t reduceTensorIndices, - miopenIndicesType_t reduceTensorIndicesType); - -/*! @brief Query a ReduceTensor descriptor object - * - * @param reduceTensorDesc Pointer to the ReduceTensor descriptor object (input) - * @param reduceTensorOp Pointer to enumerant specifying the operation used by - * ReduceTensor (output) - * @param reduceTensorCompType Pointer to enumerant specifying the data type used with - * ReduceTensor operation (output) - * @param reduceTensorNanOpt Pointer to enumerant specifying the Nan number propagation mode - * (output) - * @param reduceTensorIndices Pointer to enumerant specifying the indices modes used by - * ReduceTensor (output) - * @param reduceTensorIndicesType Pointer to enumerant specifying the data type of the indices - * (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetReduceTensorDescriptor(const miopenReduceTensorDescriptor_t reduceTensorDesc, - miopenReduceTensorOp_t* reduceTensorOp, - miopenDataType_t* reduceTensorCompType, - miopenNanPropagation_t* reduceTensorNanOpt, - miopenReduceTensorIndices_t* reduceTensorIndices, - miopenIndicesType_t* reduceTensorIndicesType); - -/*! @brief Helper function to query the minimum index space size required by the ReduceTensor call - * - * @param handle MIOpen Handle (input) - * @param reduceTensorDesc Pointer to the ReduceTensor descriptor object (input) - * @param aDesc Pointer to the input tensor descriptor (input) - * @param cDesc Pointer to the output tensor descriptor (input) - * @param sizeInBytes Pointer to data to return the minimum index space size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetReductionIndicesSize(miopenHandle_t handle, - const miopenReduceTensorDescriptor_t reduceTensorDesc, - const miopenTensorDescriptor_t aDesc, - const miopenTensorDescriptor_t cDesc, - size_t* sizeInBytes); - -/*! @brief Helper function to query the minimum workspace size required by the ReduceTensor call - * - * @param handle MIOpen Handle (input) - * @param reduceTensorDesc Pointer to the ReduceTensor descriptor object (input) - * @param aDesc Pointer to the input tensor descriptor (input) - * @param cDesc Pointer to the output tensor descriptor (input) - * @param sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetReductionWorkspaceSize(miopenHandle_t handle, - const miopenReduceTensorDescriptor_t reduceTensorDesc, - const miopenTensorDescriptor_t aDesc, - const miopenTensorDescriptor_t cDesc, - size_t* sizeInBytes); - -/*! @brief TensorReduce function doing reduction on tensor A by implementing C = alpha * reduceOp(A) - * + beta * C - * - * The length of each dimension of output tensor C must match the length of the corresponding - * dimension of - * input tensor A or must be equal to 1. The dimensions with length equal to 1 indicate the - * dimensions - * of A to be reduced. - * - * @param handle MIOpen Handle (input) - * @param reduceTensorDesc Pointer to the ReduceTensor descriptor object (input) - * @param indices Address of the allocated indices data space (output) - * @param indicesSizeInBytes Size in bytes of the allocated indices data space (input) - * @param workspace Address of the allocated workspace data (input) - * @param workspaceSizeInBytes Size in bytes of the allocated workspace data (input) - * @param alpha Pointer to scale factor for data in input tensor A (input) - * @param aDesc Pointer to the tensor descriptor for input tensor A (input) - * @param A Pointer to the data of input tensor A (input) - * @param beta Pointer to scale factor for data in output tensor C (input) - * @param cDesc Pointer to the tensor descriptor for output tensor C (input) - * @param C Pointer to the data of output tensor C (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenReduceTensor(miopenHandle_t handle, - const miopenReduceTensorDescriptor_t reduceTensorDesc, - void* indices, - size_t indicesSizeInBytes, - void* workspace, - size_t workspaceSizeInBytes, - const void* alpha, - const miopenTensorDescriptor_t aDesc, - const void* A, - const void* beta, - const miopenTensorDescriptor_t cDesc, - void* C); - -/** @} */ -// CLOSEOUT TensorReduce DOXYGEN GROUP - -// Find 2.0 API -/** @addtogroup find2 - * - * @{ - */ - -/*! @brief Describes a problem for different miopen operations. - * - * For now, this is only used for convolution, but could be used for other - * operators in the future(such as GEMM, Pooling, etc) - */ -MIOPEN_DECLARE_OBJECT(miopenProblem); - -/*! @enum miopenProblemDirection_t - * Directions of miopen operation. - */ -typedef enum -{ - miopenProblemDirectionForward = 0, - miopenProblemDirectionBackward = 1, - miopenProblemDirectionBackwardWeights = 2, -#ifdef MIOPEN_BETA_API - miopenProblemDirectionInference = 4, -#endif -} miopenProblemDirection_t; - -/*! @enum miopenTensorArgumentId_t - * Identifiers for tensor arguments of problems and operations. - */ -typedef enum -{ - miopenTensorArgumentIdInvalid = 0, - miopenTensorConvolutionX = 1, - miopenTensorConvolutionW = 2, - miopenTensorConvolutionY = 3, - - miopenTensorMhaK = 4, - miopenTensorMhaQ = 5, - miopenTensorMhaV = 6, - miopenTensorMhaDescaleK = 7, - miopenTensorMhaDescaleQ = 8, - miopenTensorMhaDescaleV = 9, - miopenTensorMhaDescaleS = 10, - miopenTensorMhaScaleS = 11, - miopenTensorMhaScaleO = 12, - miopenTensorMhaDropoutProbability = 13, - miopenTensorMhaDropoutSeed = 14, - miopenTensorMhaDropoutOffset = 15, - miopenTensorMhaO = 16, - miopenTensorMhaAmaxO = 17, - miopenTensorMhaAmaxS = 18, - miopenTensorMhaM = 19, - miopenTensorMhaZInv = 20, - miopenTensorMhaDO = 21, - miopenTensorMhaDescaleO = 22, - miopenTensorMhaDescaleDO = 23, - miopenTensorMhaDescaleDS = 24, - miopenTensorMhaScaleDS = 25, - miopenTensorMhaScaleDQ = 26, - miopenTensorMhaScaleDK = 27, - miopenTensorMhaScaleDV = 28, - miopenTensorMhaDQ = 29, - miopenTensorMhaDK = 30, - miopenTensorMhaDV = 31, - miopenTensorMhaAmaxDQ = 32, - miopenTensorMhaAmaxDK = 33, - miopenTensorMhaAmaxDV = 34, - miopenTensorMhaAmaxDS = 35, - miopenTensorMhaBias = 36, - -#ifdef MIOPEN_BETA_API - miopenTensorActivationX = 37, - miopenTensorActivationY = 38, - miopenTensorActivationDX = 39, - miopenTensorActivationDY = 40, - miopenTensorBiasX = 41, - miopenTensorBiasY = 42, - miopenTensorBias = 43, - miopenTensorSoftmaxX = 44, - miopenTensorSoftmaxY = 45, - miopenTensorSoftmaxDX = 46, - miopenTensorSoftmaxDY = 47, - miopenTensorBatchnormX = 48, - miopenTensorBatchnormY = 49, - miopenTensorBatchnormRunningMean = 50, - miopenTensorBatchnormRunningVariance = 51, - miopenTensorBatchnormSavedMean = 52, - miopenTensorBatchnormSavedVariance = 53, - miopenTensorBatchnormScale = 54, - miopenTensorBatchnormScaleDiff = 55, - miopenTensorBatchnormEstimatedMean = 56, - miopenTensorBatchnormEstimatedVariance = 57, - miopenTensorBatchnormBias = 58, - miopenTensorBatchnormBiasDiff = 59, - miopenTensorBatchnormDX = 60, - miopenTensorBatchnormDY = 61, -#endif - - miopenTensorArgumentIsScalar = 1U << 31, - - miopenTensorMhaMask = miopenTensorArgumentIsScalar | 1, -#ifdef MIOPEN_BETA_API - miopenScalarBatchnormExpAvgFactor = miopenTensorArgumentIsScalar | 2, - miopenScalarBatchnormEpsilon = miopenTensorArgumentIsScalar | 3, -#endif -} miopenTensorArgumentId_t; - -/*! @enum miopenTensorArgumentId_t - * Different ways to sort results of the find call. - */ -typedef enum -{ - miopenFindResultsOrderByTime = 0, - miopenFindResultsOrderByWorkspaceSize = 1, -} miopenFindResultsOrder_t; - -/*! @brief Initializes a problem object describing a convolution operation. - * - * @param problem Pointer to the problem to initialize - * @param operatorDesc Descriptor of the operator to be used - * @param direction Direction of the operation - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateConvProblem(miopenProblem_t* problem, - miopenConvolutionDescriptor_t operatorDesc, - miopenProblemDirection_t direction); - -/*! @brief Initializes a problem object describing a Mha operation. - * - * @param problem Pointer to the problem to initialize - * @param operatorDesc Descriptor of the operator to be used - * @param direction Direction of the operation - * @return miopenStatus_t - */ - -/*! @enum miopenMhaMask_t - * Different masks for Mha. - */ -typedef enum -{ - miopenMhaMaskNone = 0, - miopenMhaMaskCausal = 1, -} miopenMhaMask_t; - -MIOPEN_EXPORT miopenStatus_t miopenCreateMhaProblem(miopenProblem_t* problem, - miopenMhaDescriptor_t operatorDesc, - miopenProblemDirection_t direction); - -/*! @brief Creates the mha descriptor object - * - * @param mhaDesc Pointer to a mha descriptor type - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenCreateMhaDescriptor(miopenMhaDescriptor_t* mhaDesc); - -/*! @brief Sets the Mha descriptor details - * - * Sets all of the descriptor details for the Mha - * - * @param mhaDesc Pointer to a Mha descriptor - * @param scale Scale - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenSetMhaDescriptor(miopenMhaDescriptor_t mhaDesc, float scale); - -/*! @brief Gets the Mha descriptor details - * - * Retrieves all of the descriptor details for the Mha. - * - * @param mhaDesc Pointer to a Mha descriptor - * @param scale Scale (output) - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenGetMhaDescriptor(miopenMhaDescriptor_t mhaDesc, float* scale); - -/*! @brief Creates the Softmax descriptor object - * - * @param softmaxDesc Pointer to an softmax descriptor type - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenCreateSoftmaxDescriptor(miopenSoftmaxDescriptor_t* softmaxDesc); - -/*! @brief Sets the softmax descriptor details - * - * Sets all of the descriptor details for the softmax layer - * - * @param softmaxDesc Pointer to a softmax layer descriptor - * @param alpha Softmax alpha parameter - * @param beta Softmax beta parameter - * @param algorithm Softmax algorithm - * @param mode Softmax mode - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetSoftmaxDescriptor(miopenSoftmaxDescriptor_t softmaxDesc, - float alpha, - float beta, - miopenSoftmaxAlgorithm_t algorithm, - miopenSoftmaxMode_t mode); - -/*! @brief Gets the softmax layer descriptor details - * - * Retrieves all of the descriptor details for the softmax layer. - * - * @param softmaxDesc Pointer to a softmax layer descriptor (input) - * @param alpha Softmax alpha parameter (output) - * @param beta Softmax beta parameter (output) - * @param algorithm Softmax algorithm (output) - * @param mode Softmax mode (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSoftmaxDescriptor(const miopenSoftmaxDescriptor_t softmaxDesc, - float* alpha, - float* beta, - miopenSoftmaxAlgorithm_t* algorithm, - miopenSoftmaxMode_t* mode); - -/*! @brief Destroys a problem object. - * - * @param problem Problem to destroy - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyProblem(miopenProblem_t problem); - -/*! @brief Sets a tensor descriptor for the specified argument. - * - * @param problem Problem to update - * @param id Id of the argument for the descriptor - * @param descriptor Tensor descriptor to set - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenSetProblemTensorDescriptor(miopenProblem_t problem, - miopenTensorArgumentId_t id, - const miopenTensorDescriptor_t descriptor); - -/*! @brief The miopenFindOptions allows the user to configure how find will be used. - */ -MIOPEN_DECLARE_OBJECT(miopenFindOptions); - -/*! @brief Initializes miopenFindOptions object. - * - * @param options Pointer to options object to initialze - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateFindOptions(miopenFindOptions_t* options); - -/*! @brief Destroys miopenFindOptions object. - * - * @param options Options object to destroy - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroyFindOptions(miopenFindOptions_t options); - -/*! @brief Sets the tuning find option. Default value is zero. - * - * @param options Options object to update - * @param value Value of zero means no tuning, value of one means tuning enabled - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionTuning(miopenFindOptions_t options, int value); - -/*! @brief Sets the results order find option. Default value is miopenFindResultsOrderByTime. - * - * @param options Options object to update - * @param value Specifies what order should find results have - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionResultsOrder(miopenFindOptions_t options, - miopenFindResultsOrder_t value); - -/*! @brief Sets the workspace limit find option. Default value is maximum of size_t - * - * @param options Options object to update - * @param value Specifies the workspace limit for find call. All solvers exceeding the limit - * would be ignored. - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionWorkspaceLimit(miopenFindOptions_t options, - size_t value); - -/*! @brief Attaches the preallocated workspace to find options. Allocated by the library by default. - * - * @param options Options object to update - * @param buffer Specifies the workspace for find call - * @param size Specifies the size of the buffer passed - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionPreallocatedWorkspace(miopenFindOptions_t options, - void* buffer, - size_t size); - -/*! @brief Attaches a preallocated tensor to find options. If not used, buffers are allocated by - * MIOpen internally, which is not recommended. - * - * @param options Options object to update - * @param id Specifies the id of the tensor passed - * @param buffer Specifies the tensor for find call - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionPreallocatedTensor(miopenFindOptions_t options, - miopenTensorArgumentId_t id, - void* buffer); - -/*! @brief Forces library to attach kernel binaries to solutions for later saving. This allows zero - * lookup miopenRunSolution calls after miopenLoadSolution. Default value is 0. - * - * @param options Options object to update - * @param attach 1 means attaching, 0 - skipping, any other value - reserved for future use - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSetFindOptionAttachBinaries(miopenFindOptions_t options, - unsigned attach); - -/*! @brief The miopenSolution object describes a prepared solution. - */ -MIOPEN_DECLARE_OBJECT(miopenSolution); - -/*! @brief Finds solutions to a problem by running different applicable solutions. Memory is - * automatically allocated. - * - * @param handle Handle to execute the kernels - * @param problem Problem to solve - * @param options Find options. When null default values would be used - * @param solutions Pointer to the first result. Must not be null - * @param numSolutions Pointer to the amount of results. Ignored if null - * @param maxSolutions Limits the amount of results - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenFindSolutions(miopenHandle_t handle, - miopenProblem_t problem, - miopenFindOptions_t options, - miopenSolution_t* solutions, - size_t* numSolutions, - size_t maxSolutions); - -/*! @brief Values of a tensor or scalar argument for the miopenRunSolution function. - */ -struct miopenTensorArgument_t -{ - /* @brief Identifier of the tensor argument. - */ - miopenTensorArgumentId_t id; - /* @brief Tensor descriptor to override the value stored in the solution. - * - * Some solvers may support overriding input and output tensor descriptors, but right now there - * is no way to tell from the API. Intended for the future use. - */ - miopenTensorDescriptor_t* descriptor; - /* @brief Pointer to the device memory buffer to use for the operation or to the host memory if - * the value is scalar. - */ - void* buffer; -}; - -/*! @brief Runs the solution using the passed in buffers. - * - * @param handle Handle to execute the kernels - * @param solution Solution to execute - * @param nInputs Amount to inputs for the solution - * @param tensors Tensor arguments described by miopenTensorArgument_t - * @param workspace Pointer to device buffer used as workspace. May be null when not required. - * Should not be less than expected - * @param workspaceSize Size of the workspace buffer - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRunSolution(miopenHandle_t handle, - miopenSolution_t solution, - size_t nInputs, - const miopenTensorArgument_t* tensors, - void* workspace, - size_t workspaceSize); - -/*! @brief Destroys solution object. - * - * @param solution Solution to destroy - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenDestroySolution(miopenSolution_t solution); - -/*! @brief Loads solution object from binary data. - * - * @param solution Pointer to the solution to load - * @param data Data to load the solution from - * @param size Size of the solution blob - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenLoadSolution(miopenSolution_t* solution, - const char* data, - size_t size); - -/*! @brief Saves a solution object as binary data. - * - * @param solution Solution to save - * @param data Pointer to a buffer to save soltuion to - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenSaveSolution(miopenSolution_t solution, char* data); - -/*! @brief Reads the expected size of a solution. - * - * @param solution Solution to get size - * @param size Pointer to a location where to write the size of the solution blob - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSolutionSize(miopenSolution_t solution, size_t* size); - -/*! @brief Reads the amount of workspace required to exectute the solution. - * - * @param solution Solution to get required workspace size - * @param workspaceSize Pointer to a location where to write the workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSolutionWorkspaceSize(miopenSolution_t solution, - size_t* workspaceSize); - -/*! @brief Reads the time spent to execute the solution the last it was run. - * - * @param solution Solution to get exection time - * @param time Pointer to a location where to write the execution time - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSolutionTime(miopenSolution_t solution, float* time); - -/*! @brief Reads id of the solver referred by the solution. - * - * @param solution Solution to get solver id from - * @param solverId Pointer to a location where to write the solver id - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSolutionSolverId(miopenSolution_t solution, - uint64_t* solverId); - -/*! @brief Gets the convolution algorithm implemented by a solver. - * - * @param solverId Solver id to get convolution algorithm of - * @param result Pointer to a location where to write the algorithm - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetSolverIdConvAlgorithm(uint64_t solverId, - miopenConvAlgorithm_t* result); - -#ifdef MIOPEN_BETA_API - -/*! @brief Initializes a problem object describing an activation operation. - * @note As of now there is no way to actually get any solution for this kind of problems. - * - * @param problem Pointer to the problem to initialize - * @param operatorDesc Descriptor of the operator to be used - * @param direction Direction of the operation - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenCreateActivationProblem(miopenProblem_t* problem, - miopenActivationDescriptor_t operatorDesc, - miopenProblemDirection_t direction); - -/*! @brief Initializes a problem object describing an activation operation. - * @note As of now there is no way to actually get any solution for this kind of problems. - * - * @param problem Pointer to the problem to initialize - * @param mode Batchnorm mode - * @param direction Direction of the operation - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateBatchnormProblem(miopenProblem_t* problem, - miopenBatchNormMode_t mode, - bool runningMeanVariance, - miopenProblemDirection_t direction); - -/*! @brief Fuse two problems into a single one. Problems can be either regular, or fused. No - * problems are disposed in the process, so the problem2 should be destroyed manually if it is not - * needed anymore. - * @example - * miopenProblem_t problem = makeSomeProblem1(); - * miopenProblem_t problem2 = makeSomeProblem2(); - * miopenProblem_t problem3 = makeSomeProblem3(); - * miopenFuseProblems(problem, problem2); - * // Now problem contains {problem1, problem2} - * miopenFuseProblems(problem, problem3); - * // Now problem contains {problem1, problem2, problem3} - * miopenDestroyProblem(problem2); - * miopenDestroyProblem(problem3); - * @note As of now there is no way to actually get any solution for this kind of problems. - * - * @param problem1 The first problem to fuse. The result would be stored here. - * @param problem2 The second problem to fuse. - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenFuseProblems(miopenProblem_t problem1, miopenProblem_t problem2); - -/*! @brief Initializes a problem object describing an bias operation. - * @note As of now there is no way to actually get any solution for this kind of problems. - * - * @param problem Pointer to the problem to initialize - * @param direction Direction of the operation - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenCreateBiasProblem(miopenProblem_t* problem, - miopenProblemDirection_t direction); - -/*! @brief Initializes a problem object describing a softmax operation. - * - * @param problem Pointer to the problem to initialize - * @param operatorDesc Descriptor of the operator to be used - * @param direction Direction of the operation - * @return miopenStatus_t - */ - -MIOPEN_EXPORT miopenStatus_t miopenCreateSoftmaxProblem(miopenProblem_t* problem, - miopenSoftmaxDescriptor_t operatorDesc, - miopenProblemDirection_t direction); - -#endif - -/** @} */ -// CLOSEOUT find2 DOXYGEN GROUP - -#ifdef MIOPEN_BETA_API - -/*! @ingroup ReduceCalculation - * @enum miopenReduceCalculationNanPropagation_t - * Nan numbers propagation modes for reduce calculation - */ -typedef enum -{ - MIOPEN_REDUCE_CALCULATION_NOT_PROPAGATE_NAN = 0, /*!< does not propagate Nan number */ - MIOPEN_REDUCE_CALCULATION_PROPAGATE_NAN = - 1, /*!< propagate the Nan number by the Reduction operation */ -} miopenReduceCalculationNanPropagation_t; - -// ReduceCalculation APIs -/** @addtogroup reducecalculation - * - * @{ - */ - -/*! @enum miopenReduceCalculationOp_t - * Reduction Calculation operation types - */ -typedef enum -{ - MIOPEN_REDUCE_CALCULATION_PROD = - 1, /*!< the operation is multiplying the values of the reduced elements */ - MIOPEN_REDUCE_CALCULATION_SUM = - 2, /*!< the operation is adding the values of the reduced elements */ -} miopenReduceCalculationOp_t; - -/*! @brief Helper function to query the minimum workspace size required by the ReduceTensor call - * - * @param [in] handle MIOpen Handle - * @param [in] xDesc Tensor descriptor for data input tensor x - * @param [in] dim Dimension to calculation. - * @param [in] yDesc Tensor descriptor for output data tensor y - * @param [out] sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetReduceCalculationWorkspaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const int32_t dim, - const miopenReduceCalculationOp_t reduceCalculationOp, - const miopenTensorDescriptor_t reduceDesc, - size_t* sizeInBytes); - -/*! @brief Execute a reducecalculation forward layer - * - * @param [in] handle MIOpen handle - * @param [in] nanPropagation Nan number propagation mode - * @param [in] workspace Address of the allocated workspace data - * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data - * @param [in] xDesc Tensor descriptor for data input tensor x - * @param [in] x Data tensor x - * @param [in] dim Dimension to calculation. - * @param [in] yDesc Tensor descriptor for output data tensor y - * @param [out] y Data tensor y - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenReduceCalculationForward(miopenHandle_t handle, - miopenReduceCalculationNanPropagation_t nanPropagation, - void* workspace, - size_t workspaceSizeInBytes, - const miopenTensorDescriptor_t xDesc, - const void* x, - const int32_t dim, - const miopenReduceCalculationOp_t reduceCalculationOp, - const miopenTensorDescriptor_t reduceDesc, - void* y); - -/** @} */ -// CLOSEOUT REDUCE CALCULATION DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API - -/*! @ingroup ReduceExtreme - * @enum miopenReduceExtremeOp_t - * Reduction Extreme operation types - */ -typedef enum -{ - MIOPEN_REDUCE_EXTREME_ARGMIN = - 1, /*!< the operation is getting the minimum index of the reduced elements */ - MIOPEN_REDUCE_EXTREME_ARGMAX = - 2, /*!< the operation is getting the maximum index of the reduced elements */ - MIOPEN_REDUCE_EXTREME_MIN = - 3, /*!< the operation is getting the minimum value and index of the reduced elements */ - MIOPEN_REDUCE_EXTREME_MAX = - 4, /*!< the operation is getting the maximum value and index of the reduced elements */ -} miopenReduceExtremeOp_t; - -// ReduceExtreme APIs -/** @addtogroup ReduceExtreme - * - * @{ - */ - -/*! @brief Find the the extreme (minimum, maximum) value and index of a tensor across Dimension. - * - * @param handle MIOpen handle (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param dim Dimension to reduce argmax. (input) - * @param reduceExtremeOp Enumerant specifying the operation used by ReduceExtreme (input) - * @param yDesc Tensor descriptor for reduce data tensor y (input) - * @param y Data tensor y (output) - * @param indiceDesc Tensor descriptor for reduce data tensor indice only int32_t - * (input) - * @param indice Data tensor indice (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenReduceExtremeForward(miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const void* x, - const int32_t dim, - const miopenReduceExtremeOp_t reduceExtremeOp, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t indiceDesc, - void* indice); - -/** @} */ -// CLOSEOUT REDUCEEXTREME DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// GroupNorm APIs -/** @addtogroup groupnorm - * - * @{ - */ -/*! @brief Execute a groupnorm forward layer - * - * @param handle MIOpen handle (input) - * @param mode GroupNorm mode (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param weight Data tensor weight (input) - * @param biasDesc Tensor descriptor for data input tensor bias (input) - * @param bias Data tensor bias (input) - * @param num_groups nNmber of groups to separate the channels into (input) - * @param epsilon Value to stablize inverse variance calculation (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param meanDesc Tensor descriptor for output data tensor mean (input) - * @param mean Data tensor mean (output) - * @param rstdDesc Tensor descriptor for output data tensor rstd (input) - * @param rstd Data tensor rstd (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGroupNormForward(miopenHandle_t handle, - miopenNormMode_t mode, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t weightDesc, - const void* weight, - const miopenTensorDescriptor_t biasDesc, - const void* bias, - const uint64_t num_groups, - const float epsilon, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t meanDesc, - void* mean, - const miopenTensorDescriptor_t rstdDesc, - void* rstd); - -/** @} */ -// CLOSEOUT groupnorm DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// LayerNorm APIs -/** @addtogroup layernorm - * - * @{ - */ -/*! @brief Execute a add and layernorm forward layer - * - * @param handle MIOpen handle (input) - * @param mode LayerNorm mode (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param x2Desc Tensor descriptor for data input tensor x2 (input) - * @param x2 Data tensor x2 (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param weight Data tensor weight (input) - * @param biasDesc Tensor descriptor for data input tensor bias (input) - * @param bias Data tensor bias (input) - * @param epsilon Value to stablize inverse variance calculation (input) - * @param normalized_dim Nomalized dimensions in the input array (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param meanDesc Tensor descriptor for output data tensor mean (input) - * @param mean Data tensor mean (output) - * @param rstdDesc Tensor descriptor for output data tensor rstd (input) - * @param rstd Data tensor rstd (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenAddLayerNormForward(miopenHandle_t handle, - miopenNormMode_t mode, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t x2Desc, - const void* x2, - const miopenTensorDescriptor_t weightDesc, - const void* weight, - const miopenTensorDescriptor_t biasDesc, - const void* bias, - const float epsilon, - const int32_t normalized_dim, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t meanDesc, - void* mean, - const miopenTensorDescriptor_t rstdDesc, - void* rstd); - -/** @} */ -// CLOSEOUT LAYERNORM DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// LayerNorm APIs -/** @addtogroup layernorm - * - * @{ - */ -/*! @brief Execute a T5layernorm forward layer - * - * @param handle MIOpen handle (input) - * @param mode LayerNorm mode (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param weight Data tensor weight (input) - * @param epsilon Value to stablize inverse variance calculation (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @param rstdDesc Tensor descriptor for output data tensor rstd (input) - * @param rstd Data tensor rstd (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenT5LayerNormForward(miopenHandle_t handle, - miopenNormMode_t mode, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t weightDesc, - const void* weight, - const float epsilon, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t rstdDesc, - void* rstd); - -/*! @brief Helper function to query the minimum workspace size required by the T5layernorm backward - * call - * - * @param handle MIOpen Handle (input) - * @param mode LayerNorm mode (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param rstdDesc Tensor descriptor for data input tensor rstd (input) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dwDesc Tensor descriptor for output data tensor dw (input) - * @param sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetT5LayerNormBackwardWorkspaceSize(miopenHandle_t handle, - miopenNormMode_t mode, - const miopenTensorDescriptor_t dyDesc, - const miopenTensorDescriptor_t xDesc, - const miopenTensorDescriptor_t weightDesc, - const miopenTensorDescriptor_t rstdDesc, - const miopenTensorDescriptor_t dxDesc, - const miopenTensorDescriptor_t dwDesc, - size_t* sizeInBytes); - -/*! @brief Execute a T5layernorm backward layer - * - * @param handle MIOpen handle (input) - * @param mode LayerNorm mode (input) - * @param workspace Address of the allocated workspace data (input) - * @param workspaceSizeInBytes Size in bytes of the allocated workspace data (input) - * @param dyDesc Tensor descriptor for data input tensor dy (input) - * @param dy Data tensor dy (input) - * @param xDesc Tensor descriptor for output data tensor x (input) - * @param x Data tensor x (input) - * @param weightDesc Tensor descriptor for data input tensor weight (input) - * @param weight Data tensor weight (input) - * @param rstdDesc Tensor descriptor for output data tensor rstd (input) - * @param rstd Data tensor rstd (output) - * @param dxDesc Tensor descriptor for output data tensor dx (input) - * @param dx Data tensor dx (output) - * @param dwDesc Tensor descriptor for output data tensor dw (input) - * @param dw Data tensor dw (output) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenT5LayerNormBackward(miopenHandle_t handle, - miopenNormMode_t mode, - void* workspace, - size_t workspaceSizeInBytes, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t weightDesc, - const void* weight, - const miopenTensorDescriptor_t rstdDesc, - const void* rstd, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const miopenTensorDescriptor_t dwDesc, - void* dw); -/** @} */ -// CLOSEOUT LAYERNORM DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// Graph API -/** @addtogroup GraphAPI - * - * @{ - */ - -/*! @brief Descriptor type - * - * An enumerated type that indicates the type of backend descriptors. Users create a backend - * descriptor of a particular type by passing a value from this enumerate to the - * miopenBackendCreateDescriptor() function. - */ -typedef enum -{ - MIOPEN_BACKEND_CONVOLUTION_DESCRIPTOR, - MIOPEN_BACKEND_ENGINE_DESCRIPTOR, - MIOPEN_BACKEND_ENGINECFG_DESCRIPTOR, - MIOPEN_BACKEND_ENGINEHEUR_DESCRIPTOR, - MIOPEN_BACKEND_EXECUTION_PLAN_DESCRIPTOR, - MIOPEN_BACKEND_INTERMEDIATE_INFO_DESCRIPTOR, - MIOPEN_BACKEND_KNOB_CHOICE_DESCRIPTOR, - MIOPEN_BACKEND_KNOB_INFO_DESCRIPTOR, - MIOPEN_BACKEND_LAYOUT_INFO_DESCRIPTOR, - MIOPEN_BACKEND_MATMUL_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_CONCAT_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_MATMUL_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_RESAMPLE_BWD_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_RESHAPE_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_RNG_DESCRIPTOR, - MIOPEN_BACKEND_OPERATION_SIGNAL_DESCRIPTOR, - MIOPEN_BACKEND_OPERATIONGRAPH_DESCRIPTOR, - MIOPEN_BACKEND_POINTWISE_DESCRIPTOR, - MIOPEN_BACKEND_REDUCTION_DESCRIPTOR, - MIOPEN_BACKEND_RESAMPLE_DESCRIPTOR, - MIOPEN_BACKEND_RNG_DESCRIPTOR, - MIOPEN_BACKEND_TENSOR_DESCRIPTOR, - MIOPEN_BACKEND_VARIANT_PACK_DESCRIPTOR, -} miopenBackendDescriptorType_t; - -/*! @brief Backend Descriptor's Attribute - * - * An enumerated type that indicates the backend descriptor attributes - * that can be set or get using miopenBackendSetAttribute() and miopenBackendGetAttribute() - * functions. The backend descriptor to which an attribute belongs is - * identified by the prefix of the attribute name. - */ -typedef enum -{ - MIOPEN_ATTR_POINTWISE_MODE = 0, - MIOPEN_ATTR_POINTWISE_MATH_PREC = 1, - MIOPEN_ATTR_POINTWISE_NAN_PROPAGATION = 2, - MIOPEN_ATTR_POINTWISE_RELU_LOWER_CLIP = 3, - MIOPEN_ATTR_POINTWISE_RELU_UPPER_CLIP = 4, - MIOPEN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE = 5, - MIOPEN_ATTR_POINTWISE_ELU_ALPHA = 6, - MIOPEN_ATTR_POINTWISE_SOFTPLUS_BETA = 7, - MIOPEN_ATTR_POINTWISE_SWISH_BETA = 8, - MIOPEN_ATTR_POINTWISE_AXIS = 9, - - MIOPEN_ATTR_CONVOLUTION_COMP_TYPE = 100, - MIOPEN_ATTR_CONVOLUTION_CONV_MODE = 101, - MIOPEN_ATTR_CONVOLUTION_DILATIONS = 102, - MIOPEN_ATTR_CONVOLUTION_FILTER_STRIDES = 103, - MIOPEN_ATTR_CONVOLUTION_POST_PADDINGS = 104, - MIOPEN_ATTR_CONVOLUTION_PRE_PADDINGS = 105, - MIOPEN_ATTR_CONVOLUTION_SPATIAL_DIMS = 106, - - MIOPEN_ATTR_ENGINEHEUR_MODE = 200, - MIOPEN_ATTR_ENGINEHEUR_OPERATION_GRAPH = 201, - MIOPEN_ATTR_ENGINEHEUR_RESULTS = 202, - MIOPEN_ATTR_ENGINEHEUR_SM_COUNT_TARGET = 203, - - MIOPEN_ATTR_ENGINECFG_ENGINE = 300, - MIOPEN_ATTR_ENGINECFG_INTERMEDIATE_INFO = 301, - MIOPEN_ATTR_ENGINECFG_KNOB_CHOICES = 302, - - MIOPEN_ATTR_EXECUTION_PLAN_HANDLE = 400, - MIOPEN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG = 401, - MIOPEN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE = 402, - MIOPEN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS = 403, - MIOPEN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS = 404, - MIOPEN_ATTR_EXECUTION_PLAN_JSON_REPRESENTATION = 405, - - MIOPEN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID = 500, - MIOPEN_ATTR_INTERMEDIATE_INFO_SIZE = 501, - MIOPEN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS = 502, - MIOPEN_ATTR_INTERMEDIATE_INFO_DEPENDENT_ATTRIBUTES = 503, - - MIOPEN_ATTR_KNOB_CHOICE_KNOB_TYPE = 600, - MIOPEN_ATTR_KNOB_CHOICE_KNOB_VALUE = 601, - - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA = 700, - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA = 701, - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC = 702, - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_W = 703, - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_X = 704, - MIOPEN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y = 705, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA = 706, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA = 707, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC = 708, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W = 709, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX = 710, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY = 711, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA = 712, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA = 713, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC = 714, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW = 715, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X = 716, - MIOPEN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY = 717, - MIOPEN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR = 750, - MIOPEN_ATTR_OPERATION_POINTWISE_XDESC = 751, - MIOPEN_ATTR_OPERATION_POINTWISE_BDESC = 752, - MIOPEN_ATTR_OPERATION_POINTWISE_YDESC = 753, - MIOPEN_ATTR_OPERATION_POINTWISE_ALPHA1 = 754, - MIOPEN_ATTR_OPERATION_POINTWISE_ALPHA2 = 755, - MIOPEN_ATTR_OPERATION_POINTWISE_DXDESC = 756, - MIOPEN_ATTR_OPERATION_POINTWISE_DYDESC = 757, - MIOPEN_ATTR_OPERATION_POINTWISE_TDESC = 758, - - MIOPEN_ATTR_OPERATION_GENSTATS_MODE = 770, - MIOPEN_ATTR_OPERATION_GENSTATS_MATH_PREC = 771, - MIOPEN_ATTR_OPERATION_GENSTATS_XDESC = 772, - MIOPEN_ATTR_OPERATION_GENSTATS_SUMDESC = 773, - MIOPEN_ATTR_OPERATION_GENSTATS_SQSUMDESC = 774, - - MIOPEN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE = 780, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC = 781, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC = 782, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC = 783, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC = 784, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC = 785, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC = 786, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC = 787, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC = 788, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC = 789, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC = 790, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC = 791, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC = 792, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC = 793, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC = 794, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC = 795, - MIOPEN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERATE_FACTOR_DESC = 796, - - MIOPEN_ATTR_OPERATIONGRAPH_HANDLE = 800, - MIOPEN_ATTR_OPERATIONGRAPH_OPS = 801, - MIOPEN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT = 802, - - MIOPEN_ATTR_TENSOR_BYTE_ALIGNMENT = 900, - MIOPEN_ATTR_TENSOR_DATA_TYPE = 901, - MIOPEN_ATTR_TENSOR_DIMENSIONS = 902, - MIOPEN_ATTR_TENSOR_STRIDES = 903, - MIOPEN_ATTR_TENSOR_VECTOR_COUNT = 904, - MIOPEN_ATTR_TENSOR_VECTORIZED_DIMENSION = 905, - MIOPEN_ATTR_TENSOR_UNIQUE_ID = 906, - MIOPEN_ATTR_TENSOR_IS_VIRTUAL = 907, - MIOPEN_ATTR_TENSOR_IS_BY_VALUE = 908, - MIOPEN_ATTR_TENSOR_REORDERING_MODE = 909, - MIOPEN_ATTR_TENSOR_RAGGED_OFFSET_DESC = 910, - - MIOPEN_ATTR_VARIANT_PACK_UNIQUE_IDS = 1000, - MIOPEN_ATTR_VARIANT_PACK_DATA_POINTERS = 1001, - MIOPEN_ATTR_VARIANT_PACK_INTERMEDIATES = 1002, - MIOPEN_ATTR_VARIANT_PACK_WORKSPACE = 1003, - - MIOPEN_ATTR_LAYOUT_INFO_TENSOR_UID = 1100, - MIOPEN_ATTR_LAYOUT_INFO_TYPES = 1101, - - MIOPEN_ATTR_KNOB_INFO_TYPE = 1200, - MIOPEN_ATTR_KNOB_INFO_MAXIMUM_VALUE = 1201, - MIOPEN_ATTR_KNOB_INFO_MINIMUM_VALUE = 1202, - MIOPEN_ATTR_KNOB_INFO_STRIDE = 1203, - - MIOPEN_ATTR_ENGINE_OPERATION_GRAPH = 1300, - MIOPEN_ATTR_ENGINE_GLOBAL_INDEX = 1301, - MIOPEN_ATTR_ENGINE_KNOB_INFO = 1302, - MIOPEN_ATTR_ENGINE_NUMERICAL_NOTE = 1303, - MIOPEN_ATTR_ENGINE_LAYOUT_INFO = 1304, - MIOPEN_ATTR_ENGINE_BEHAVIOR_NOTE = 1305, - MIOPEN_ATTR_ENGINE_SM_COUNT_TARGET = 1306, - - MIOPEN_ATTR_MATMUL_COMP_TYPE = 1500, - MIOPEN_ATTR_MATMUL_PADDING_VALUE = 1501, - - MIOPEN_ATTR_OPERATION_MATMUL_ADESC = 1520, - MIOPEN_ATTR_OPERATION_MATMUL_BDESC = 1521, - MIOPEN_ATTR_OPERATION_MATMUL_CDESC = 1522, - MIOPEN_ATTR_OPERATION_MATMUL_DESC = 1523, - MIOPEN_ATTR_OPERATION_MATMUL_IRREGULARLY_STRIDED_BATCH_COUNT = 1524, - MIOPEN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC = 1525, - MIOPEN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC = 1526, - MIOPEN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC = 1527, - - MIOPEN_ATTR_REDUCTION_OPERATOR = 1600, - MIOPEN_ATTR_REDUCTION_COMP_TYPE = 1601, - - MIOPEN_ATTR_OPERATION_REDUCTION_XDESC = 1610, - MIOPEN_ATTR_OPERATION_REDUCTION_YDESC = 1611, - MIOPEN_ATTR_OPERATION_REDUCTION_DESC = 1612, - - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC = 1620, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC = 1621, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC = 1622, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC = 1623, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC = 1624, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC = 1625, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC = 1626, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC = 1627, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC = 1628, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC = 1629, - MIOPEN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS = 1630, - - MIOPEN_ATTR_RESAMPLE_MODE = 1700, - MIOPEN_ATTR_RESAMPLE_COMP_TYPE = 1701, - MIOPEN_ATTR_RESAMPLE_SPATIAL_DIMS = 1702, - MIOPEN_ATTR_RESAMPLE_POST_PADDINGS = 1703, - MIOPEN_ATTR_RESAMPLE_PRE_PADDINGS = 1704, - MIOPEN_ATTR_RESAMPLE_STRIDES = 1705, - MIOPEN_ATTR_RESAMPLE_WINDOW_DIMS = 1706, - MIOPEN_ATTR_RESAMPLE_NAN_PROPAGATION = 1707, - MIOPEN_ATTR_RESAMPLE_PADDING_MODE = 1708, - - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_XDESC = 1710, - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_YDESC = 1711, - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC = 1712, - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA = 1713, - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_BETA = 1714, - MIOPEN_ATTR_OPERATION_RESAMPLE_FWD_DESC = 1716, - - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_DXDESC = 1720, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_DYDESC = 1721, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_IDXDESC = 1722, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_ALPHA = 1723, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_BETA = 1724, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_DESC = 1725, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_XDESC = 1726, - MIOPEN_ATTR_OPERATION_RESAMPLE_BWD_YDESC = 1727, - - MIOPEN_ATTR_OPERATION_CONCAT_AXIS = 1800, - MIOPEN_ATTR_OPERATION_CONCAT_INPUT_DESCS = 1801, - MIOPEN_ATTR_OPERATION_CONCAT_INPLACE_INDEX = 1802, - MIOPEN_ATTR_OPERATION_CONCAT_OUTPUT_DESC = 1803, - - MIOPEN_ATTR_OPERATION_SIGNAL_MODE = 1900, - MIOPEN_ATTR_OPERATION_SIGNAL_FLAGDESC = 1901, - MIOPEN_ATTR_OPERATION_SIGNAL_VALUE = 1902, - MIOPEN_ATTR_OPERATION_SIGNAL_XDESC = 1903, - MIOPEN_ATTR_OPERATION_SIGNAL_YDESC = 1904, - - MIOPEN_ATTR_OPERATION_NORM_FWD_MODE = 2000, - MIOPEN_ATTR_OPERATION_NORM_FWD_PHASE = 2001, - MIOPEN_ATTR_OPERATION_NORM_FWD_XDESC = 2002, - MIOPEN_ATTR_OPERATION_NORM_FWD_MEAN_DESC = 2003, - MIOPEN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC = 2004, - MIOPEN_ATTR_OPERATION_NORM_FWD_SCALE_DESC = 2005, - MIOPEN_ATTR_OPERATION_NORM_FWD_BIAS_DESC = 2006, - MIOPEN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC = 2007, - MIOPEN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC = 2008, - MIOPEN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC = 2009, - MIOPEN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC = 2010, - MIOPEN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC = 2011, - MIOPEN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC = 2012, - MIOPEN_ATTR_OPERATION_NORM_FWD_YDESC = 2013, - MIOPEN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS = 2014, - - MIOPEN_ATTR_OPERATION_NORM_BWD_MODE = 2100, - MIOPEN_ATTR_OPERATION_NORM_BWD_XDESC = 2101, - MIOPEN_ATTR_OPERATION_NORM_BWD_MEAN_DESC = 2102, - MIOPEN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC = 2103, - MIOPEN_ATTR_OPERATION_NORM_BWD_DYDESC = 2104, - MIOPEN_ATTR_OPERATION_NORM_BWD_SCALE_DESC = 2105, - MIOPEN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC = 2106, - MIOPEN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC = 2107, - MIOPEN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC = 2108, - MIOPEN_ATTR_OPERATION_NORM_BWD_DXDESC = 2109, - MIOPEN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS = 2110, - - MIOPEN_ATTR_OPERATION_RESHAPE_XDESC = 2200, - MIOPEN_ATTR_OPERATION_RESHAPE_YDESC = 2201, - - MIOPEN_ATTR_RNG_DISTRIBUTION = 2300, - MIOPEN_ATTR_RNG_NORMAL_DIST_MEAN = 2301, - MIOPEN_ATTR_RNG_NORMAL_DIST_STANDARD_DEVIATION = 2302, - MIOPEN_ATTR_RNG_UNIFORM_DIST_MAXIMUM = 2303, - MIOPEN_ATTR_RNG_UNIFORM_DIST_MINIMUM = 2304, - MIOPEN_ATTR_RNG_BERNOULLI_DIST_PROBABILITY = 2305, - - MIOPEN_ATTR_OPERATION_RNG_YDESC = 2310, - MIOPEN_ATTR_OPERATION_RNG_SEED = 2311, - MIOPEN_ATTR_OPERATION_RNG_DESC = 2312, - MIOPEN_ATTR_OPERATION_RNG_OFFSET_DESC = 2313, - -} miopenBackendAttributeName_t; - -/*! @brief Data type of an attribute of a backend descriptor - * - * Specifies the data type of an attribute of a backend descriptor. - * It is used to specify the type of data pointed to by the - * void *arrayOfElements argument of miopenBackendSetAttribute() - * and miopenBackendGetAttribute() - */ -typedef enum -{ - MIOPEN_TYPE_HANDLE = 0, /*!< miopenHandle_t */ - MIOPEN_TYPE_DATA_TYPE, /*!< miopenDataType_t */ - MIOPEN_TYPE_BOOLEAN, /*!< bool */ - MIOPEN_TYPE_INT64, /*!< int64_t */ - MIOPEN_TYPE_FLOAT, /*!< float */ - MIOPEN_TYPE_DOUBLE, /*!< double */ - MIOPEN_TYPE_VOID_PTR, /*!< void * */ - MIOPEN_TYPE_CONVOLUTION_MODE, /*!< miopenConvolutionMode_t */ - MIOPEN_TYPE_HEUR_MODE, /*!< miopenBackendHeurMode_t */ - MIOPEN_TYPE_KNOB_TYPE, /*!< miopenBackendKnobType_t */ - MIOPEN_TYPE_NAN_PROPOGATION, /*!< miopenNanPropagation_t */ - MIOPEN_TYPE_NUMERICAL_NOTE, /*!< miopenBackendNumericalNote_t */ - MIOPEN_TYPE_LAYOUT_TYPE, /*!< miopenBackendLayoutType_t */ - MIOPEN_TYPE_ATTRIB_NAME, /*!< miopenBackendAttributeName_t */ - MIOPEN_TYPE_POINTWISE_MODE, /*!< miopenPointwiseMode_t */ - MIOPEN_TYPE_BACKEND_DESCRIPTOR, /*!< miopenBackendDescriptor_t */ - MIOPEN_TYPE_GENSTATS_MODE, /*!< miopenGenStatsMode_t */ - MIOPEN_TYPE_BN_FINALIZE_STATS_MODE, /*!< miopenBnFinalizeStatsMode_t */ - MIOPEN_TYPE_REDUCTION_OPERATOR_TYPE, /*!< miopenReduceTensorOp_t */ - MIOPEN_TYPE_BEHAVIOR_NOTE, /*!< miopenBackendBehaviorNote_t */ - MIOPEN_TYPE_TENSOR_REORDERING_MODE, /*!< miopenBackendTensorReordering_t */ - MIOPEN_TYPE_RESAMPLE_MODE, /*!< miopenResampleMode_t */ - MIOPEN_TYPE_PADDING_MODE, /*!< miopenPaddingMode_t */ - MIOPEN_TYPE_INT32, /*!< int32_t */ - MIOPEN_TYPE_CHAR, /*!< char */ - MIOPEN_TYPE_SIGNAL_MODE, /*!< miopenSignalMode_t */ - MIOPEN_TYPE_FRACTION, /*!< miopenFraction_t */ - MIOPEN_TYPE_NORM_MODE, /*!< miopenBackendNormMode_t */ - MIOPEN_TYPE_NORM_FWD_PHASE, /*!< miopenBackendNormFwdPhase_t */ - MIOPEN_TYPE_RNG_DISTRIBUTION /*!< miopenRngDistribution_t */ -} miopenBackendAttributeType_t; - -/*! @brief Intended poinwise math operation for a pointwise operation descriptor - * - * An enumerated type to indicate the intended pointwise math operation in the backend pointwise - * operation descriptor - */ -typedef enum -{ - /*! A pointwise addition between two tensors is computed.*/ - MIOPEN_POINTWISE_ADD, - - /*! A pointwise addition between the first tensor and the square of the second tensor is - computed. */ - MIOPEN_POINTWISE_ADD_SQUARE, - - /*! A pointwise true division of the first tensor by second tensor is computed. */ - MIOPEN_POINTWISE_DIV, - - /*! A pointwise maximum is taken between two tensors. */ - MIOPEN_POINTWISE_MAX, - - /*! A pointwise minimum is taken between two tensors. */ - MIOPEN_POINTWISE_MIN, - - /*! A pointwise floating-point remainder of the first tensor’s division by the second tensor is - computed. */ - MIOPEN_POINTWISE_MOD, - - /*! A pointwise multiplication between two tensors is computed. */ - MIOPEN_POINTWISE_MUL, - - /*! A pointwise value from the first tensor to the power of the second tensor is computed. */ - MIOPEN_POINTWISE_POW, - - /*! A pointwise subtraction between two tensors is computed. */ - MIOPEN_POINTWISE_SUB, - - /*! A pointwise absolute value of the input tensor is computed. */ - MIOPEN_POINTWISE_ABS, - - /*! A pointwise ceiling of the input tensor is computed. */ - MIOPEN_POINTWISE_CEIL, - - /*! A pointwise trigonometric cosine of the input tensor is computed. */ - MIOPEN_POINTWISE_COS, - - /*! A pointwise exponential of the input tensor is computed. */ - MIOPEN_POINTWISE_EXP, - - /*! A pointwise floor of the input tensor is computed. */ - MIOPEN_POINTWISE_FLOOR, - - /*! A pointwise natural logarithm of the input tensor is computed. */ - MIOPEN_POINTWISE_LOG, - - /*! A pointwise numerical negative of the input tensor is computed. */ - MIOPEN_POINTWISE_NEG, - - /*! A pointwise reciprocal of the square root of the input tensor is computed. */ - MIOPEN_POINTWISE_RSQRT, - - /*! A pointwise trigonometric sine of the input tensor is computed. */ - MIOPEN_POINTWISE_SIN, - - /*! A pointwise square root of the input tensor is computed. */ - MIOPEN_POINTWISE_SQRT, - - /*! A pointwise trigonometric tangent of the input tensor is computed. */ - MIOPEN_POINTWISE_TAN, - - /*! A pointwise Error Function is computed. */ - MIOPEN_POINTWISE_ERF, - - /*! No computation is performed. As with other pointwise modes, this mode provides implicit - conversions by specifying the data type of the input tensor as one type, and the data type of - the output tensor as another. */ - MIOPEN_POINTWISE_IDENTITY, - - /*! A pointwise rectified linear activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_RELU_FWD, - - /*! A pointwise tanh activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_TANH_FWD, - - /*! A pointwise sigmoid activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_SIGMOID_FWD, - - /*! A pointwise Exponential Linear Unit activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_ELU_FWD, - - /*! A pointwise Gaussian Error Linear Unit activation function of the input tensor is computed. - */ - MIOPEN_POINTWISE_GELU_FWD, - - /*! A pointwise softplus activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_SOFTPLUS_FWD, - - /*! A pointwise swish activation function of the input tensor is computed. */ - MIOPEN_POINTWISE_SWISH_FWD, - - /*! A pointwise tanh approximation of the Gaussian Error Linear Unit activation function of the - input tensor is computed. The tanh GELU approximation is computed as \f$0.5x\left( - 1+\tanh\left[ \sqrt{2/\pi}\left( x+0.044715x^{3} \right) \right] \right)\f$ */ - MIOPEN_POINTWISE_GELU_APPROX_TANH_FWD, - - /*! A pointwise first derivative of rectified linear activation of the input tensor is computed. - */ - MIOPEN_POINTWISE_RELU_BWD, - - /*! A pointwise first derivative of tanh activation of the input tensor is computed. */ - MIOPEN_POINTWISE_TANH_BWD, - - /*! A pointwise first derivative of sigmoid activation of the input tensor is computed. */ - MIOPEN_POINTWISE_SIGMOID_BWD, - - /*! A pointwise first derivative of Exponential Linear Unit activation of the input tensor is - computed. */ - MIOPEN_POINTWISE_ELU_BWD, - - /*! A pointwise first derivative of Gaussian Error Linear Unit activation of the input tensor is - computed. */ - MIOPEN_POINTWISE_GELU_BWD, - - /*! A pointwise first derivative of softplus activation of the input tensor is computed. */ - MIOPEN_POINTWISE_SOFTPLUS_BWD, - - /*! A pointwise first derivative of swish activation of the input tensor is computed. */ - MIOPEN_POINTWISE_SWISH_BWD, - - /*! A pointwise first derivative of the tanh approximation of the Gaussian Error Linear Unit - activation of the input tensor is computed. This is computed as \f$0.5\left( 1+\tanh\left( - b\left( x+cx^{3} \right) \right)+bxsech^{2}\left( b\left( cx^{3}+x \right) \right)\left( - 3cx^{2}+1 \right)dy \right)\f$ where \f$b\f$ is \f$\sqrt{2/\pi}\f$ and \f$c\f$ is - \f$0.044715\f$ */ - MIOPEN_POINTWISE_GELU_APPROX_TANH_BWD, - - /*! A pointwise truth value of the first tensor equal to the second tensor is computed. */ - MIOPEN_POINTWISE_CMP_EQ, - - /*! A pointwise truth value of the first tensor not equal to the second tensor is computed. */ - MIOPEN_POINTWISE_CMP_NEQ, - - /*! A pointwise truth value of the first tensor greater than the second tensor is computed. */ - MIOPEN_POINTWISE_CMP_GT, - - /*! A pointwise truth value of the first tensor greater than equal to the second tensor is - computed. */ - MIOPEN_POINTWISE_CMP_GE, - - /*! A pointwise truth value of the first tensor less than the second tensor is computed. */ - MIOPEN_POINTWISE_CMP_LT, - - /*! A pointwise truth value of the first tensor less than equal to the second tensor is - computed. */ - MIOPEN_POINTWISE_CMP_LE, - - /*! A pointwise truth value of the first tensor logical AND second tensor is computed. */ - MIOPEN_POINTWISE_LOGICAL_AND, - - /*! A pointwise truth value of the first tensor logical OR second tensor is computed. */ - MIOPEN_POINTWISE_LOGICAL_OR, - - /*! A pointwise truth value of input tensors logical NOT is computed. */ - MIOPEN_POINTWISE_LOGICAL_NOT, - - /*! A pointwise index value of the input tensor is generated along a given axis. */ - MIOPEN_POINTWISE_GEN_INDEX, - - /*! A pointwise value is selected amongst two input tensors based on a given predicate tensor. - */ - MIOPEN_POINTWISE_BINARY_SELECT, - - /*! A pointwise reciprocal of the input tensor is computed. In other words, for every element x - in the input tensor, 1/x is computed. */ - MIOPEN_POINTWISE_RECIPROCAL -} miopenPointwiseMode_t; - -/*! @brief Distribution for random number generation - * - * An enumerated type to indicate the distribution to be used in the backend Rng (random number - * generator) operation. - */ -typedef enum -{ - MIOPEN_RNG_DISTRIBUTION_BERNOULLI, - MIOPEN_RNG_DISTRIBUTION_UNIFORM, - MIOPEN_RNG_DISTRIBUTION_NORMAL, -} miopenRngDistribution_t; - -typedef enum -{ - /* IDENTITY alpha = 1.0 and beta = 0.0 */ - /* SCALE alpha = 4.2 and beta = 0.0 */ - /* BILINEAR alpha = 3.2 and beta = 1.1 */ - /* ERROR_STATE alpha = 0.0 and beta = 3.1 */ - - DEFAULT = 0, /* alpha = 1.0 and beta = 0.0.*/ - SCALE = 1, /* alpha with some value and beta 0.0*/ - BILINEAR = 2, /* both alpha and beta with some value*/ - ERROR_STATE = 3, /* alpha 0.0 and beta with some value, this should not occur. - But used to check for errors.*/ -} miopenAlphaBetaCase_t; -/*! @brief Operation mode of CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR - * - * An enumerated type to indicate the operation mode of a CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR - */ -typedef enum -{ - MIOPEN_HEUR_MODE_INSTANT = 0, - MIOPEN_HEUR_MODE_B = 1, - MIOPEN_HEUR_MODE_FALLBACK = 2, - MIOPEN_HEUR_MODE_A = 3, - MIOPEN_HEUR_MODES_COUNT = 4, -} miopenBackendHeurMode_t; - -/*! @brief Backend descriptor - * - * A typedef void pointer to one of many opaque descriptor structures. - * The type of structure that it points to is determined by the argument when allocating the memory - * for the opaque structure using miopenBackendCreateDescriptor(). - * - * Attributes of a descriptor can be set using miopenBackendSetAttribute(). After all required - * attributes of a descriptor are set, the descriptor can be finalized by miopenBackendFinalize(). - * From a finalized descriptor, one can query its queryable attributes using - * miopenBackendGetAttribute(). Finally, the memory allocated for a descriptor can be freed using - * miopenBackendDestroyDescriptor(). - */ -MIOPEN_DECLARE_OBJECT(miopenBackendDescriptor) - -/*! @brief Creates a backend descriptor - * - * Allocates memory for a given descriptorType at the location pointed - * by the descriptor - * - * @param [in] descriptorType One among the enumerated miopenBackendDescriptorType_t - * @param [out] descriptor Pointer to a descriptor - * - * @retval miopenStatusSuccess The creation was successful - * @retval miopenStatusUnsupportedOp Creating a descriptor of a given type is not supported - * @retval miopenStatusAllocFailed The memory allocation failed - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendCreateDescriptor( - miopenBackendDescriptorType_t descriptorType, miopenBackendDescriptor_t* descriptor); - -/*! @brief Sets an attribute of a descriptor - * - * This function sets an attribute of a descriptor to values provided as a pointer. - * Returns miopenStatusUnsupportedOp if the descriptor is already - * successfully finalized using miopenBackendFinalize(). - * - * @param [in] descriptor Instance of miopenBackendDescriptor_t whose attribute is being set - * @param [in] attributeName The name of the attribute being set on the descriptor - * @param [in] attributeType The type of attribute - * @param [in] elementCount Number of elements being set - * @param [in] arrayOfElements The starting location for an array from where to read the values - * from. The elements of the array are expected to be of the datatype - * of the attributeType. The datatype of the attributeType is listed - * in the mapping table of miopenBackendAttributeType_t. - * - * @retval miopenStatusSuccess The attributeName was set to the descriptor - * @retval miopenStatusNotInitialized The backend descriptor pointed to by the descriptor is - * already in the finalized state - * @retval miopenStatusBadParm The function is called with arguments that correspond to - * invalid values. Some examples include: - * * attributeName is not a settable attribute of descriptor. - * * attributeType is incorrect for this attributeName. - * * elemCount value is unexpected. - * * arrayOfElements contains values invalid for the - * attributeType. - * @retval miopenStatusUnsupportedOp The values to which the attributes are being set are not - * supported by the current version - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendSetAttribute(miopenBackendDescriptor_t descriptor, - miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements); - -/*! @brief Finalizes a backend descriptor - * - * Finalizes the memory pointed to by the descriptor. The type of finalization is done depending on - * the descriptorType argument with which the descriptor was created using - * miopenBackendCreateDescriptor() or initialized using miopenBackendInitialize(). - * - * @param [in] descriptor Instance of miopenBackendDescriptor_t to finalize - * - * @retval miopenStatusSuccess The descriptor was finalized successfully - * @retval miopenStatusBadParm Invalid descriptor attribute values or combination thereof is - * encountered - * @retval miopenStatusUnsupportedOp Descriptor attribute values or combinations therefore not - * supported by the current version - * @retval miopenStatusInternalError Some internal errors are encountered - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendFinalize(miopenBackendDescriptor_t descriptor); - -/*! @brief Retrieves backend descriptor's attribute - * - * This function retrieves the values of an attribute of a descriptor. attributeName is the name of - * the attribute whose value is requested. attributeType is the type of attribute. - * requestsedElementCount is the number of elements to be potentially retrieved. The number of - * elements for the requested attribute is stored in elementCount. The retrieved values are stored - * in arrayOfElements. When the attribute is expected to have a single value, arrayOfElements can be - * pointer to the output value. This function will return miopenStatusNotInitialized if the - * descriptor has not been successfully finalized using miopenBackendFinalize() - * - * @param [in] descriptor Instance of miopenBackendDescriptor_t whose attribute to - * retrieve - * @param [in] attributeName The name of the attribute being get from the descriptor - * @param [in] attributeType The type of attribute - * @param [in] requestedElementCount Number of elements to output to arrayOfElements - * @param [out] elementCount Output pointer for the number of elements the descriptor - * attribute has. Note that miopenBackendGetAttribute() will - * only write the least of this and requestedElementCount - * elements to arrayOfElements - * @param [out] arrayOfElements Array of elements of the datatype of the attributeType. The - * data type of the attributeType is listed in the mapping - * table of miopenBackendAttributeType_t - * - * @retval miopenStatusSuccess The attributeName was retrieved from the descriptor - * successfully - * @retval miopenStatusBadParm One or more invalid or inconsistent argument values were - * encountered. Some examples include: - * * attributeName is not a valid attribute for the descriptor. - * * attributeType is not one of the valid types for the - * attribute. - * @retval miopenStatusNotInitialized The descriptor has not been successfully finalized using - * miopenBackendFinalize() - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendGetAttribute(miopenBackendDescriptor_t descriptor, - miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements); - -/*! @brief Executes a graph - * - * Executes the given Engine Configuration Plan on the VariantPack and the finalized ExecutionPlan - * on the data. The data and the working space are encapsulated in the VariantPack - * - * @param [in] handle An instance of miopenHandle_t - * @param [in] executionPlan Descriptor of the finalized ExecutionPlan - * @param [in] variantPack Descriptor of the finalized VariantPack consisting of: - * * Data pointer for each non-virtual pointer of the operation set in - * the execution plan. - * * Pointer to user-allocated workspace in global memory at least as - * large as the size queried - * - * @retval miopenStatusSuccess The ExecutionPlan was executed successfully - * @retval miopenStatusBadParm An incorrect or inconsistent value is encountered. For - * example, a required data pointer is invalid - * @retval miopenStatusInternalError Some internal errors were encountered - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendExecute(miopenHandle_t handle, - miopenBackendDescriptor_t executionPlan, - miopenBackendDescriptor_t variantPack); - -/*! @brief Destroys an instance of miopenBackendDescriptor_t - * - * Destroys instances of miopenBackendDescriptor_t that were previously created using - * miopenBackendCreateDescriptor(). The value pointed by the descriptor will be undefined after the - * memory is free and done. - * - * **Undefined Behavior** if the descriptor was altered between the 'Create' and 'Destroy - * Descriptor' - * - * @param [in] descriptor Instance of miopenBackendDescriptor_t previously created by - * miopenBackendCreateDescriptor() - * - * @retval miopenStatusSuccess The memory was destroyed successfully - * @retval miopenStatusAllocFailed The destruction of memory failed - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendDestroyDescriptor(miopenBackendDescriptor_t descriptor); - -/*! @brief Repurposes an instance of miopenBackendDescriptor_t - * - * Repurposes a pre-allocated memory pointed to by a descriptor of size sizeInByte to a backend - * descriptor of type descriptorType. The finalized state of the descriptor is set to false. - * - * @param [in] descriptor Instance of miopenBackendDescriptor_t to be initialized - * @param [in] descriptorType Enumerated value for the type miopenBackendDescriptorType_t - * @param [in] sizeInBytes Size of memory pointed to by descriptor - * - * @retval miopenStatusSuccess The memory was initialized successfully - * @retval miopenStatusBadParm An invalid or inconsistent argument value is encountered. Some - * examples include: - * * descriptor is a nullptr - * * sizeInBytes is less than the size required by the descriptor - * type - * @retval miopenStatusUnknownError The error information was not gathered - */ -MIOPEN_EXPORT miopenStatus_t miopenBackendInitialize(miopenBackendDescriptor_t descriptor, - miopenBackendDescriptorType_t descriptorType, - size_t sizeInBytes); - -/** @} */ -// CLOSEOUT BackendAPI DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// FusedAdam APIs -/** @addtogroup SGD - * - * @{ - */ -/*! @brief Perform Fused Adam optimization for a single tensor (Adaptive Moment Estimation). - * - * This function implements the Fused Adam optimization algorithm. Adam, short for Adaptive Moment - * Estimation, extends the RMSProp optimizer. It combines the advantages of AdaGrad and RMSProp by - * adaptively adjusting learning rates for each parameter using the first and second moments of - * gradients. Fused Adam optimization efficiently combines multiple operations into a single kernel, - * reducing memory access overhead and improving performance. - * - * Additionally, Fused Adam can be utilized in both adam w and Automatic Mixed Precision (AMP), - * enabling accelerated model training and reduced memory consumption. AMP supports FP16 - * computation, optimizing model calculations using a mixture of FP32 and FP16 precision to enhance - * training speed. When utilizing AMP, FoundInf, ScaleGrad, and step tensors should be employed. In - * AMP mode, the execution of Adam is determined based on the FoundInf value. State Step accepts - * both int values and int tensors. If a Step tensor is employed, the step received as an int is - * disregarded, and if Adam is executed, the step tensor is incremented by 1. - * - * @code - * // Execute Adam - * miopenFusedAdam(handle, - * paramDesc, - * param, - * gradDesc, - * grad, - * expAvgDesc, - * expAvg, - * expAvgSqDesc, - * expAvgSq, - * NULL, // Unused maxExpAvgSqDesc because amsgrad is false - * NULL, - * NULL, // Unused stateStep Tensor because use step integer argument - * NULL, - * step, - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * false, // amsgrad - * false, // maximize - * false, // adamw - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * - * // Execute AdamW - * miopenFusedAdam(handle, - * paramDesc, - * param, - * gradDesc, - * grad, - * expAvgDesc, - * expAvg, - * expAvgSqDesc, - * expAvgSq, - * NULL, // Unused maxExpAvgSqDesc because amsgrad is false - * NULL, - * NULL, // Unused stateStep Tensor because use step integer argument - * NULL, - * step, - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * false, // amsgrad - * false, // maximize - * true, // adamw - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * - * // Execute AMP Adam - * miopenFusedAdam(handle, - * paramDesc, - * param, - * gradDesc, - * grad, - * expAvgDesc, - * expAvg, - * expAvgSqDesc, - * expAvgSq, - * NULL, // Unused maxExpAvgSqDesc because amsgrad is false - * NULL, - * stateStepDesc, - * stateStep, - * -1, // Ignore step value because stateStep Tensor is used - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * false, // amsgrad - * false, // maximize - * false, // adamw - * gradScaleDesc, - * gradScale, - * foundInfDesc, - * foundInf); - * @endcode - * - * @param handle MIOpen handle (input) - * @param paramDesc Tensor descriptor for the input parameter tensor (input) - * @param param Input parameter tensor (input) - * @param gradDesc Tensor descriptor for the input gradient tensor (input) - * @param grad Input gradient tensor (input) - * @param expAvgDesc Tensor descriptor for the input exponential moving average tensor - * (input) - * @param expAvg Input exponential moving average tensor (input) - * @param expAvgSqDesc Tensor descriptor for the input exponential moving average squared - * tensor (input) - * @param expAvgSq Input exponential moving average squared tensor (input) - * @param maxExpAvgSqDesc Tensor descriptor for the input maximum exponential moving average - * squared tensor. Used when amsgrad is true (input, optional) - * @param maxExpAvgSq Input maximum exponential moving average squared tensor. Used when - * amsgrad is true (input, optional) - * @param stateStepDesc Tensor descriptor for the input state step tensor (input) - * @param stateStep Input state step tensor (input) - * @param state_step Input state step. used when the step tensor is null (input) - * @param lr Learning rate (input) - * @param beta1 Coefficient used for computing the first moment running average of - * gradient (input) - * @param beta2 Coefficient used for computing the second moment running average of - * gradient (input) - * @param weight_decay Weight decay (input) - * @param eps Term added to the denominator to improve numerical stability (input) - * @param amsgrad Flag indicating whether to use the AMSGrad variant of Adam (input) - * @param maximize Flag indicating whether to maximize the objective with respect to the - * parameters (input) - * @param adamw If true, the operation becomes AdamW (input) - * @param gradScaleDesc Tensor descriptor for the input grad scale tensor (input, optional) - * @param gradScale Input grad scale tensor (input, optional) - * @param foundInfDesc Tensor descriptor for the input found inf tensor (input, optional) - * @param foundInf Tensor indicating the presence of inf or NaN in gradients. If true, - * skips operation and step update (input, optional) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenFusedAdam(miopenHandle_t handle, - const miopenTensorDescriptor_t paramDesc, - void* param, - const miopenTensorDescriptor_t gradDesc, - const void* grad, - const miopenTensorDescriptor_t expAvgDesc, - void* expAvg, - const miopenTensorDescriptor_t expAvgSqDesc, - void* expAvgSq, - const miopenTensorDescriptor_t maxExpAvgSqDesc, - void* maxExpAvgSq, - const miopenTensorDescriptor_t stateStepDesc, - void* stateStep, - const unsigned int state_step, - const float lr, - const float beta1, - const float beta2, - const float weight_decay, - const float eps, - const bool amsgrad, - const bool maximize, - const bool adamw, - const miopenTensorDescriptor_t gradScaleDesc, - const void* gradScale, - const miopenTensorDescriptor_t foundInfDesc, - const void* foundInf); - -/*! @brief Execute single tensor Adam optimization and receive the result in a separate output - * tensor. - * - * This function is equivalent to miopenFusedAdam but receives the result in a separate output - * tensor. - * @see miopenFusedAdam - * - * @code - * // Execute Adam - * miopenFusedAdamWithOutput(handle, - * paramInDesc, - * paramIn, - * paramOutDesc, - * paramOut, - * NULL, // Unused paramOutFloat16 tensor because is not amp - * NULL, - * gradInDesc, - * gradIn, - * expAvgInDesc, - * expAvgIn, - * expAvgOutDesc, - * expAvgOut, - * expAvgInSqDesc, - * expAvgSqIn, - * expAvgSqOutDesc, - * expAvgSqOut, - * NULL, // Unused maxExpAvgSqIn tensor because amsgrad is false - * NULL, - * NULL, // Unused maxExpAvgSqOut tensor because amsgrad is false - * NULL, - * NULL, // Unused stateStepIn tensor because use step integer argument - * NULL, - * NULL, // Unused stateStepOut tensor because use step integer argument - * NULL, - * step, - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * false, // amsgrad - * false, // maximize - * false, // adamw - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * - * // Execute Amp Adam - * miopenFusedAdamWithOutput(handle, - * paramInDesc, - * paramIn, - * paramOutDesc, - * paramOut, - * paramOutFloat16Desc, // paramOutFloat16 tensor is optional in amp - * paramOutFloat16, - * gradInDesc, - * gradIn, - * expAvgInDesc, - * expAvgIn, - * expAvgOutDesc, - * expAvgOut, - * expAvgInSqDesc, - * expAvgSqIn, - * expAvgSqIn, - * expAvgSqOutDesc, - * expAvgSqOut, - * NULL, // Unused maxExpAvgSqIn tensor because amsgrad is false - * NULL, - * NULL, // Unused maxExpAvgSqOut tensor because amsgrad is false - * NULL, - * stateStepInDesc, - * stateStepIn, - * stateStepOutDesc, - * stateStepOut - * -1, // Ignore step value because stateStep Tensor is used - * lr, beta1, beta2, weight_decay, eps, - * false, // amsgrad - * false, // maximize - * false, // adamw - * gradScaleDesc, - * gradScale, - * foundInfDesc, - * foundInf); - * @endcode - * - * @param handle MIOpen handle (input) - * @param paramInDesc Tensor descriptor for the input parameter tensor (input) - * @param paramIn Input parameter tensor (input) - * @param paramOutDesc Tensor descriptor for the output parameter tensor (input) - * @param paramOut Output parameter tensor (output) - * @param paramOutFloat16Desc Tensor descriptor for the output parameter tensor float16 (input, - * optional) - * @param paramOutFloat16 Output parameter tensor (output, optional) - * @param gradInDesc Tensor descriptor for the input gradient tensor (input) - * @param gradIn Input gradient tensor (input) - * @param expAvgInDesc Tensor descriptor for the input exponential moving average tensor - * (input) - * @param expAvgIn Input exponential moving average tensor (input) - * @param expAvgOutDesc Tensor descriptor for the output exponential moving average tensor - * (input) - * @param expAvgOut Output exponential moving average tensor (output) - * @param expAvgSqInDesc Tensor descriptor for the input exponential moving average squared - * tensor (input) - * @param expAvgSqIn Input exponential moving average squared tensor (input) - * @param expAvgSqOutDesc Tensor descriptor for the output exponential moving average squared - * tensor (input) - * @param expAvgSqOut Output exponential moving average squared tensor (output) - * @param maxExpAvgSqInDesc Tensor descriptor for the input maximum exponential moving average - * squared tensor. Used when amsgrad is true (input, optional) - * @param maxExpAvgSqIn Input maximum exponential moving average squared tensor. Used when - * amsgrad is true (input, optional) - * @param maxExpAvgSqOutDesc Tensor descriptor for the output maximum exponential moving average - * squared tensor. Used when amsgrad is true (input, optional) - * @param maxExpAvgSqOut Output maximum exponential moving average squared tensor. Used when - * amsgrad is true (output, optional) - * @param stateStepInDesc Tensor descriptor for the input state step tensor (input, optional) - * @param stateStepIn Input state step tensor (input, optional) - * @param stateStepOutDesc Tensor descriptor for the output state step tensor (input, optional) - * @param stateStepOut Output state step tensor that stores the updated step value. (output, - * optional) - * @param state_step Input state step, It is used when the step tensor is null. (input) - * @param lr Learning rate (input) - * @param beta1 Coefficient used for computing the first moment running average of - * gradient (input) - * @param beta2 Coefficient used for computing the second moment running average of - * gradient (input) - * @param weight_decay Weight decay (input) - * @param eps Term added to the denominator to improve numerical stability (input) - * @param amsgrad Flag indicating whether to use the AMSGrad variant of Adam (input) - * @param maximize Flag indicating whether to maximize the objective with respect to the - * parameters (input) - * @param adamw If it is true, the operation becomes AdamW (input) - * @param gradScaleDesc Tensor descriptor for the input grad scale tensor (input, optional) - * @param gradScale Input grad scale tensor (input, optional) - * @param foundInfDesc Tensor descriptor for the input found inf tensor (input, optional) - * @param foundInf Tensor indicating presence of inf or nan in gradients. If true, skips - * operation and step update. (input, optional) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenFusedAdamWithOutput(miopenHandle_t handle, - const miopenTensorDescriptor_t paramInDesc, - void* paramIn, - const miopenTensorDescriptor_t paramOutDesc, - void* paramOut, - const miopenTensorDescriptor_t paramOutFloat16Desc, - void* paramOutFloat16, - const miopenTensorDescriptor_t gradInDesc, - const void* gradIn, - const miopenTensorDescriptor_t expAvgInDesc, - void* expAvgIn, - const miopenTensorDescriptor_t expAvgOutDesc, - void* expAvgOut, - const miopenTensorDescriptor_t expAvgSqInDesc, - void* expAvgSqIn, - const miopenTensorDescriptor_t expAvgSqOutDesc, - void* expAvgSqOut, - const miopenTensorDescriptor_t maxExpAvgSqInDesc, - void* maxExpAvgSqIn, - const miopenTensorDescriptor_t maxExpAvgSqOutDesc, - void* maxExpAvgSqOut, - const miopenTensorDescriptor_t stateStepInDesc, - void* stateStepIn, - const miopenTensorDescriptor_t stateStepOutDesc, - void* stateStepOut, - const unsigned int state_step, - const float lr, - const float beta1, - const float beta2, - const float weight_decay, - const float eps, - const bool amsgrad, - const bool maximize, - const bool adamw, - const miopenTensorDescriptor_t gradScaleDesc, - const void* gradScale, - const miopenTensorDescriptor_t foundInfDesc, - const void* foundInf); - -/** @} */ -// CLOSEOUT SGD DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// TransformersAdamW APIs -/** @addtogroup SGD - * - * @{ - */ -/*! @brief Implements Adam algorithm with weight decay fix as introduced in - * Decoupled Weight Decay Regularization. - * This is the fused kernel version of AdamW included in the Hugging Face Transformers module. - * - * @see miopenFusedAdam - * - * @code - * // Execute Adam - * miopenTransformersAdamW(handle, - * paramDesc, - * param, - * gradDesc, - * grad, - * expAvgDesc, - * expAvg, - * expAvgSqDesc, - * expAvgSq, - * NULL, // Unused stateStep Tensor because use step integer argument - * NULL, - * step, - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * true, // correct_bias - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * - * // Execute AMP Adam - * miopenTransformersAdamW(handle, - * paramDesc, - * param, - * gradDesc, - * grad, - * expAvgDesc, - * expAvg, - * expAvgSqDesc, - * expAvgSq, - * stateStepDesc, - * stateStep, - * -1, // Ignore step value because stateStep Tensor is used - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * true, // correct_bias - * gradScaleDesc, - * gradScale, - * foundInfDesc, - * foundInf); - * @endcode - * - * @param handle MIOpen handle (input) - * @param paramDesc Tensor descriptor for the input parameter tensor (input) - * @param param Input parameter tensor (input) - * @param gradDesc Tensor descriptor for the input gradient tensor (input) - * @param grad Input gradient tensor (input) - * @param expAvgDesc Tensor descriptor for the input exponential moving average tensor - * (input) - * @param expAvg Input exponential moving average tensor (input) - * @param expAvgSqDesc Tensor descriptor for the input exponential moving average squared - * tensor (input) - * @param expAvgSq Input exponential moving average squared tensor (input) - * @param stateStepDesc Tensor descriptor for the input state step tensor (input) - * @param stateStep Input state step tensor (input) - * @param state_step Input state step. used when the step tensor is null (input) - * @param lr Learning rate (input) - * @param beta1 Coefficient used for computing the first moment running average of - * gradient (input) - * @param beta2 Coefficient used for computing the second moment running average of - * gradient (input) - * @param weight_decay Weight decay (input) - * @param eps Term added to the denominator to improve numerical stability (input) - * @param correct_bias Whether or not to correct bias in Adam (for instance, in Bert TF - * repository they use False). - * @param gradScaleDesc Tensor descriptor for the input grad scale tensor (input, optional) - * @param gradScale Input grad scale tensor (input, optional) - * @param foundInfDesc Tensor descriptor for the input found inf tensor (input, optional) - * @param foundInf Tensor indicating the presence of inf or NaN in gradients. If true, - * skips operation and step update (input, optional) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenTransformersAdamW(miopenHandle_t handle, - const miopenTensorDescriptor_t paramDesc, - void* param, - const miopenTensorDescriptor_t gradDesc, - const void* grad, - const miopenTensorDescriptor_t expAvgDesc, - void* expAvg, - const miopenTensorDescriptor_t expAvgSqDesc, - void* expAvgSq, - const miopenTensorDescriptor_t stateStepDesc, - void* stateStep, - const unsigned int state_step, - const float lr, - const float beta1, - const float beta2, - const float weight_decay, - const float eps, - const bool correct_bias, - const miopenTensorDescriptor_t gradScaleDesc, - const void* gradScale, - const miopenTensorDescriptor_t foundInfDesc, - const void* foundInf); - -/*! @brief Execute single tensor Adam optimization and receive the result in a separate output - * tensor. - * - * This function is equivalent to miopenTransformersAdam but receives the result in a separate - * output tensor. - * @see miopenTransformersAdamW - * @see miopenFusedAdamWithOutput - * - * @code - * // Execute Adam - * miopenTransformersAdamWWithOutput(handle, - * paramInDesc, - * paramIn, - * paramOutDesc, - * paramOut, - * NULL, // Unused paramOutFloat16 tensor because is not amp - * NULL, - * gradInDesc, - * gradIn, - * expAvgInDesc, - * expAvgIn, - * expAvgOutDesc, - * expAvgOut, - * expAvgInSqDesc, - * expAvgSqIn, - * expAvgSqOutDesc, - * expAvgSqOut, - * NULL, // Unused stateStepIn tensor because use step int - * NULL, - * NULL, // Unused stateStepOut tensor because use step int - * NULL, - * step, - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * -1, // step_size - * true, // correct_bias - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * - * // Execute Amp Adam - * miopenTransformersAdamWWithOutput(handle, - * paramInDesc, - * paramIn, - * paramOutDesc, - * paramOut, - * paramOutFloat16Desc, // optional in amp - * paramOutFloat16, - * gradInDesc, - * gradIn, - * expAvgInDesc, - * expAvgIn, - * expAvgOutDesc, - * expAvgOut, - * expAvgInSqDesc, - * expAvgSqIn, - * expAvgSqIn, - * expAvgSqOutDesc, - * expAvgSqOut, - * stateStepInDesc, - * stateStepIn, - * stateStepOutDesc, - * stateStepOut - * -1, // Ignore step value because stateStep Tensor is used - * lr, - * beta1, - * beta2, - * weight_decay, - * eps, - * -1, // step_size - * true, // correct_bias - * NULL, // Unused gradScale Tensor because not amp - * NULL, - * NULL, // Unused foundInf Tensor because not amp - * NULL); - * @endcode - * - * @param handle MIOpen handle (input) - * @param paramInDesc Tensor descriptor for the input parameter tensor (input) - * @param paramIn Input parameter tensor (input) - * @param paramOutDesc Tensor descriptor for the output parameter tensor (input) - * @param paramOut Output parameter tensor (output) - * @param paramOutFloat16Desc Tensor descriptor for the output parameter tensor float16 (input, - * optional) - * @param paramOutFloat16 Output parameter tensor (output, optional) - * @param gradInDesc Tensor descriptor for the input gradient tensor (input) - * @param gradIn Input gradient tensor (input) - * @param expAvgInDesc Tensor descriptor for the input exponential moving average tensor - * (input) - * @param expAvgIn Input exponential moving average tensor (input) - * @param expAvgOutDesc Tensor descriptor for the output exponential moving average tensor - * (input) - * @param expAvgOut Output exponential moving average tensor (output) - * @param expAvgSqInDesc Tensor descriptor for the input exponential moving average squared - * tensor (input) - * @param expAvgSqIn Input exponential moving average squared tensor (input) - * @param expAvgSqOutDesc Tensor descriptor for the output exponential moving average squared - * tensor (input) - * @param expAvgSqOut Output exponential moving average squared tensor (output) - * @param stateStepInDesc Tensor descriptor for the input state step tensor (input, optional) - * @param stateStepIn Input state step tensor (input, optional) - * @param stateStepOutDesc Tensor descriptor for the output state step tensor (input, optional) - * @param stateStepOut Output state step tensor that stores the updated step value. (output, - * optional) - * @param state_step Input state step, It is used when the step tensor is null. (input) - * @param lr Learning rate (input) - * @param beta1 Coefficient used for computing the first moment running average of - * gradient (input) - * @param beta2 Coefficient used for computing the second moment running average of - * gradient (input) - * @param weight_decay Weight decay (input) - * @param eps Term added to the denominator to improve numerical stability (input) - * @param step_size Pre-calculated step_size, used for performance enhancement (input) - * @param correct_bias Whether or not to correct bias in Adam (for instance, in Bert TF - * repository they use False) (input) - * @param gradScaleDesc Tensor descriptor for the input grad scale tensor (input, optional) - * @param gradScale Input grad scale tensor (input, optional) - * @param foundInfDesc Tensor descriptor for the input found inf tensor (input, optional) - * @param foundInf Tensor indicating presence of inf or nan in gradients. If true, skips - * operation and step update. (input, optional) - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenTransformersAdamWWithOutput(miopenHandle_t handle, - const miopenTensorDescriptor_t paramInDesc, - void* paramIn, - const miopenTensorDescriptor_t paramOutDesc, - void* paramOut, - const miopenTensorDescriptor_t paramOutFloat16Desc, - void* paramOutFloat16, - const miopenTensorDescriptor_t gradInDesc, - const void* gradIn, - const miopenTensorDescriptor_t expAvgInDesc, - void* expAvgIn, - const miopenTensorDescriptor_t expAvgOutDesc, - void* expAvgOut, - const miopenTensorDescriptor_t expAvgSqInDesc, - void* expAvgSqIn, - const miopenTensorDescriptor_t expAvgSqOutDesc, - void* expAvgSqOut, - const miopenTensorDescriptor_t stateStepInDesc, - void* stateStepIn, - const miopenTensorDescriptor_t stateStepOutDesc, - void* stateStepOut, - const unsigned int state_step, - const float lr, - const float beta1, - const float beta2, - const float weight_decay, - const float eps, - const float step_size, - const bool correct_bias, - const miopenTensorDescriptor_t gradScaleDesc, - const void* gradScale, - const miopenTensorDescriptor_t foundInfDesc, - const void* foundInf); - -/** @} */ -// CLOSEOUT SGD DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// GetItem APIs -/** @addtogroup getitem - * - * @{ - */ -/*! @brief Helper function to query the minimum workspace size required by the getitem call - * - * @param [in] handle MIOpen Handle - * @param [in] indexCount Number of input tensor indexs - * @param [in] indexDescs Tensor descriptor of input tensor indexs - * @param [out] sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetGetitemWorkspaceSize(miopenHandle_t handle, - uint32_t indexCount, - const miopenTensorDescriptor_t* indexDescs, - size_t* sizeInBytes); - -/*! @brief Execute a getitem backward layer - * - * Backward of getitem for tensor indexing, slicing, masking. - * - * @param [in] handle MIOpen handle - * @param [in] workspace Address of the allocated workspace data - * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data - * @param [in] dyDesc Tensor descriptor of input tensor dy - * @param [in] dy Source data tensor dy - * @param [in] indexCount Number of input tensor indexs - * @param [in] indexDescs Tensor descriptor of input tensor indexs(All indexs same - * size) - * @param [in] indexs Source data tensor indexs - * @param [in] dxDesc Tensor descriptor of output tensor dx - * @param [out] dx Data tensor dx(It must be initialized to 0) - * @param [in] errorDesc Tensor descriptor of output tensor error - * @param [out] error Data tensor error(It must be initialized to 0) - * @param [in] dimCount Number of dimensions - * @param [in] dims Dimensions - * @param [in] sliceCount Number of slices - * @param [in] slices Slices - * @param [in] offset Offset of output tensor dx - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenGetitemBackward(miopenHandle_t handle, - void* workspace, - size_t workspaceSizeInBytes, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - uint32_t indexCount, - const miopenTensorDescriptor_t* indexDescs, - const void* const* indexs, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const miopenTensorDescriptor_t errorDesc, - void* error, - uint32_t dimCount, - const int32_t* dims, - uint32_t sliceCount, - const int32_t* slices, - uint32_t offset); - -/** @} */ -// CLOSEOUT GETITEM DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -// RotaryPositionalEmbeddings APIs -/** @addtogroup RotaryPositionalEmbeddings - * - * @{ - */ -/*! @brief Execute a rope forward layer - * - * @param [in] handle MIOpen handle - * @param [in] xDesc Tensor descriptor for data input tensor x - * @param [in] x Data tensor x - * @param [in] cosDesc Tensor descriptor for data input tensor cos - * @param [in] cos Data tensor cos - * @param [in] sinDesc Tensor descriptor for data input tensor sin - * @param [in] sin Data tensor sin - * @param [in] yDesc Tensor descriptor for output data tensor y - * @param [out] y Data tensor y - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRoPEForward(miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t cosDesc, - const void* cos, - const miopenTensorDescriptor_t sinDesc, - const void* sin, - const miopenTensorDescriptor_t yDesc, - void* y); - -/*! @brief Execute a rope backward layer - * - * @param [in] handle MIOpen handle - * @param [in] dyDesc Tensor descriptor for data input tensor dy - * @param [in] dy Data tensor dy - * @param [in] cosDesc Tensor descriptor for output data tensor cos - * @param [in] cos Data tensor cos - * @param [in] sinDesc Tensor descriptor for data input tensor sin - * @param [in] sin Data tensor sin - * @param [in] dxDesc Tensor descriptor for output data tensor dx - * @param [out] dx Data tensor dx - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t miopenRoPEBackward(miopenHandle_t handle, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t cosDesc, - const void* cos, - const miopenTensorDescriptor_t sinDesc, - const void* sin, - const miopenTensorDescriptor_t dxDesc, - void* dx); -/** @} */ -// CLOSEOUT ROPE DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef MIOPEN_BETA_API -/** @addtogroup ReLU - * - * @{ - */ - -/*! @brief Helper function to query the minimum workspace size required by the PReLU backward call - * - * @param handle MIOpen Handle (input) - * @param inputDesc Tensor descriptor for input tensor (input) - * @param weightDesc Tensor descriptor for weight tensor (input) - * @param sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t - */ -MIOPEN_EXPORT miopenStatus_t -miopenGetPReLUBackwardWorkspaceSize(miopenHandle_t handle, - miopenTensorDescriptor_t inputDesc, - miopenTensorDescriptor_t weightDesc, - size_t* sizeInBytes); - -/*! @brief Execute a PReLU backward layer - * - * @param handle MIOpen handle (input) - * @param workspace Address of the allocated workspace data (input) - * @param workspaceSizeInBytes Size in bytes of the allocated workspace data (input) - * @param inputDesc Tensor descriptor for input tensor (input) - * @param input Data tensor input (input) - * @param weightDesc Tensor descriptor for weight tensor (input) - * @param weight Data tensor weight (input) - * @param doutputDesc Tensor descriptor for output gradient (input) - * @param doutput Gradient of output (input) - * @param dinputDesc Tensor descriptor for input gradient (input) - * @param dinput Gradient of input (output) - * @param dweightDesc Tensor descriptor for weight gradient (input) - * @param dweight Gradient of weight (output) - */ -MIOPEN_EXPORT miopenStatus_t miopenPReLUBackward(miopenHandle_t handle, - void* workspace, - size_t workspaceSizeInBytes, - miopenTensorDescriptor_t inputDesc, - const void* input, - miopenTensorDescriptor_t weightDesc, - const void* weight, - miopenTensorDescriptor_t doutputDesc, - const void* doutput, - miopenTensorDescriptor_t dinputDesc, - void* dinput, - miopenTensorDescriptor_t dweightDesc, - void* dweight); - -/** @} */ -// CLOSEOUT RELU DOXYGEN GROUP -#endif // MIOPEN_BETA_API - -#ifdef __cplusplus -} -#endif - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -#endif // MIOPEN_GUARD_MIOPEN_H_ diff --git a/requirements.txt b/requirements.txt index ffe4c3acb5..e69de29bb2 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +0,0 @@ -sqlite3@3.43.2 -DCMAKE_POSITION_INDEPENDENT_CODE=On -boost@1.83 -DCMAKE_POSITION_INDEPENDENT_CODE=On --build -DCMAKE_CXX_FLAGS=" -std=c++14 -Wno-enum-constexpr-conversion -Wno-deprecated-builtins -Wno-deprecated-declarations " -facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake -# ROCm/half@10abd99e7815f0ca5d892f58dd7d15a23b7cf92c --build -ROCm/rocMLIR@rocm-5.5.0 -H sha256:a5f62769d28a73e60bc8d61022820f050e97c977c8f6f6275488db31512e1f42 -DBUILD_FAT_LIBROCKCOMPILER=1 -DCMAKE_IGNORE_PATH="/opt/conda/envs/py_3.8;/opt/conda/envs/py_3.9;/opt/conda/envs/py_3.10" -DCMAKE_IGNORE_PREFIX_PATH=/opt/conda -nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off -ROCm/FunctionalPlus@v0.2.18-p0 -ROCm/eigen@3.4.0 -ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@9c0811f39a2262dbe4d71b81898187951c1e11ba -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON -google/googletest@v1.14.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c4ffeede18..e69de29bb2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,1051 +0,0 @@ -################################################################################ -# -# MIT License -# -# Copyright (c) 2017 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -cmake_policy(SET CMP0057 NEW) - -include(ExportHeader) -if(MIOPEN_ENABLE_SQLITE) - add_subdirectory(sqlite) -endif() - -# Truncation rounding or (default) rounding to nearest even (RNE) is enabled. -# This switch controls two related but different aspects of MIOpen behavior -# 1. How host code performs conversions of float to bfloat16, important only -# for testing. -# 2. How BF16 kernels (which are kind of mixed-precision now and expected to -# remain in the future) perform final conversion (and rounding) of FP32 -# to BF16 results. This affects the main functionality of the library. -option( MIOPEN_USE_RNE_BFLOAT16 "Sets rounding scheme for bfloat16 type" ON ) -option( MIOPEN_FP8_IEEE_EXPONENT_BIAS "Sets the FP8 exponent bias to IEEE" OFF) -option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) -set ( MIOPEN_DEFAULT_FIND_MODE "DynamicHybrid" CACHE STRING "Sets the default find mode") -set_property(CACHE MIOPEN_DEFAULT_FIND_MODE PROPERTY STRINGS - Normal Fast Hybrid FastHybrid DynamicHybrid) - -option( MIOPEN_FP8_CLIPPING "Sets the FP8 clipping" ON) - -configure_file("${PROJECT_SOURCE_DIR}/include/miopen/config.h.in" "${PROJECT_BINARY_DIR}/include/miopen/config.h") - -# configure a header file to pass the CMake version settings to the source, and package the header files in the output archive -configure_file( "${PROJECT_SOURCE_DIR}/include/miopen/version.h.in" "${PROJECT_BINARY_DIR}/include/miopen/version.h" ) - -if(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY) -#Copy header to Build Dir for generating backward compatibility -configure_file( "${PROJECT_SOURCE_DIR}/include/miopen/miopen.h" "${PROJECT_BINARY_DIR}/include/miopen/miopen.h" ) -endif() - -message( STATUS "MIOpen_VERSION= ${MIOpen_VERSION}" ) -if(NOT MIOPEN_GENERATOR_IS_MULTI_CONFIG) - message( STATUS "CMAKE_BUILD_TYPE= ${CMAKE_BUILD_TYPE}" ) -endif() - -# This is incremented when the ABI to the library changes -set( MIOpen_SOVERSION 1.0 ) - -function(add_kernels FILE_NAME VAR_PREFIX VAR_SUFFIX KERNEL_FILES) - set(INIT_KERNELS_LIST) - set(KERNELS_DECLS) - foreach(KERNEL_FILE ${KERNEL_FILES}) - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${KERNEL_FILE}) - get_filename_component(KERNEL_FILENAME ${KERNEL_FILE} NAME) - get_filename_component(BASE_NAME ${KERNEL_FILE} NAME_WE) - string(TOUPPER "${BASE_NAME}" KEY_NAME) - string(MAKE_C_IDENTIFIER "${KEY_NAME}" VAR_NAME) - string(APPEND KERNELS_DECLS "extern const size_t ${VAR_PREFIX}${VAR_NAME}${VAR_SUFFIX}_SIZE;\n") - string(APPEND KERNELS_DECLS "extern const char ${VAR_PREFIX}${VAR_NAME}${VAR_SUFFIX}[];\n") - list(APPEND INIT_KERNELS_LIST " { \"${KERNEL_FILENAME}\", { ${VAR_PREFIX}${VAR_NAME}${VAR_SUFFIX}, ${VAR_PREFIX}${VAR_NAME}${VAR_SUFFIX}_SIZE } }") - endforeach() - string(REPLACE ";" ",\n" INIT_KERNELS "${INIT_KERNELS_LIST}") - configure_file(kernels/${FILE_NAME}.in ${PROJECT_BINARY_DIR}/${FILE_NAME}) -endfunction() - -set( MIOpen_Source - activ/problem_description.cpp - activ_api.cpp - adam/problem_description.cpp - adam_api.cpp - addlayernorm_api.cpp - api/find2_0_commons.cpp - batch_norm.cpp - batch_norm_api.cpp - batchnorm/problem_description.cpp - buffer_info.cpp - cat_api.cpp - cat/problem_description.cpp - check_numerics.cpp - conv/invokers/gcn_asm_1x1u.cpp - conv/invokers/gcn_asm_1x1u_ss.cpp - conv/invokers/gcn_asm_1x1u_us.cpp - conv/invokers/gcn_asm_wino.cpp - conv/invokers/gen_x_w_y_pad.cpp - conv/invokers/impl_gemm.cpp - conv/invokers/impl_gemm_dynamic.cpp - conv/invokers/ocl_wrw_rdc.cpp - conv/kernel_interface/winograd_kernel_interface.cpp - conv/problem_description.cpp - conv/solver_finders.cpp - conv_algo_name.cpp - convolution.cpp - convolution_api.cpp - ctc.cpp - ctc_api.cpp - db.cpp - db_record.cpp - driver_arguments.cpp - dropout.cpp - dropout_api.cpp - env.cpp - execution_context.cpp - expanduser.cpp - find_controls.cpp - find_db.cpp - fused_api.cpp - fusion.cpp - fusion/problem_description.cpp - generic_search.cpp - getitem_api.cpp - glu/problem_description.cpp - glu_api.cpp - graphapi/convolution.cpp - graphapi/conv_bias_res_add_activ_forward_executor.cpp - graphapi/engine.cpp - graphapi/enginecfg.cpp - graphapi/engineheur.cpp - graphapi/execution_plan.cpp - graphapi/find_engine.cpp - graphapi/graphapi.cpp - graphapi/matmul.cpp - graphapi/opgraph.cpp - graphapi/pointwise.cpp - graphapi/reduction.cpp - graphapi/reshape.cpp - graphapi/rng.cpp - graphapi/tensor.cpp - graphapi/variant_pack.cpp - groupnorm_api.cpp - groupnorm/problem_description.cpp - handle_api.cpp - invoker_cache.cpp - getitem/problem_description.cpp - kernel_build_params.cpp - kernel_warnings.cpp - layernorm_api.cpp - layernorm/problem_description.cpp - load_file.cpp - lock_file.cpp - logger.cpp - lrn_api.cpp - mha/mha_descriptor.cpp - mha/problem_description.cpp - op_args.cpp - operator.cpp - performance_config.cpp - pooling/problem_description.cpp - pooling_api.cpp - prelu/problem_description.cpp - prelu_api.cpp - problem.cpp - process.cpp - ramdb.cpp - readonlyramdb.cpp - reducecalculation_api.cpp - reduceextreme_api.cpp - reducetensor.cpp - reducetensor_api.cpp - reduce/problem_description.cpp - rnn.cpp - rnn_api.cpp - rnn/rnn_util.cpp - rnn/selector.cpp - rnn/Solutions/rnn_transformer.cpp - rnn/Solutions/Base/bw_weights_modular.cpp - rnn/Solutions/Base/bw_data_modular.cpp - rnn/Solutions/bwd_s_stream.cpp - rnn/Solutions/bwd_multi_stream.cpp - rnn/Solutions/bww_s_steam.cpp - rnn/Solutions/bww_multi_stream.cpp - rope_api.cpp - rope/problem_description.cpp - scalar.cpp - softmax.cpp - softmax_api.cpp - softmax/problem_description.cpp - solution.cpp - solver.cpp - solver/activ/bwd_0.cpp - solver/activ/bwd_1.cpp - solver/activ/fwd_0.cpp - solver/activ/fwd_1.cpp - solver/adam/adam.cpp - solver/adam/transformers_adam_w.cpp - solver/batchnorm/backward_ck.cpp - solver/batchnorm/backward_per_activation.cpp - solver/batchnorm/backward_per_activation_fused.cpp - solver/batchnorm/backward_spatial_multiple.cpp - solver/batchnorm/backward_spatial_single.cpp - solver/batchnorm/forward_inference.cpp - solver/batchnorm/forward_inference_ck.cpp - solver/batchnorm/forward_inference_fused.cpp - solver/batchnorm/forward_per_activation.cpp - solver/batchnorm/forward_per_activation_fused.cpp - solver/batchnorm/forward_spatial_multiple.cpp - solver/batchnorm/forward_spatial_single.cpp - solver/batchnorm/forward_training_ck.cpp - solver/cat/forward_cat.cpp - solver/conv/conv_asm_1x1u.cpp - solver/conv/conv_asm_1x1u_stride2.cpp - solver/conv/conv_asm_3x3u.cpp - solver/conv/conv_asm_5x10u2v2b1.cpp - solver/conv/conv_asm_5x10u2v2f1.cpp - solver/conv/conv_asm_7x7c3h224w224k64u2v2p3q3f1.cpp - solver/conv/conv_asm_dir_BwdWrW1x1.cpp - solver/conv/conv_asm_dir_BwdWrW3x3.cpp - solver/conv/conv_asm_implicit_gemm_bwd_v4r1_dynamic.cpp - solver/conv/conv_asm_implicit_gemm_gtc_bwd.cpp - solver/conv/conv_asm_implicit_gemm_gtc_bwd_nhwc.cpp - solver/conv/conv_asm_implicit_gemm_gtc_fwd.cpp - solver/conv/conv_asm_implicit_gemm_gtc_fwd_nhwc.cpp - solver/conv/conv_asm_implicit_gemm_gtc_fwd_nchwc.cpp - solver/conv/conv_asm_implicit_gemm_gtc_perf_config.cpp - solver/conv/conv_asm_implicit_gemm_gtc_wrw_nhwc.cpp - solver/conv/conv_asm_implicit_gemm_v4r1_dynamic.cpp - solver/conv/conv_asm_implicit_gemm_wrw_gtc_dynamic_xdlops.cpp - solver/conv/conv_asm_implicit_gemm_wrw_v4r1_dynamic.cpp - solver/conv/conv_bin_wino3x3U.cpp - solver/conv/conv_bin_winoRxS.cpp - solver/conv/conv_ck_igemm_fwd_v6r1_dlops_nchw.cpp - solver/conv/conv_direct_naive_conv.cpp - solver/conv/conv_direct_naive_conv_bwd.cpp - solver/conv/conv_direct_naive_conv_fwd.cpp - solver/conv/conv_direct_naive_conv_wrw.cpp - solver/conv/conv_hip_implicit_gemm_bwd_data_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_bwd_v1r1.cpp - solver/conv/conv_hip_implicit_gemm_bwd_v1r1_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_bwd_v4r1.cpp - solver/conv/conv_hip_implicit_gemm_bwd_v4r1_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_fwd_v4r1.cpp - solver/conv/conv_hip_implicit_gemm_fwd_v4r4.cpp - solver/conv/conv_hip_implicit_gemm_fwd_v4r4_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_fwd_v4r4_xdlops_padded_gemm.cpp - solver/conv/conv_hip_implicit_gemm_fwd_v4r5_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_fwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_grouped_fwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_grouped_bwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_grouped_wrw_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_fwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_bwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_wrw_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_nonxdlops_common.cpp - solver/conv/conv_hip_implicit_gemm_wrw_v4r4.cpp - solver/conv/conv_hip_implicit_gemm_wrw_v4r4_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_wrw_v4r4_xdlops_padded_gemm.cpp - solver/conv/conv_MP_bidirectional_winograd.cpp - solver/conv/conv_mlir_igemm_bwd.cpp - solver/conv/conv_mlir_igemm_bwd_xdlops.cpp - solver/conv/conv_mlir_igemm_fwd.cpp - solver/conv/conv_mlir_igemm_fwd_xdlops.cpp - solver/conv/conv_mlir_igemm_wrw.cpp - solver/conv/conv_mlir_igemm_wrw_xdlops.cpp - solver/conv/conv_multipass_wino3x3WrW.cpp - solver/conv/conv_ocl_dir2D_bwdWrW_1x1.cpp - solver/conv/conv_ocl_dir2D_bwdWrW_2.cpp - solver/conv/conv_ocl_dir2D_bwdWrW_53.cpp - solver/conv/conv_ocl_dir2D11x11.cpp - solver/conv/conv_ocl_dir2Dfwd.cpp - solver/conv/conv_ocl_dir2Dfwd_exhaustive_search.cpp - solver/conv/conv_ocl_dir2Dfwd1x1.cpp - solver/conv/conv_ocl_dir2Dfwdgen.cpp - solver/conv/conv_wino_fury_RxS.cpp - solver/conv/conv_winoRxS.cpp - solver/conv/fft.cpp - solver/conv/gemm.cpp - solver/conv/gemm_bwd.cpp - solver/conv/gemm_common.cpp - solver/conv/gemm_wrw.cpp - solver/conv_asm_1x1u_bias_activ_fused.cpp - solver/conv_bin_winoRxS_fused.cpp - solver/conv_ck_igemm_fwd_bias_activ_fused.cpp - solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp - solver/conv_ocl_dir2Dfwd_fused.cpp - solver/conv_winoRxS_fused.cpp - solver/glu/backward_glu.cpp - solver/glu/forward_glu.cpp - solver/groupnorm/forward_groupnorm.cpp - solver/getitem/backward_getitem.cpp - solver/layernorm/backward_t5layernorm.cpp - solver/layernorm/forward_addlayernorm.cpp - solver/layernorm/forward_layernorm.cpp - solver/layernorm/forward_layernorm2d_ck.cpp - solver/layernorm/forward_layernorm4d_ck.cpp - solver/layernorm/forward_t5layernorm.cpp - solver/mha/mha_solver_backward.cpp - solver/mha/mha_solver_forward.cpp - solver/pooling/forward2d.cpp - solver/pooling/forwardNaive.cpp - solver/pooling/forwardNd.cpp - solver/pooling/backward2d.cpp - solver/pooling/backwardNd.cpp - solver/prelu/backward_prelu_multi_weights.cpp - solver/prelu/backward_prelu_single_weight.cpp - solver/prelu/utils.cpp - solver/reduce/forward_argmax.cpp - solver/reduce/forward_argmin.cpp - solver/reduce/forward_max.cpp - solver/reduce/forward_min.cpp - solver/reduce/forward_prod.cpp - solver/reduce/forward_sum.cpp - solver/rope/backward_rope.cpp - solver/rope/forward_rope.cpp - solver/softmax/attn_softmax.cpp - solver/softmax/softmax.cpp - subbuffers.cpp - t5layernorm_api.cpp - target_properties.cpp - temp_file.cpp - tensor.cpp - tensor_api.cpp - transformers_adam_w_api.cpp - seq_tensor.cpp -) - -if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) - list(APPEND MIOpen_Source conv/heuristics/ai_heuristics.cpp) - list(APPEND MIOpen_Source anyramdb.cpp) -endif() - -list(APPEND MIOpen_Source tmp_dir.cpp binary_cache.cpp md5.cpp) -if(MIOPEN_ENABLE_SQLITE) - list(APPEND MIOpen_Source sqlite_db.cpp) -endif() - -if(MIOPEN_ENABLE_SQLITE AND MIOPEN_ENABLE_SQLITE_KERN_CACHE) - list(APPEND MIOpen_Source kern_db.cpp bz2.cpp) -endif() - -if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") - file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_INCLUDE "kernels/static_composable_kernel/include/*/*.hpp") - file(GLOB_RECURSE STATIC_COMPOSABLE_KERNEL_SOURCE "kernels/static_composable_kernel/src/*/*.cpp") - file(GLOB_RECURSE COMPOSABLE_KERNEL_INCLUDE "composable_kernel/composable_kernel/include/*.hpp") - file(GLOB_RECURSE COMPOSABLE_KERNEL_SOURCE "composable_kernel/composable_kernel/src/*.cpp") - file(GLOB_RECURSE COMPOSABLE_KERNEL_DYNAMIC_ASM_SOURCE "kernels/dynamic_igemm/*.s") - file(GLOB_RECURSE COMPOSABLE_KERNEL_DYNAMIC_ASM_INCLUDE "kernels/dynamic_igemm/*.inc") - file(GLOB_RECURSE COMPOSABLE_KERNEL_DYNAMIC_CPP_SOURCE "kernels/dynamic_igemm/*.cpp") - file(GLOB_RECURSE GPU_REFERENCE_KERNEL_HIP "kernels/gpu_reference_kernel/*.cpp") - file(GLOB_RECURSE GPU_REFERENCE_KERNEL_ASM "kernels/gpu_reference_kernel/*.s") - file(GLOB_RECURSE GPU_BATCHED_TRANSPOSE_KERNEL_HIP "kernels/gpu_batched_transpose_kernel/*.cpp") - file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_INCLUDE "kernels/gpu_general_tensor_reorder_kernel/*.hpp") - file(GLOB_RECURSE GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_SOURCE "kernels/gpu_general_tensor_reorder_kernel/*.cpp") - - - set(MIOPEN_KERNEL_INCLUDES - ${STATIC_COMPOSABLE_KERNEL_INCLUDE} - ${COMPOSABLE_KERNEL_INCLUDE} - ${COMPOSABLE_KERNEL_DYNAMIC_ASM_INCLUDE} - ${GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_INCLUDE} - include/miopen/implicitgemm_params.hpp - kernels/activation_functions.hpp - kernels/gpu_reference_kernel/fp8_kern_types.h - kernels/Conv_Winograd_v13_3_12_fp16dot_stride1.inc - kernels/Conv_Winograd_v13_3_12_fp16dot_stride2_dec.inc - kernels/Conv_Winograd_v13_3_12_fp16dot_stride2_dil.inc - kernels/Conv_Winograd_v14_3_3_fp16dot_stride1.inc - kernels/Conv_Winograd_v14_3_3_fp16dot_stride2_dec.inc - kernels/Conv_Winograd_v14_3_3_fp16dot_stride2_dil.inc - kernels/Conv_Winograd_v13_3_12_epilogue.inc - kernels/Conv_Winograd_v13_3_12_prologue.inc - kernels/Conv_Winograd_v16_5_0_epilogue.inc - kernels/Conv_Winograd_v16_5_0_prologue.inc - kernels/Conv_Winograd_v16_5_0_stride1.inc - kernels/conv_3x3_wheel_alpha_v9_2_7_epilogue.inc - kernels/conv_3x3_wheel_alpha_v9_2_7_prologue.inc - kernels/conv_3x3_wheel_alpha_v9_2_7_gfx8_stride_2_dec.inc - kernels/conv_3x3_wheel_alpha_v9_2_7_gfx8.inc - kernels/conv_3x3_wheel_alpha_v3_0b_epilogue.inc - kernels/conv_3x3_wheel_alpha_v3_0b_prologue.inc - kernels/conv_3x3_wheel_alpha_v3_0b.inc - kernels/conv_3x3_wheel_alpha_v7_0_3b_epilogue.inc - kernels/conv_3x3_wheel_alpha_v7_0_3b_prologue.inc - kernels/conv_3x3_wheel_alpha_v7_0_3b.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_epilogue.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_prologue.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx8_stride_2_dil.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx8_stride_2_dec.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx8.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx9_stride_2_dil.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx9_stride_2_dec.inc - kernels/conv_3x3_wheel_alpha_v9_0_15_gfx9.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp16_dot2_edc_f2x3_dilation2.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp16_dot2_edc_f2x3_stride1.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp16_dot2_edc_f2x3_stride2.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp16_dot2_edc_f3x2_stride1.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp32_f2x3_dilation2.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp32_f2x3_stride1.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp32_f2x3_stride2.inc - kernels/Conv_Winograd_v21_1_3_gfx9_fp32_f3x2_stride1.inc - kernels/Conv_Winograd_v21_1_3_metadata.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f2x3_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f2x3_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f2x3_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp16_dot2_edc_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx94x_fp32_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp16_dot2_edc_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx9_fp32_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp16_dot2_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx10_fp32_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp16_dot2_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f3x2_dilation2.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f3x2_stride1.inc - kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f3x2_stride2.inc - kernels/Conv_Winograd_v30_3_1_metadata.inc - kernels/MIOpenReduceCalculation.hpp - kernels/MIOpenReduceExtreme.hpp - kernels/bfloat16_dev.hpp - kernels/block_reduce.hpp - kernels/conv_common.inc - kernels/conv_sizes.inc - kernels/float_types.h - kernels/gpr_alloc.inc - kernels/hip_atomic.hpp - kernels/hip_f8_impl.hpp - kernels/hip_float8.hpp - kernels/inst_wrappers.inc - kernels/miopen_cstdint.hpp - kernels/miopen_limits.hpp - kernels/miopen_rocrand.hpp - kernels/miopen_type_traits.hpp - kernels/miopen_utility.hpp - kernels/neuron.inc - kernels/rocm_version.inc - kernels/stride_array.hpp - kernels/tensor_view.hpp - kernels/utilities.inc - kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1536vgprs_fp16_fp16acc_f2x3_c16_stride1.inc - kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1536vgprs_fp16_fp16acc_f2x3_c32_stride1.inc - kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1024vgprs_fp16_fp16acc_f2x3_c16_stride1.inc - kernels/winograd/Conv_Winograd_Fury_v2_4_1_metadata.inc - kernels/workaround_issue_1431.hpp - kernels/warp_reduce.hpp - kernels/xform_bidirect_winograd_code.inc - kernels/xform_data_filter.inc - kernels/xform_kd_cov2.inc - kernels/xform_metadata.inc - ) - - set(MIOPEN_KERNELS - ${STATIC_COMPOSABLE_KERNEL_SOURCE} - ${COMPOSABLE_KERNEL_SOURCE} - ${COMPOSABLE_KERNEL_DYNAMIC_ASM_SOURCE} - ${COMPOSABLE_KERNEL_DYNAMIC_CPP_SOURCE} - ${GPU_REFERENCE_KERNEL_HIP} - ${GPU_REFERENCE_KERNEL_ASM} - ${GPU_BATCHED_TRANSPOSE_KERNEL_HIP} - ${GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_SOURCE} - kernels/MIOpenAdam.cpp - kernels/MIOpenCat.cpp - kernels/MIOpenCheckNumerics.cpp - kernels/MIOpenBatchNormActivBwdPerAct.cl - kernels/MIOpenBatchNormActivBwdSpatial.cl - kernels/MIOpenBatchNormActivFwdTrainPerAct.cl - kernels/MIOpenBatchNormActivFwdTrainSpatial.cl - kernels/MIOpenBatchNormFwdTrainSpatial.cl - kernels/MIOpenBatchNormFwdTrainPerAct.cl - kernels/MIOpenBatchNormFwdInferSpatial.cl - kernels/MIOpenBatchNormFwdInferPerAct.cl - kernels/MIOpenBatchNormBwdSpatial.cl - kernels/MIOpenBatchNormBwdPerAct.cl - kernels/MIOpenConvDirUni.cl - kernels/MIOpenConvDirBatchNormActiv.cl - kernels/MIOpenConvDirGenFwd.cl - kernels/MIOpenGLU.cpp - kernels/MIOpenGroupNorm.cpp - kernels/MIOpenGetitem.cpp - kernels/MIOpenLayerNorm.cpp - kernels/MIOpenLRNBwd.cl - kernels/MIOpenLRNFwd.cl - kernels/MIOpenNeuron.cl - kernels/MIOpenPReLU.cpp - kernels/MIOpenPooling.cl - kernels/MIOpenPoolingBwd.cl - kernels/MIOpenPoolingBwdND.cl - kernels/MIOpenPoolingForwardNaive.cl - kernels/MIOpenPoolingND.cl - kernels/MIOpenConv1x1S.cl - kernels/MIOpenConv1x1J1.cl - kernels/MIOpenConv1x1J1_stride.cl - kernels/MIOpenReduceCalculation.cpp - kernels/MIOpenReduceExtreme.cpp - kernels/MIOpenRoPE.cpp - kernels/MIOpenReduceSum.cpp - kernels/MIOpenSoftmax.cl - kernels/MIOpenSoftmaxAttn.cpp - kernels/MIOpenUtilKernels3.cl - kernels/MIOpenUtilKernels4.cl - kernels/MIOpenUtilKernels5.cl - kernels/MIOpenVecAdd.cpp - kernels/MIOpenVecAddOCL.cl - kernels/MIOpenIm2d2Col.cl - kernels/MIOpenIm3d2Col.cl - kernels/MIOpenCol2Im2d.cl - kernels/MIOpenCol2Im3d.cl - kernels/MIOpenConvBwdWrWS2.cl - kernels/MIOpenGroupConvBwdWrWS2.cl - kernels/MIOpenConvBwdWrW_LxG_P53.cl - kernels/MIOpenGroupConvBwdWrW_LxG_P53.cl - kernels/MIOpenConvBwdWrW_LxG_5x5.cl - kernels/MIOpenConvBwdWrW1x1_PAD_read4.cl - kernels/MIOpenConvFwd_LxL_11.cl - kernels/MIOpenConvFFT.cl - kernels/MIOpenRNNHiddenStateUpdate.cl - kernels/bugzilla_34765_detect.s - kernels/dummy_kernel.s - kernels/conv3x3.s - kernels/conv1x1u.s - kernels/conv1x1u_stride2.s - kernels/conv1x1u_bias_activ.s - kernels/conv3x3wrw.s - kernels/conv1x1wrw.s - kernels/conv5x10u2v2f1.s - kernels/conv5x10u2v2b1.s - kernels/conv7x7c3h224w224k64u2v2p3q3f1.s - kernels/xform_out.s - kernels/gcnAsmBNBwdTrainSpatial.s - kernels/MIOpenTensorKernels.cl - kernels/MIOpenTensorKernelsHip.cpp - kernels/MIOpenSubTensorOpWithScalarKernel.cl - kernels/MIOpenSubTensorOpWithSubTensorKernel.cl - kernels/MIOpenSubTensorOpWithCastTensorKernel.cl - kernels/MIOpenSubTensorOpWithTransformKernel.cl - kernels/Conv_Winograd_v13_3_12_fp16dot_stride1.s - kernels/Conv_Winograd_v13_3_12_fp16dot_stride2_dec.s - kernels/Conv_Winograd_v13_3_12_fp16dot_stride2_dil.s - kernels/Conv_Winograd_v14_3_3_fp16dot_stride1.s - kernels/Conv_Winograd_v14_3_3_fp16dot_stride2_dec.s - kernels/Conv_Winograd_v14_3_3_fp16dot_stride2_dil.s - kernels/Conv_Winograd_v16_5_0_stride1.s - kernels/conv_3x3_wheel_alpha_v9_0_15_stride_2_dil.s - kernels/conv_3x3_wheel_alpha_v9_0_15_stride_2_dec.s - kernels/conv_3x3_wheel_alpha_v9_0_15.s - kernels/conv_3x3_wheel_alpha_v7_0_3b.s - kernels/conv_3x3_wheel_alpha_v3_0b.s - kernels/conv_3x3_wheel_alpha_v9_2_7.s - kernels/conv_3x3_wheel_alpha_v9_2_7_stride_2_dec.s - kernels/Conv_Winograd_v21_1_3_fp16_dot2_f2x3_dilation2.s - kernels/Conv_Winograd_v21_1_3_fp16_dot2_f2x3_stride1.s - kernels/Conv_Winograd_v21_1_3_fp16_dot2_f2x3_stride2.s - kernels/Conv_Winograd_v21_1_3_fp16_dot2_f3x2_stride1.s - kernels/Conv_Winograd_v21_1_3_fp32_f2x3_dilation2.s - kernels/Conv_Winograd_v21_1_3_fp32_f2x3_stride1.s - kernels/Conv_Winograd_v21_1_3_fp32_f2x3_stride2.s - kernels/Conv_Winograd_v21_1_3_fp32_f3x2_stride1.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f2x3_dilation2.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f2x3_stride1.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f2x3_stride2.s - kernels/Conv_Winograd_v30_3_1_fp32_f2x3_dilation2.s - kernels/Conv_Winograd_v30_3_1_fp32_f2x3_stride1.s - kernels/Conv_Winograd_v30_3_1_fp32_f2x3_stride2.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f3x2_dilation2.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f3x2_stride1.s - kernels/Conv_Winograd_v30_3_1_fp16_dot2_f3x2_stride2.s - kernels/Conv_Winograd_v30_3_1_fp32_f3x2_dilation2.s - kernels/Conv_Winograd_v30_3_1_fp32_f3x2_stride1.s - kernels/Conv_Winograd_v30_3_1_fp32_f3x2_stride2.s - kernels/MIOpenConvBwdBias.cl - kernels/MIOpenBatchNormActivInfer.cl - kernels/MIOpenBatchNormActivInferHIP.cpp - kernels/MIOpenCTCLoss.cl - kernels/MIOpenDropoutHIP.cpp - kernels/winograd/Conv_Winograd_Fury_v2_4_1_fp16_fp16acc_f2x3_c16_stride1.s - kernels/winograd/Conv_Winograd_Fury_v2_4_1_fp16_fp16acc_f2x3_c32_stride1.s - kernels/xform_data.s - kernels/xform_filter.s - kernels/xform_bidirect_winograd_data.s - kernels/xform_bidirect_winograd_filter.s - kernels/xform_bidirect_winograd_out.s - kernels/UniversalTranspose.cl) - - # Kernels in development lists. - # Should be ALWAYS empty in develop branch (at the time of PR merge) - # Intention: to speed up kernel development rebuild time - set(MIOPEN_DEVELOPMENT_KERNELS) - - # Only referenced by MIOPEN_DEVELOPMENT_KERNELS - set(MIOPEN_DEVELOPMENT_KERNEL_INCLUDES) - - LIST(LENGTH MIOPEN_DEVELOPMENT_KERNELS MIOPEN_DEVELOPMENT_KERNELS_COUNT) - LIST(LENGTH MIOPEN_DEVELOPMENT_KERNEL_INCLUDES MIOPEN_DEVELOPMENT_KERNEL_INCLUDES_COUNT) - - add_kernels("kernel.cpp" "MIOPEN_KERNEL_" "" "${MIOPEN_KERNELS}") - add_kernels("kernel_includes.cpp" "MIOPEN_KERNEL_" "_INCLUDE" "${MIOPEN_KERNEL_INCLUDES}") - - if(${MIOPEN_DEVELOPMENT_KERNELS_COUNT}) - add_kernels("kernel.cpp" "MIOPEN_KERNEL_" "" "${MIOPEN_DEVELOPMENT_KERNELS}") - endif() - - if(${MIOPEN_DEVELOPMENT_KERNEL_INCLUDES_COUNT}) - add_kernels("kernel_includes.cpp" "MIOPEN_KERNEL_" "_INCLUDE" "${MIOPEN_DEVELOPMENT_KERNEL_INCLUDES}") - endif() - - configure_file(db_path.cpp.in ${PROJECT_BINARY_DIR}/db_path.cpp) - list(APPEND MIOpen_Source - activ.cpp - adam.cpp - addlayernorm.cpp - cat.cpp - groupnorm.cpp - getitem.cpp - glu.cpp - kernel_cache.cpp - layernorm.cpp - lrn.cpp - mlo_dir_conv.cpp - exec_utils.cpp - ocl/activ_ocl.cpp - ocl/batchnormocl.cpp - ocl/convolutionocl.cpp - ocl/lrn_ocl.cpp - ocl/mloNorm.cpp - ocl/pooling_ocl.cpp - ocl/tensorocl.cpp - ocl/rnnocl.cpp - ocl/utilocl.cpp - ocl/ctcocl.cpp - ocl/dropoutocl.cpp - ocl/gcn_asm_utils.cpp - ocl/rnn_util_ocl.cpp - hip/hip_build_utils.cpp - hip/batched_transpose_sol.cpp - hip/general_tensor_reorder_sol.cpp - pooling.cpp - t5layernorm.cpp - ocl/fusionopconvocl.cpp - ocl/fusionopbiasbnactivocl.cpp - prelu.cpp - reducecalculation.cpp - reduceextreme.cpp - rope.cpp - transformers_adam_w.cpp - ${PROJECT_BINARY_DIR}/db_path.cpp - ) - - list(INSERT MIOpen_Source 0 - ${PROJECT_BINARY_DIR}/kernel.cpp - ${PROJECT_BINARY_DIR}/kernel_includes.cpp - ) -endif() - -if(MIOPEN_USE_ROCBLAS OR MIOPEN_USE_HIPBLASLT) - list(APPEND MIOpen_Source - gemm_v2.cpp - ) -endif() - -if( MIOPEN_BACKEND STREQUAL "OpenCL" ) - list(APPEND MIOpen_Source - ocl/handleocl.cpp - ocl_kernel.cpp - ocl/oclerrors.cpp - ocl/clhelper.cpp - ) -endif() - -if( MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP") - list(APPEND MIOpen_Source - hip/hiperrors.cpp - hip/handlehip.cpp - hipoc/hipoc_kernel.cpp - hipoc/hipoc_program.cpp - ) -endif() - -if( MIOPEN_BACKEND STREQUAL "HIPNOGPU") - list(APPEND MIOpen_Source - hip/hiperrors.cpp - nogpu/handle.cpp - hipoc/hipoc_kernel.cpp - hipoc/hipoc_program.cpp - ) -endif() - -if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") - set(KERNELS_SRC_BATCH_FACTOR 50 CACHE STRING "Amount of kernel source files to inline to a single object file.") - set(KERNELS_BATCH_ID 0) - - function(inline_kernels_src BATCH_FACTOR KERNELS KERNEL_INCLUDES EXTRA_OPTIONS MESSAGE_SUFFIX) - set(KERNELS_BATCH) - set(KERNELS_BATCH_SIZE 0) - set(PROCESSED 0) - list(LENGTH KERNELS KERNELS_NUMBER) - - foreach(KERNEL ${KERNELS}) - list(APPEND KERNELS_BATCH ${KERNEL}) - list(LENGTH KERNELS_BATCH KERNELS_BATCH_SIZE) - math(EXPR PROCESSED "1+${PROCESSED}") - if((KERNELS_BATCH_SIZE EQUAL ${BATCH_FACTOR}) OR (PROCESSED EQUAL KERNELS_NUMBER)) - set(KERNEL_SRC_HPP_FILENAME batch_${KERNELS_BATCH_ID}.cpp.hpp) - set(KERNEL_SRC_HPP_PATH ${PROJECT_BINARY_DIR}/inlined_kernels/${KERNEL_SRC_HPP_FILENAME}) - set(KERNEL_SRC_CPP_PATH ${PROJECT_BINARY_DIR}/inlined_kernels/batch_${KERNELS_BATCH_ID}.cpp) - - add_custom_command( - OUTPUT ${KERNEL_SRC_HPP_PATH} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS addkernels ${KERNELS_BATCH} ${KERNEL_INCLUDES} - COMMAND $ -target ${KERNEL_SRC_HPP_PATH} -extern ${EXTRA_OPTIONS} -source ${KERNELS_BATCH} - COMMENT "Inlining kernels batch #${KERNELS_BATCH_ID}${MESSAGE_SUFFIX}" - ) - configure_file(kernels/kernels_batch.cpp.in ${KERNEL_SRC_CPP_PATH}) - list(APPEND MIOpen_Source ${KERNEL_SRC_CPP_PATH} ${KERNEL_SRC_HPP_PATH}) - - set(KERNELS_BATCH) - math(EXPR KERNELS_BATCH_ID "1+${KERNELS_BATCH_ID}") - endif() - endforeach() - - set(KERNELS_BATCH_ID ${KERNELS_BATCH_ID} PARENT_SCOPE) - set(MIOpen_Source ${MIOpen_Source} PARENT_SCOPE) - endfunction() - - inline_kernels_src(${KERNELS_SRC_BATCH_FACTOR} "${MIOPEN_KERNELS}" "${MIOPEN_KERNEL_INCLUDES}" "" "") - inline_kernels_src(${KERNELS_SRC_BATCH_FACTOR} "${MIOPEN_KERNEL_INCLUDES}" "" "-no-recurse;-mark-includes" " (includes)") - - set(MIOPEN_DEVELOPMENT_KERNELS_DEPS ${MIOPEN_KERNEL_INCLUDES}) - list(APPEND MIOPEN_DEVELOPMENT_KERNELS_DEPS ${MIOPEN_DEVELOPMENT_KERNEL_INCLUDES}) - - if(${MIOPEN_DEVELOPMENT_KERNELS_COUNT}) - inline_kernels_src(${KERNELS_SRC_BATCH_FACTOR} "${MIOPEN_DEVELOPMENT_KERNELS}" "${MIOPEN_DEVELOPMENT_KERNELS_DEPS}" "" " (dev kernels)") - endif() - - if(${MIOPEN_DEVELOPMENT_KERNEL_INCLUDES_COUNT}) - inline_kernels_src(${KERNELS_SRC_BATCH_FACTOR} "${MIOPEN_DEVELOPMENT_KERNEL_INCLUDES}" "" "-no-recurse;-mark-includes" " (dev includes)") - endif() - -endif() - -if(MIOPEN_USE_COMGR) - list(APPEND MIOpen_Source comgr.cpp) -endif() - -if(MIOPEN_USE_MLIR) - list(APPEND MIOpen_Source - conv/invokers/mlir_impl_gemm.cpp - mlir_build.cpp - solver/conv/mlir_common.cpp - ) -endif() - -# build library -if(MIOPEN_ENABLE_SQLITE) - add_library( MIOpen - ${MIOpen_Source} - $ - ) -else() - add_library( MIOpen - ${MIOpen_Source} - ) -endif() - -rocm_set_soversion(MIOpen ${MIOpen_SOVERSION}) - -clang_tidy_check(MIOpen) - -if(HAS_LIB_STD_FILESYSTEM) - target_link_libraries(MIOpen PRIVATE stdc++fs) -endif() - -find_package(zstd) -if(zstd_FOUND) - target_link_libraries(MIOpen PRIVATE $,zstd::libzstd_shared,zstd::libzstd_static>) -endif() - -function(target_internal_library TARGET) - target_link_libraries(${TARGET} PRIVATE ${ARGN}) - target_link_libraries(${TARGET} INTERFACE $) -endfunction() - -target_include_directories(MIOpen PUBLIC - $ -) - -if(MIOPEN_USE_COMPOSABLEKERNEL) -set(MIOPEN_CK_LINK_FLAGS composable_kernel::device_other_operations composable_kernel::device_gemm_operations composable_kernel::device_conv_operations composable_kernel::device_reduction_operations hip::host) -endif() - -if(WIN32) - # Refer to https://en.cppreference.com/w/cpp/language/types for details. - target_compile_options(MIOpen PRIVATE $:-U__LP64__>>) -endif() - -target_include_directories(MIOpen SYSTEM PUBLIC $) -# Workaround : change in rocm-cmake was causing linking error so had to add ${CMAKE_DL_LIBS} -# We can remove ${CMAKE_DL_LIBS} once root cause is identified. -target_link_libraries(MIOpen PRIVATE ${CMAKE_DL_LIBS} Threads::Threads BZip2::BZip2 ${MIOPEN_CK_LINK_FLAGS}) -miopen_generate_export_header(MIOpen) - -if(WIN32) - # Temporary workaround on rocMLIR not exporting correctly libraries it depends on. - target_link_libraries(MIOpen PRIVATE ntdll) -endif() - -if(BUILD_TESTING) - # On Windows, export selected internal symbols only when tests are built. The officially released - # binaries must not have internals exposed because doing so violates the threats model requirements. - # We cannot use the CMake property CMAKE_EXPORT_ALL_SYMBOLS here because the number of automatically - # exported symbols exceeds the maximum allowed number in a DLL library (64K). - # See details here: https://learn.microsoft.com/en-us/cpp/build/exporting-from-a-dll?view=msvc-170 - generate_export_header(MIOpen BASE_NAME MIOPEN_INTERNALS EXPORT_FILE_NAME ${CMAKE_BINARY_DIR}/include/miopen/export_internals.h) - target_compile_definitions(MIOpen PUBLIC $) -endif() - -if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) - target_link_libraries(MIOpen PRIVATE frugally-deep::fdeep Eigen3::Eigen) - if(NOT TARGET nlohmann_json) - # frugally-deep has broken linking to nlohmann_json - add_library(nlohmann_json INTERFACE IMPORTED GLOBAL) - target_link_libraries(nlohmann_json INTERFACE nlohmann_json::nlohmann_json) - endif() - file(GLOB MODEL_FILES CONFIGURE_DEPENDS kernels/*.model) - if(NOT ENABLE_ASAN_PACKAGING ) - install(FILES ${MODEL_FILES} DESTINATION ${DATABASE_INSTALL_DIR}) - endif() - foreach(MODEL_FILE ${MODEL_FILES}) - get_filename_component(MODEL_FILE_FILENAME "${MODEL_FILE}" NAME) - configure_file("${MODEL_FILE}" "${PROJECT_BINARY_DIR}/${DATABASE_INSTALL_DIR}/${MODEL_FILE_FILENAME}" COPYONLY) - endforeach() -endif() - -############################################################ -# MIOpen depends on OpenCL -if( MIOPEN_BACKEND STREQUAL "OpenCL") - MESSAGE( STATUS "MIOpen linking OpenCL: ${OPENCL_INCLUDE_DIRS}" ) - target_include_directories(MIOpen SYSTEM PUBLIC ${OPENCL_INCLUDE_DIRS} ) - target_link_libraries( MIOpen PUBLIC ${OPENCL_LIBRARIES} ) - list(APPEND PACKAGE_DEPENDS PACKAGE OpenCL) -elseif(MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIP") - target_link_libraries( MIOpen PRIVATE hip::device ) - target_link_libraries( MIOpen INTERFACE hip::host ) - if(MIOPEN_USE_HIPRTC) - if(WIN32) - target_link_libraries( MIOpen PRIVATE hiprtc::hiprtc ) - else() - target_link_libraries( MIOpen PRIVATE hiprtc) - endif() - endif() - if(ENABLE_HIP_WORKAROUNDS) - # Workaround hip not setting its usage requirements correctly - target_compile_definitions( MIOpen PRIVATE -D__HIP_PLATFORM_AMD__=1 ) - endif() - # This is helpful for the tests - target_link_libraries( MIOpen INTERFACE $ ) - list(APPEND PACKAGE_DEPENDS PACKAGE hip) -endif() - -if(MIOPEN_USE_COMGR) - list(APPEND PACKAGE_DEPENDS PACKAGE amd_comgr) - target_internal_library(MIOpen amd_comgr) -endif() - -if(MIOPEN_OFFLINE_COMPILER_PATHS_V2) - # Adding rocm-core library dependency for API getROCmInstallPath() - target_link_libraries(MIOpen PRIVATE rocm-core) -endif() - -if(rocblas_FOUND) - target_link_libraries( MIOpen INTERFACE $ ) - target_link_libraries( MIOpen PRIVATE roc::rocblas ) - list(APPEND PACKAGE_STATIC_DEPENDS PACKAGE rocblas) -endif() - -if(hipblaslt_FOUND) - target_link_libraries( MIOpen PRIVATE roc::hipblaslt ) -endif() - -# For backward compatibility with ROCm 5.3 -# Build with library libMLIRMIOpen -if(LIBMLIRMIOPEN) - target_link_libraries(MIOpen PRIVATE ${LIBMLIRMIOPEN}) -endif() - -# Build with package rocMLIR -if(rocMLIR_FOUND) - target_link_libraries(MIOpen PRIVATE rocMLIR::rockCompiler) -endif() - -target_link_libraries(MIOpen PRIVATE nlohmann_json::nlohmann_json) - -target_internal_library(MIOpen - Boost::filesystem -) -list(APPEND PACKAGE_STATIC_DEPENDS PACKAGE Boost COMPONENTS filesystem) -if(NOT WIN32 AND NOT APPLE) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/lib.def " -MIOPEN_${MIOPEN_BACKEND}_1 -{ -global: - miopen*; - extern \"C++\" { - miopen::*; - }; -local: - *boost*; - extern \"C++\" { - std::*; - }; -}; -") - target_link_libraries(MIOpen PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_BINARY_DIR}/lib.def") - target_link_libraries(MIOpen PRIVATE "-Wl,--exclude-libs,ALL") - # set_target_properties(MIOpen PROPERTIES CXX_VISIBILITY_PRESET hidden) - set_target_properties(MIOpen PROPERTIES VISIBILITY_INLINES_HIDDEN 1) -endif() -####################################### -if(MIOPEN_ENABLE_SQLITE) - # MIOpen depends on SQLite - target_link_libraries(MIOpen PRIVATE SQLite::SQLite3) -endif() -############################################################ -# MIOpen depends on librt for Boost.Interprocess -if(NOT WIN32 AND NOT APPLE) - find_library(LIBRT rt) - if(LIBRT) - message(STATUS "Librt: " ${LIBRT}) - target_internal_library(MIOpen ${LIBRT}) - endif() -endif() - -if(MIOPEN_USE_ROCTRACER) - target_link_libraries(MIOpen PRIVATE roctx64) -endif() - -############################################################ -# Installation -set(MIOPEN_CXX_HEADER_PATH) -if(MIOPEN_INSTALL_CXX_HEADERS) -set(MIOPEN_CXX_HEADER_PATH ${PROJECT_SOURCE_DIR}/src/include) -endif() - -rocm_install_targets( - TARGETS MIOpen - INCLUDE - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_BINARY_DIR}/include - ${MIOPEN_CXX_HEADER_PATH} -) - -rocm_export_targets( - TARGETS MIOpen - DEPENDS - ${PACKAGE_DEPENDS} - STATIC_DEPENDS - ${PACKAGE_STATIC_DEPENDS} -) - -# Install db files -if(NOT MIOPEN_EMBED_DB STREQUAL "") - include(embed) - if(MIOPEN_EMBED_BINCACHE AND MIOPEN_BINCACHE_PATH STREQUAL "") - if(MIOPEN_NO_LFS_PULLED) - message(WARNING "Binary cache files have not been pulled down from git-lfs, will not embed.") - else() - set(MIOPEN_BINCACHE_PATH ${KERNELS_BINARY_DIR}) - message("MIOPEN_BINCACHE_PATH: ${MIOPEN_BINCACHE_PATH}") - endif() - else() - message(WARNING "MIOPEN_EMBED_BINCACHE is set and MIOPEN_BINCACHE_PATH was used to override default binary cache files. Proceed at your own risk!") - endif() -# embed find db - foreach(EMBED_ARCH ${MIOPEN_EMBED_DB}) - message(STATUS "Adding find db for arch: ${EMBED_ARCH}") - list(APPEND CODE_OBJECTS "${PROJECT_BINARY_DIR}/${DATABASE_INSTALL_DIR}/${EMBED_ARCH}.${MIOPEN_BACKEND}.fdb.txt") - message(STATUS "Adding perf db for arch: ${EMBED_ARCH}") - list(APPEND CODE_OBJECTS "${PROJECT_BINARY_DIR}/${DATABASE_INSTALL_DIR}/${EMBED_ARCH}.db${PERFDB_SUFFIX}") - endforeach() -# Embed Bin Cache - if(NOT MIOPEN_BINCACHE_PATH STREQUAL "") - foreach(EMBED_ARCH ${MIOPEN_EMBED_DB}) - message(STATUS "Adding binary cache for arch: ${EMBED_ARCH}") - download_binary(OUTPUT_PATH "${MIOPEN_BINCACHE_PATH}" "${EMBED_ARCH}") - list(APPEND CODE_OBJECTS "${OUTPUT_PATH}") - endforeach() - endif() - add_embed_library(miopen_data ${CODE_OBJECTS}) - target_link_libraries(MIOpen PRIVATE $ ) -endif() - -if(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY) - #Generating Wrapper of each headers for backward compatibility - rocm_wrap_header_file( - export.h config.h version.h miopen.h - HEADER_LOCATION include/miopen - GUARDS WRAPPER - WRAPPER_LOCATIONS miopen/${CMAKE_INSTALL_INCLUDEDIR}/miopen - ORIGINAL_FILES ${PROJECT_BINARY_DIR}/include/miopen/version.h ${PROJECT_BINARY_DIR}/include/miopen/config.h - ) - - #Installing Wrapper Headers - rocm_install( - DIRECTORY - "${PROJECT_BINARY_DIR}/miopen/include" - DESTINATION "miopen" ) - message( STATUS "Backward Compatible Sym Link Created for include directories" ) -endif() diff --git a/src/api/find2_0_commons.cpp b/src/api/find2_0_commons.cpp index 0c0e23898e..e69de29bb2 100644 --- a/src/api/find2_0_commons.cpp +++ b/src/api/find2_0_commons.cpp @@ -1,532 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -template -static miopenStatus_t MakeProblem(miopenProblem_t* problem, - OperationDescriptor operatorDesc, - miopenProblemDirection_t direction) -{ - return miopen::try_([&] { - auto& in_problem_deref = miopen::deref(problem); - in_problem_deref = new miopen::ProblemContainer(); - auto& container_deref = miopen::deref(*problem); - - container_deref.item = miopen::Problem(); - auto& problem_deref = std::get(container_deref.item); - auto& operator_deref = miopen::deref(operatorDesc); - - problem_deref.SetOperatorDescriptor(operator_deref); - problem_deref.SetDirection(direction); - }); -} - -extern "C" { -miopenStatus_t miopenCreateConvProblem(miopenProblem_t* problem, - miopenConvolutionDescriptor_t operatorDesc, - miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, operatorDesc, direction); - return MakeProblem(problem, operatorDesc, direction); -} - -miopenStatus_t miopenCreateActivationProblem(miopenProblem_t* problem, - miopenActivationDescriptor_t operatorDesc, - miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, operatorDesc, direction); - return MakeProblem(problem, operatorDesc, direction); -} - -miopenStatus_t miopenCreateBiasProblem(miopenProblem_t* problem, miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, direction); - - return miopen::try_([&] { - auto& container_ptr = miopen::deref(problem); - container_ptr = new miopen::ProblemContainer(); - auto& container_deref = miopen::deref(*problem); - - container_deref.item = miopen::Problem(); - auto& problem_deref = std::get(container_deref.item); - - problem_deref.SetOperatorDescriptor(miopen::BiasDescriptor{}); - problem_deref.SetDirection(direction); - }); -} - -miopenStatus_t miopenCreateMhaProblem(miopenProblem_t* problem, - miopenMhaDescriptor_t operatorDesc, - miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, direction); - return MakeProblem(problem, operatorDesc, direction); -} - -miopenStatus_t miopenCreateSoftmaxProblem(miopenProblem_t* problem, - miopenSoftmaxDescriptor_t operatorDesc, - miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, direction); - return MakeProblem(problem, operatorDesc, direction); -} - -miopenStatus_t miopenCreateBatchnormProblem(miopenProblem_t* problem, - miopenBatchNormMode_t mode, - bool runningMeanVariance, - miopenProblemDirection_t direction) -{ - MIOPEN_LOG_FUNCTION(problem, mode, direction); - - return miopen::try_([&] { - auto& container_ptr = miopen::deref(problem); - container_ptr = new miopen::ProblemContainer(); - auto& container_deref = miopen::deref(*problem); - - container_deref.item = miopen::Problem(); - auto& problem_deref = std::get(container_deref.item); - - problem_deref.SetOperatorDescriptor(miopen::BatchnormDescriptor{mode, runningMeanVariance}); - problem_deref.SetDirection(direction); - }); -} - -miopenStatus_t miopenFuseProblems(miopenProblem_t problem1, miopenProblem_t problem2) -{ - MIOPEN_LOG_FUNCTION(problem1, problem2); - return miopen::try_([&] { - auto& problem1_deref = miopen::deref(problem1); - - auto emplace_problem2 = [problem2](auto& problems) { - const auto impl2 = boost::hof::match( - [&](miopen::Problem& problem2_inner) { problems.emplace_back(problem2_inner); }, - [&](const miopen::FusedProblem& problem2_inner) { - problems.reserve(problems.size() + problem2_inner.problems.size()); - std::copy(problem2_inner.problems.begin(), - problem2_inner.problems.end(), - std::back_inserter(problems)); - }); - - std::visit(impl2, miopen::deref(problem2).item); - }; - - std::visit(boost::hof::match( - [&](miopen::Problem& problem1_inner) { - auto tmp = miopen::FusedProblem{}; - tmp.problems.reserve(2); - tmp.problems.emplace_back(problem1_inner); - emplace_problem2(tmp.problems); - problem1_deref.item = std::move(tmp); - }, - [&](miopen::FusedProblem& problem1_inner) { - emplace_problem2(problem1_inner.problems); - }), - miopen::deref(problem1).item); - - std::get(miopen::deref(problem1).item).PropagateDescriptors(); - }); -} - -miopenStatus_t miopenDestroyProblem(miopenProblem_t problem) -{ - MIOPEN_LOG_FUNCTION(problem); - return miopen::try_([&] { miopen_destroy_object(problem); }); -} - -miopenStatus_t miopenSetProblemTensorDescriptor(miopenProblem_t problem, - miopenTensorArgumentId_t id, - const miopenTensorDescriptor_t descriptor) -{ - MIOPEN_LOG_FUNCTION(problem, id, descriptor); - - return miopen::try_([&] { - const auto impl = boost::hof::match( - [&](miopen::Problem& problem) { - problem.RegisterTensorDescriptor(id, miopen::deref(descriptor)); - }, - [&](const miopen::FusedProblem&) { - MIOPEN_THROW(miopenStatusBadParm, - "Attempt to set tensor descriptor of a fused problem"); - }); - - std::visit(impl, miopen::deref(problem).item); - }); -} - -miopenStatus_t miopenCreateFindOptions(miopenFindOptions_t* options) -{ - MIOPEN_LOG_FUNCTION(options); - return miopen::try_([&] { - auto& options_ptr = miopen::deref(options); - options_ptr = new miopen::FindOptions(); - }); -} - -miopenStatus_t miopenDestroyFindOptions(miopenFindOptions_t options) -{ - MIOPEN_LOG_FUNCTION(options); - return miopen::try_([&] { miopen_destroy_object(options); }); -} - -miopenStatus_t miopenSetFindOptionTuning(miopenFindOptions_t options, int value) -{ - MIOPEN_LOG_FUNCTION(options, value); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.exhaustive_search = value != 0; - }); -} - -miopenStatus_t miopenSetFindOptionResultsOrder(miopenFindOptions_t options, - miopenFindResultsOrder_t value) -{ - MIOPEN_LOG_FUNCTION(options, value); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.results_order = value; - }); -} - -miopenStatus_t miopenSetFindOptionWorkspaceLimit(miopenFindOptions_t options, size_t value) -{ - MIOPEN_LOG_FUNCTION(options, value); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.workspace_limit = value; - }); -} - -miopenStatus_t -miopenSetFindOptionPreallocatedWorkspace(miopenFindOptions_t options, void* buffer, size_t size) -{ - MIOPEN_LOG_FUNCTION(options, buffer, size); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.preallocated_workspace = {DataCast(buffer), size}; - }); -} - -miopenStatus_t miopenSetFindOptionPreallocatedTensor(miopenFindOptions_t options, - miopenTensorArgumentId_t id, - void* buffer) -{ - MIOPEN_LOG_FUNCTION(options, id, buffer); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.preallocated_tensors.emplace(id, DataCast(buffer)); - }); -} - -miopenStatus_t miopenSetFindOptionAttachBinaries(miopenFindOptions_t options, unsigned attach) -{ - MIOPEN_LOG_FUNCTION(options, attach); - - return miopen::try_([&] { - auto& options_deref = miopen::deref(options); - options_deref.attach_binaries = (attach == 1); - }); -} - -miopenStatus_t miopenFindSolutions(miopenHandle_t handle, - miopenProblem_t problem, - miopenFindOptions_t options, - miopenSolution_t* solutions, - size_t* numSolutions, - size_t maxSolutions) -{ - MIOPEN_LOG_FUNCTION(handle, problem, options, solutions, numSolutions, maxSolutions); - - return miopen::try_([&] { - auto& handle_deref = miopen::deref(handle); - const auto& problem_deref = miopen::deref(problem).item; - - std::visit([](auto&& problem) { problem.LogDriverCommand(); }, problem_deref); - - const auto& options_deref = - options == nullptr ? miopen::FindOptions{} : miopen::deref(options); - - auto solutions_deref = std::visit( - [&](auto&& problem) { - return problem.FindSolutions(handle_deref, options_deref, maxSolutions); - }, - problem_deref); - - for(auto i = 0; i < solutions_deref.size(); ++i) - { - auto& theSolution = miopen::deref(solutions + i); - theSolution = new miopen::Solution{std::move(solutions_deref[i])}; - } - - if(numSolutions != nullptr) - *numSolutions = solutions_deref.size(); - }); -} - -inline std::ostream& operator<<(std::ostream& stream, const miopenTensorArgument_t& tensor) -{ - switch(tensor.id) - { - case miopenTensorConvolutionW: stream << "ConvW"; break; - case miopenTensorConvolutionX: stream << "ConvX"; break; - case miopenTensorConvolutionY: stream << "ConvY"; break; - case miopenTensorActivationX: stream << "ActivX"; break; - case miopenTensorActivationDX: stream << "ActivDX"; break; - case miopenTensorActivationY: stream << "ActivY"; break; - case miopenTensorActivationDY: stream << "ActivDY"; break; - case miopenTensorBias: stream << "Bias"; break; - case miopenTensorBiasX: stream << "BiasX"; break; - case miopenTensorBiasY: stream << "BiasY"; break; - case miopenTensorMhaK: stream << "MhaK"; break; - case miopenTensorMhaQ: stream << "MhaQ"; break; - case miopenTensorMhaV: stream << "MhaV"; break; - case miopenTensorMhaDescaleK: stream << "MhaDescaleK"; break; - case miopenTensorMhaDescaleQ: stream << "DescaleQ"; break; - case miopenTensorMhaDescaleV: stream << "DescaleV"; break; - case miopenTensorMhaDescaleS: stream << "MhaDescaleS"; break; - case miopenTensorMhaScaleS: stream << "MhaScaleS"; break; - case miopenTensorMhaScaleO: stream << "MhaScaleO"; break; - case miopenTensorMhaDropoutProbability: stream << "MhaDropoutProbability"; break; - case miopenTensorMhaDropoutSeed: stream << "MhaDropoutSeed"; break; - case miopenTensorMhaDropoutOffset: stream << "MhaDropoutOffset"; break; - case miopenTensorMhaO: stream << "MhaO"; break; - case miopenTensorMhaAmaxO: stream << "MhaAmaxO"; break; - case miopenTensorMhaAmaxS: stream << "MhaAmaxS"; break; - case miopenTensorMhaM: stream << "MhaM"; break; - case miopenTensorMhaZInv: stream << "MhaZInv"; break; - case miopenTensorMhaDO: stream << "miopenTensorMhaDO"; break; - case miopenTensorMhaDescaleO: stream << "miopenTensorMhaDescaleO"; break; - case miopenTensorMhaDescaleDO: stream << "miopenTensorMhaDescaleDO"; break; - case miopenTensorMhaDescaleDS: stream << "miopenTensorMhaDescaleDS"; break; - case miopenTensorMhaScaleDS: stream << "miopenTensorMhaScaleDS"; break; - case miopenTensorMhaScaleDQ: stream << "miopenTensorMhaScaleDQ"; break; - case miopenTensorMhaScaleDK: stream << "miopenTensorMhaScaleDK"; break; - case miopenTensorMhaScaleDV: stream << "miopenTensorMhaScaleDV"; break; - case miopenTensorMhaDQ: stream << "miopenTensorMhaDQ"; break; - case miopenTensorMhaDK: stream << "miopenTensorMhaDK"; break; - case miopenTensorMhaDV: stream << "miopenTensorMhaDV"; break; - case miopenTensorMhaAmaxDQ: stream << "miopenTensorMhaAmaxDQ"; break; - case miopenTensorMhaAmaxDK: stream << "miopenTensorMhaAmaxDK"; break; - case miopenTensorMhaAmaxDV: stream << "miopenTensorMhaAmaxDV"; break; - case miopenTensorMhaAmaxDS: stream << "miopenTensorMhaAmaxDS"; break; - case miopenTensorMhaBias: stream << "miopenTensorMhaBias"; break; - case miopenTensorMhaMask: stream << "miopenTensorMhaMask"; break; - case miopenTensorSoftmaxX: stream << "SoftmaxX"; break; - case miopenTensorSoftmaxY: stream << "SoftmaxY"; break; - case miopenTensorSoftmaxDX: stream << "SoftmaxDX"; break; - case miopenTensorSoftmaxDY: stream << "SoftmaxDY"; break; - case miopenTensorArgumentIsScalar: stream << "ScalarArgument"; break; - case miopenTensorBatchnormX: stream << "miopenTensorBatchnormX"; break; - case miopenTensorBatchnormY: stream << "miopenTensorBatchnormY"; break; - case miopenTensorBatchnormRunningMean: stream << "miopenTensorBatchnormRunningMean"; break; - case miopenTensorBatchnormRunningVariance: - stream << "miopenTensorBatchnormRunningVariance"; - break; - case miopenTensorBatchnormSavedMean: stream << "miopenTensorBatchnormSavedMean"; break; - case miopenTensorBatchnormSavedVariance: stream << "miopenTensorBatchnormSavedVariance"; break; - case miopenTensorBatchnormScale: stream << "miopenTensorBatchnormScale"; break; - case miopenTensorBatchnormScaleDiff: stream << "miopenTensorBatchnormScaleDiff"; break; - case miopenTensorBatchnormEstimatedMean: stream << "miopenTensorBatchnormEstimatedMean"; break; - case miopenTensorBatchnormEstimatedVariance: - stream << "miopenTensorBatchnormEstimatedVariance"; - break; - case miopenTensorBatchnormBias: stream << "miopenTensorBatchnormBias"; break; - case miopenTensorBatchnormBiasDiff: stream << "miopenTensorBatchnormBiasDiff"; break; - case miopenTensorBatchnormDX: stream << "miopenTensorBatchnormDX"; break; - case miopenTensorBatchnormDY: stream << "miopenTensorBatchnormDY"; break; - case miopenScalarBatchnormEpsilon: stream << "miopenScalarBatchnormEpsilon"; break; - case miopenScalarBatchnormExpAvgFactor: stream << "miopenScalarBatchnormExpAvgFactor"; break; - case miopenTensorArgumentIdInvalid: stream << "Invalid"; break; - } - - stream << ": "; - if(tensor.descriptor != nullptr) - stream << miopen::deref(tensor.descriptor); - else - stream << "NULL"; - stream << " -> "; - stream << tensor.buffer; - stream << ","; - - return stream; -} - -miopenStatus_t miopenRunSolution(miopenHandle_t handle, - miopenSolution_t solution, - size_t nInputs, - const miopenTensorArgument_t* tensors, - void* workspace, - size_t workspaceSize) -{ - const auto tensors_vector = std::vector{tensors, tensors + nInputs}; - MIOPEN_LOG_FUNCTION(handle, solution, nInputs, tensors_vector, workspace, workspaceSize); - - return miopen::try_([&] { - auto& handle_deref = miopen::deref(handle); - auto& solution_deref = miopen::deref(solution); - - solution_deref.LogDriverCommand(); - - const auto inputs_deref = [&]() { - auto ret = std::unordered_map{}; - - ret.reserve(tensors_vector.size()); - for(auto&& tensor : tensors_vector) - ret.emplace(std::make_pair(tensor.id, miopen::Solution::RunInput{tensor})); - - return ret; - }(); - - solution_deref.Run(handle_deref, inputs_deref, DataCast(workspace), workspaceSize); - }); -} - -miopenStatus_t miopenDestroySolution(miopenSolution_t solution) -{ - MIOPEN_LOG_FUNCTION(solution); - return miopen::try_([&] { miopen_destroy_object(solution); }); -} - -miopenStatus_t miopenLoadSolution(miopenSolution_t* solution, const char* data, size_t size) -{ - MIOPEN_LOG_FUNCTION(solution, data, size); - - return miopen::try_([&] { - if(data == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Data parameter should not be a nullptr."); - - auto json = nlohmann::json::from_msgpack(data, data + size); - auto& solution_ptr_deref = miopen::deref(solution); - solution_ptr_deref = new miopen::Solution{json.get()}; - }); -} - -miopenStatus_t miopenSaveSolution(miopenSolution_t solution, char* data) -{ - MIOPEN_LOG_FUNCTION(solution, data); - - return miopen::try_([&] { - if(data == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Data parameter should not be a nullptr."); - - auto& solution_deref = miopen::deref(solution); - - if(solution_deref.serialization_cache.empty()) - { - const nlohmann::json json = solution_deref; - solution_deref.serialization_cache = nlohmann::json::to_msgpack(json); - } - - std::memcpy(data, - solution_deref.serialization_cache.data(), - solution_deref.serialization_cache.size()); - - solution_deref.serialization_cache = {}; - }); -} - -miopenStatus_t miopenGetSolutionSize(miopenSolution_t solution, size_t* size) -{ - MIOPEN_LOG_FUNCTION(solution); - - return miopen::try_([&] { - if(size == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Size parameter should not be a nullptr."); - - auto& solution_deref = miopen::deref(solution); - - if(solution_deref.serialization_cache.empty()) - { - const nlohmann::json json = solution_deref; - solution_deref.serialization_cache = nlohmann::json::to_msgpack(json); - } - - *size = solution_deref.serialization_cache.size(); - }); -} - -miopenStatus_t miopenGetSolutionWorkspaceSize(miopenSolution_t solution, size_t* workspaceSize) -{ - MIOPEN_LOG_FUNCTION(solution); - - return miopen::try_([&] { - const auto& solution_deref = miopen::deref(solution); - *workspaceSize = solution_deref.GetWorkspaceSize(); - }); -} - -miopenStatus_t miopenGetSolutionTime(miopenSolution_t solution, float* time) -{ - MIOPEN_LOG_FUNCTION(solution); - - return miopen::try_([&] { - const auto& solution_deref = miopen::deref(solution); - *time = solution_deref.GetTime(); - }); -} - -miopenStatus_t miopenGetSolutionSolverId(miopenSolution_t solution, uint64_t* solverId) -{ - MIOPEN_LOG_FUNCTION(solution); - - return miopen::try_([&] { - const auto& solution_deref = miopen::deref(solution); - *solverId = solution_deref.GetSolver().Value(); - }); -} - -miopenStatus_t miopenGetSolverIdConvAlgorithm(uint64_t solverId, miopenConvAlgorithm_t* result) -{ - MIOPEN_LOG_FUNCTION(solverId); - - return miopen::try_([&] { - const auto id_deref = miopen::solver::Id{solverId}; - - if(!id_deref.IsValid() || id_deref.GetPrimitive() != miopen::solver::Primitive::Convolution) - MIOPEN_THROW(miopenStatusInvalidValue); - - *result = id_deref.GetAlgo(); - }); -} -} diff --git a/src/conv/heuristics/ai_heuristics.cpp b/src/conv/heuristics/ai_heuristics.cpp index dc922bd2eb..e69de29bb2 100644 --- a/src/conv/heuristics/ai_heuristics.cpp +++ b/src/conv/heuristics/ai_heuristics.cpp @@ -1,784 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#if MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK || MIOPEN_ENABLE_AI_KERNEL_TUNING -#include -#include - -namespace miopen { -namespace ai { -namespace common { - -nlohmann::json LoadJSON(const fs::path& path) -{ - if(!fs::exists(path)) - MIOPEN_THROW(miopenStatusInternalError, "Unable to load file: " + path); - return nlohmann::json::parse(std::ifstream(path)); -} - -template -std::unordered_map ReverseMap(const std::unordered_map& map) -{ - std::unordered_map reversed_map = {}; - for(const auto& it : map) - reversed_map.emplace(make_pair(it.second, it.first)); - return reversed_map; -} - -template -std::vector LookupValues(const std::vector& keys, const std::unordered_map& map) -{ - std::vector values = {}; - values.reserve(keys.size()); - std::transform(keys.begin(), keys.end(), std::back_inserter(values), [&](const U& key) { - return map.at(key); - }); - return values; -} -} // namespace common - -#if MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK -namespace immed_mode { -Metadata::Metadata(const std::string& arch) - : json(common::LoadJSON(GetSystemDbPath() / (arch + "_metadata.tn.model"))), - direction_encodings(json["encodings"]["Direction"]), - precision_encodings(json["encodings"]["Precision"]), - layout_encodings(json["encodings"]["Layout"]), - features(json["conv_params_used_as_features"]), - num_inputs(json["num_inputs"]), - num_outputs(json["num_outputs"]), - num_solvers(json["num_solvers"]), - solver_map(common::ReverseMap(json["encodings"]["solver"])), - features_mean(common::LookupValues( - features, json["stats"]["overall"]["features"]["mean"])), - features_std(common::LookupValues( - features, json["stats"]["overall"]["features"]["std"])), - test_features_mean(common::LookupValues( - features, json["stats"]["test"]["features"]["mean"])), - test_features_std(common::LookupValues( - features, json["stats"]["test"]["features"]["std"])) -{ -} - -size_t Metadata::EncodeDirection(miopen::conv::Direction dir) const -{ - if(dir == conv::Direction::BackwardWeights) - return direction_encodings.at("W"); - else if(dir == conv::Direction::BackwardData) - return direction_encodings.at("B"); - else - return direction_encodings.at("F"); -} - -size_t Metadata::EncodePrecision(miopenDataType_t data_type) const -{ - if(data_type == miopenBFloat16) - return precision_encodings.at("BF16"); - else if(data_type == miopenHalf) - return precision_encodings.at("FP16"); - else if(data_type == miopenFloat) - return precision_encodings.at("FP32"); - MIOPEN_THROW("Unsupported data type passed to TunaNet"); -} - -size_t Metadata::EncodeLayout(const std::string& layout) const -{ - if(layout != "NCDHW" && layout != "NCHW") // TunaNet supports NCHW and NCDHW layouts only atm - MIOPEN_THROW("Unsupported layout passed to TunaNet"); - return layout_encodings.at(layout); -} - -/** `Model` encapuslates the machinery required to run inference on a TunaNet model - * - * The `Model` class encapuslates all the machinery needed to run inference on a - * TunaNet model, including loading the TunaNet model, formatting a problem so that it - * can be fed into TunaNet for inference, and getting TunaNet's predictions etc. - * - * @param arch Architecture - */ -class Model -{ -public: - Metadata metadata; - Model(const std::string& arch) - : metadata(Metadata(arch)), - model(fdeep::load_model(ModelPath(arch), true, fdeep::dev_null_logger)), - input_shape(fdeep::tensor_shape(metadata.num_inputs)), - offset(metadata.num_outputs - metadata.num_solvers) - { - } - virtual ~Model() = default; - /** Is given problem supported by TunaNet? - * - * A TunaNet model can only work with problems "similar" to the problems it was trained on. - * Since our training data has changed over time, a TunaNet model trained for an earlier - * GPU might not support the same set of problems as a TunaNet model trained for a later - * GPU might. Thus, each subclass of `Model`, specializing `Model` to a specific GPU, must - * implement its own `IsProblemSupported` function. - * - * @param problem Problem - * @param ctx Execution context - */ - virtual bool IsProblemSupported(const conv::ProblemDescription& problem, - const ExecutionContext& ctx) const = 0; - /** Forward (i.e., run inference on) problem through TunaNet - * - * This function takes in a problem, converts it to a numeric vector and feeds it TunaNet - * for inference. Its output is a numeric vector that represents a probability distribution. - * Each index in this vector represents a solver (as given in metadata.solver_map) and the - * value at each index represents the probability that that solver is the fastest for given - * convolution problem. - * - * @param problem Problem - */ - std::vector Forward(const conv::ProblemDescription& problem) const - { - std::vector features = ToFeatures(problem); - std::vector output = model.predict({fdeep::tensor(input_shape, features)}); - std::vector output_vector = output.front().to_vector(); - std::vector res(output_vector.begin() + offset, output_vector.end()); - return res; - } - -protected: - const fdeep::model model; // TunaNet model - const fdeep::tensor_shape input_shape; // Shape of input tensor required by TunaNet - const size_t offset; // Some TunaNet models output some "fluff" before they output kernel - // probabilites. This offset tells how many indexes of fluff need to - // be skipped in order to get to kernel probabilities. - /** Path to model file for given GPU - * - * The model files for each GPU are identified by the GPU architecture. This function takes - * in a GPU architecture and returns the path to its TunaNet model. - * - * @param arch Architecture - */ - static std::string ModelPath(const std::string& arch) - { - const auto file_path = GetSystemDbPath() / (arch + ".tn.model"); - if(!fs::exists(file_path)) - MIOPEN_THROW(miopenStatusInternalError, "Unable to load AI model file:" + file_path); - return file_path.string(); - } - /** Convert given problem to a numeric vector - * - * TunaNet takes in a numeric vector representing the given problem. The exact details - * of this vector vary from one TunaNet model to another, and thus this function, which - * converts a problem into a numeric vector that can be fed to TunaNet, must be implemented - * by each sub-class of `Model` on its own. - * - * @param problem Problem - */ - virtual std::vector ToFeatures(const conv::ProblemDescription& problem) const = 0; -}; - -class Gfx908Model final : public Model -{ -public: - Gfx908Model() : Model("gfx908") {} - bool IsProblemSupported(const conv::ProblemDescription& problem, - const ExecutionContext& ctx) const override - { - // check if problem is of the kind TunaNet was trained to handle - if(!problem.Is2d()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Problem not 2D"); - return false; - } - if(problem.GetGroupCount() != 1) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Group count not 1"); - return false; - } - if(problem.GetInLayout() != "NCHW" && problem.GetInLayout() != "NCDHW") - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Layout not supported"); - return false; - } - if(problem.GetWeightsHeight() != problem.GetWeightsWidth()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Filters must be square (fil_h == fil_w)"); - return false; - } - if(problem.GetPadH() != problem.GetPadW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Padding must be equal along all axes"); - return false; - } - if(problem.GetKernelStrideH() != problem.GetKernelStrideW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Stride must be equal along all axes"); - return false; - } - if(problem.GetDilationH() != 1 || problem.GetDilationW() != 1) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Dilation must be 1"); - return false; - } - const auto data_type = problem.GetInDataType(); - if(data_type != miopenFloat && data_type != miopenHalf && data_type != miopenBFloat16) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Unsupported data type"); - return false; - } - - // check if the context is s.t. no solver TunaNet may predict would be applicable - size_t applicable_solvers = 0; - for(const auto& solver_name : metadata.solver_map) - { - auto solver_id = solver::Id{solver_name.second}; - auto solver = solver_id.GetSolver(); - if(solver.IsApplicable(ctx, problem)) - { - applicable_solvers++; - break; - } - } - if(applicable_solvers == 0) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: No solver that TunaNet may predict applies"); - return false; - } - return true; - } - -protected: - std::vector ToFeatures(const conv::ProblemDescription& problem) const override - { - const bool isFwd = problem.GetDirection() == conv::Direction::Forward; - std::vector features = { - static_cast(isFwd ? problem.GetInChannels() : problem.GetOutChannels()), - static_cast(isFwd ? problem.GetInDepth() : problem.GetOutDepth()), - static_cast(isFwd ? problem.GetInHeight() : problem.GetOutHeight()), - static_cast(isFwd ? problem.GetInWidth() : problem.GetOutWidth()), - static_cast(problem.GetWeightsDepth()), - static_cast(problem.GetWeightsHeight()), - static_cast(problem.GetWeightsWidth()), - static_cast(isFwd ? problem.GetOutChannels() : problem.GetInChannels()), - static_cast(isFwd ? problem.GetOutDepth() : problem.GetInDepth()), - static_cast(isFwd ? problem.GetOutHeight() : problem.GetInHeight()), - static_cast(isFwd ? problem.GetOutWidth() : problem.GetInWidth()), - static_cast(problem.GetOutBatchSize()), - static_cast(1), // TunaNet was trained on a dataset of 2D - // problems where PadD was incorrectly set to 1 - static_cast(problem.GetPadH()), - static_cast(problem.GetPadW()), - static_cast(1), // TunaNet was trained on a dataset of 2D - // problems where StrideD was incorrectly set to 1 - static_cast(problem.GetKernelStrideH()), - static_cast(problem.GetKernelStrideW()), - static_cast(problem.GetDilationH()), - static_cast(problem.GetDilationW()), - static_cast(metadata.EncodeLayout(problem.GetInLayout())), - static_cast(metadata.EncodePrecision(problem.GetInDataType())), - static_cast(metadata.EncodeDirection(problem.GetDirection())), - static_cast(problem.GetGroupCount())}; - - // normalize - for(size_t i = 0; i < features.size(); ++i) - features[i] = (features[i] - metadata.features_mean[i]) / metadata.features_std[i]; - - return features; - } -}; - -class Gfx90aModel final : public Model -{ -public: - Gfx90aModel() : Model("gfx90a") {} - bool IsProblemSupported(const conv::ProblemDescription& problem, - const ExecutionContext& ctx) const override - { - // check if problem is of the kind TunaNet was trained to handle - if(!problem.Is2d()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Problem not 2D"); - return false; - } - if(problem.GetInLayout() != "NCHW") - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Layout not supported"); - return false; - } - if(problem.GetKernelStrideH() != problem.GetKernelStrideW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Stride must be equal along all axes"); - return false; - } - if(problem.GetDilationH() != problem.GetDilationW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Dilation must be 1"); - return false; - } - if(problem.GetBias() != 0) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Bias must be 0"); - return false; - } - const auto data_type = problem.GetInDataType(); - if(data_type != miopenFloat && data_type != miopenHalf && data_type != miopenBFloat16) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Unsupported data type"); - return false; - } - - // check if the context is s.t. no solver TunaNet may predict would be applicable - size_t applicable_solvers = 0; - for(const auto& solver_name : metadata.solver_map) - { - auto solver_id = solver::Id{solver_name.second}; - auto solver = solver_id.GetSolver(); - if(solver.IsApplicable(ctx, problem)) - { - applicable_solvers++; - break; - } - } - if(applicable_solvers == 0) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: No solver that TunaNet may predict applies"); - return false; - } - MIOPEN_LOG_I2("TunaNet Applicable"); - return true; - } - -protected: - std::vector ToFeatures(const conv::ProblemDescription& problem) const override - { - const bool isFwd = problem.GetDirection() == conv::Direction::Forward; - std::vector features = { - static_cast(isFwd ? problem.GetInChannels() : problem.GetOutChannels()), - static_cast(isFwd ? problem.GetInHeight() : problem.GetOutHeight()), - static_cast(isFwd ? problem.GetInWidth() : problem.GetOutWidth()), - static_cast(isFwd ? problem.GetOutChannels() : problem.GetInChannels()), - static_cast(isFwd ? problem.GetOutHeight() : problem.GetInHeight()), - static_cast(isFwd ? problem.GetOutWidth() : problem.GetInWidth()), - static_cast(problem.GetWeightsHeight()), - static_cast(problem.GetWeightsWidth()), - static_cast(problem.GetPadH()), - static_cast(problem.GetPadW()), - static_cast(problem.GetKernelStrideH()), - static_cast(problem.GetKernelStrideW()), - static_cast(problem.GetDilationH()), - static_cast(problem.GetDilationW()), - static_cast(problem.GetOutBatchSize()), - static_cast(metadata.EncodePrecision(problem.GetInDataType())), - static_cast(metadata.EncodeDirection(problem.GetDirection())), - static_cast(problem.GetGroupCount())}; - - // normalize - for(size_t i = 0; i < features.size(); ++i) - features[i] = (features[i] - metadata.features_mean[i]) / metadata.features_std[i]; - - return features; - } -}; - -class Gfx942Model final : public Model -{ -public: - Gfx942Model() : Model("gfx942") {} - bool IsProblemSupported(const conv::ProblemDescription& problem, - const ExecutionContext& ctx) const override - { - // check if problem is of the kind TunaNet was trained to handle - if(!problem.Is2d()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Problem not 2D"); - return false; - } - if(problem.GetInLayout() != "NCHW") - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Layout not supported"); - return false; - } - if(problem.GetKernelStrideH() != problem.GetKernelStrideW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Stride must be equal along all axes"); - return false; - } - if(problem.GetDilationH() != problem.GetDilationW()) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Dilation must be 1"); - return false; - } - if(problem.GetBias() != 0) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Bias must be 0"); - return false; - } - const auto data_type = problem.GetInDataType(); - if(data_type != miopenFloat && data_type != miopenHalf && data_type != miopenBFloat16) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: Unsupported data type"); - return false; - } - - // check if the context is s.t. no solver TunaNet may predict would be applicable - size_t applicable_solvers = 0; - for(const auto& solver_name : metadata.solver_map) - { - auto solver_id = solver::Id{solver_name.second}; - auto solver = solver_id.GetSolver(); - if(solver.IsApplicable(ctx, problem)) - { - applicable_solvers++; - break; - } - } - if(applicable_solvers == 0) - { - MIOPEN_LOG_I2("TunaNet Inapplicable: No solver that TunaNet may predict applies"); - return false; - } - MIOPEN_LOG_I2("TunaNet Applicable"); - return true; - } - -protected: - std::vector ToFeatures(const conv::ProblemDescription& problem) const override - { - const bool isFwd = problem.GetDirection() == conv::Direction::Forward; - std::vector features = { - static_cast(isFwd ? problem.GetInChannels() : problem.GetOutChannels()), - static_cast(isFwd ? problem.GetInHeight() : problem.GetOutHeight()), - static_cast(isFwd ? problem.GetInWidth() : problem.GetOutWidth()), - static_cast(isFwd ? problem.GetOutChannels() : problem.GetInChannels()), - static_cast(isFwd ? problem.GetOutHeight() : problem.GetInHeight()), - static_cast(isFwd ? problem.GetOutWidth() : problem.GetInWidth()), - static_cast(problem.GetWeightsHeight()), - static_cast(problem.GetWeightsWidth()), - static_cast(problem.GetPadH()), - static_cast(problem.GetPadW()), - static_cast(problem.GetKernelStrideH()), - static_cast(problem.GetKernelStrideW()), - static_cast(problem.GetDilationH()), - static_cast(problem.GetDilationW()), - static_cast(problem.GetOutBatchSize()), - static_cast(metadata.EncodePrecision(problem.GetInDataType())), - static_cast(metadata.EncodeDirection(problem.GetDirection())), - static_cast(problem.GetGroupCount())}; - - // normalize - for(size_t i = 0; i < features.size(); ++i) - features[i] = - (features[i] - metadata.test_features_mean[i]) / metadata.test_features_std[i]; - - return features; - } -}; - -std::unique_ptr GetModel(const std::string& device) -{ - if(device == "gfx942") - return std::make_unique(); - if(device == "gfx90a") - return std::make_unique(); - return std::make_unique(); // default model if GPU-specific model is not available -} - -std::vector PredictSolver(const conv::ProblemDescription& problem, - const ExecutionContext& ctx, - const std::string& device) -{ - const static std::unique_ptr model = GetModel(device); - if(!model || !model->IsProblemSupported(problem, ctx)) - return {}; - - std::string est_name = ":memory:" + device; - auto& db = AnyRamDb::GetCached(est_name); - auto db_res = db.FindRecord(problem); - if(db_res) - { - MIOPEN_LOG_I2("Cached heuristic (TunaNet) result found"); - std::vector db_sol(db_res->size()); - // cast returned record to solver ids - std::transform(db_res->begin(), db_res->end(), db_sol.begin(), [](boost::any id) { - return boost::any_cast(id); - }); - if(miopen::IsLogging(LoggingLevel::Info2)) - { - std::stringstream ss; - for(auto& id : db_sol) - ss << solver::Id{id}.ToString() << " ID:" << id << ", "; - MIOPEN_LOG_I2("Cached solvers: " << ss.str()); - } - return db_sol; - } - - MIOPEN_LOG_I2("Evaluating TunaNet"); - std::vector res = model->Forward(problem); // res[i] gives the probability that the - // i-th solver is the fastest for given - // problem. ( The exact name of the i-th - // solver may be obtained as follows: - // model->metadata.solver_map.at(i) ) - - // sort solvers in order of their probabilities - std::vector> sort_res(res.size()); - for(auto idx = 0; idx < res.size(); idx++) - sort_res[idx] = {idx, res[idx]}; - const auto cmp = [](const std::pair& a, const std::pair& b) -> bool { - return a.second > b.second; - }; - std::sort(sort_res.begin(), sort_res.end(), cmp); - - // map solver idx to solver id and then to anysolver - std::vector sol; - std::vector any_sol; - for(const auto& kinder : sort_res) - { - const auto id = kinder.first; // index of solver in probability vector - const auto sol_id = solver::Id{model->metadata.solver_map.at(id)}; - if(!sol_id.IsValid()) - { - MIOPEN_LOG_I2("Invalid solver " << model->metadata.solver_map.at(id) << " removed"); - continue; - } - sol.push_back(sol_id.Value()); - any_sol.push_back(sol_id.Value()); - } - db.StoreRecord(problem, any_sol); - if(miopen::IsLogging(LoggingLevel::Info2)) - { - std::stringstream ss; - for(auto& id : sol) - ss << solver::Id{id}.ToString() << " ID:" << id << ", "; - MIOPEN_LOG_I2("TunaNet Result: " << ss.str()); - } - return sol; -} -} // namespace immed_mode -#endif // MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK - -#if MIOPEN_ENABLE_AI_KERNEL_TUNING -namespace tuning { - -Metadata::Metadata(const std::string& arch, const std::string& solver) -{ - const nlohmann::json metadata = - common::LoadJSON(GetSystemDbPath() / (arch + "_" + solver + "_metadata.ktn.model")); - predict_type = metadata["predict_type"].get(); - num_tuning_params = - metadata["num_tuning_params"].get>(); - tuning_decodings = - metadata["decodings"]["tunings"].get>(); -} - -class Model -{ -public: - Metadata metadata; - Model(const std::string& arch, const std::string& solver) - : metadata(Metadata(arch, solver)), - encoder(fdeep::load_model(EncoderPath(arch, solver), true, fdeep::dev_null_logger)), - decoder(fdeep::load_model(DecoderPath(arch, solver), true, fdeep::dev_null_logger)) - { - } - virtual ~Model() = default; - /** - * Encode the input features into a "context" tensor - * - * @param features Input features - * @param dim Dimension (must be equal to len(features) if transform - * is True and sqrt(len(features)) otherwise) - * @param transform Reshape input features into a square matrix? - */ - fdeep::tensors Encode(const std::vector& features, std::size_t dim, bool transform) const - { - // if transform==True, reshape input features into a matrix of `dim x dim` dimensions. - // otherwise, have them as a vector of size `dim`. - const auto tensor_shape_depth = transform ? dim : 1; - fdeep::tensor input_tensor = - fdeep::tensor(fdeep::tensor_shape(dim, tensor_shape_depth), features); - - return encoder.predict({input_tensor}); - } - /** - * Decode the next token based on the previous token and the encoded context. - * - * Decoder predicts the next token based on the previous token and the context predicted - * by the Encoder. A token is a representation of a kernel parameter, i.e., each unique - * token maps to a unique kernel parameter, with the only exception being the token '-1' - * which signals the end of the decoding process (i.e., all kernel parameters have been - * obtained). - * - * @param prev_token Previous token - * @param context Context vector obtained from encoder - */ - fdeep::tensors Decode(const float prev_token, const fdeep::tensors& context) const - { - return decoder.predict( - {{fdeep::tensor(fdeep::tensor_shape(1), std::vector(1, prev_token)), - context[0], - context[1], - context[2], - context[3]}}); - } - -private: - const fdeep::model encoder; - const fdeep::model decoder; - static std::string EncoderPath(const std::string& arch, const std::string& solver) - { - const auto path = GetSystemDbPath() / (arch + "_" + solver + "_encoder.ktn.model"); - if(!fs::exists(path)) - MIOPEN_THROW(miopenStatusInternalError, "Unable to load file: " + path); - return path.string(); - } - static std::string DecoderPath(const std::string& arch, const std::string& solver) - { - const auto path = GetSystemDbPath() / (arch + "_" + solver + "_decoder.ktn.model"); - if(!fs::exists(path)) - MIOPEN_THROW(miopenStatusInternalError, "Unable to load file: " + path); - return path.string(); - } -}; - -/** - * Return the KernelTuningNet model for given architecture and solver - * - * KernelTuningNet models are specific to each solver and are fine-tuned for each - * GPU skew. This function constructs the KernelTuningNet model for the given - * architecture and solver and stores it in a static map, so that the next time - * the same model is required it doesn't have to be constructed anew. - * - * @param arch GPU Architecture - * @param solver Solver - */ -std::shared_ptr GetModel(const std::string& arch, const std::string& solver) -{ - static std::map> models; - auto it = models.find(solver); - if(it == models.end()) - { - std::shared_ptr model = std::make_shared(arch, solver); - models[solver] = model; - return model; - } - else - { - return it->second; - } -} - -/** - * Set kernel parameters for given solver - * - * @param arch GPU Architecture - * @param solver Solver - * @param direction Convolution Direction - * @param features Input features for KernelTuningNet model - * @param transform_features Whether or not to reshape features into a square - * matrix before feeding them to KernelTuningNet - * @param validator A boolean function that accepts an index `i` and a string `v`, and returns - * True iff `v` is a valid kernel parameter value at index `i` - */ -bool ModelSetParams(const std::string& arch, - const std::string& solver, - miopen::conv::Direction direction, - const std::vector& features, - bool transform_features, - std::function validator) -{ - auto model = GetModel(arch, solver); - - // get context - int dim = 0; - if(transform_features) - dim = std::sqrt(features.size()); - else - dim = features.size(); - auto start = std::chrono::high_resolution_clock::now(); - fdeep::tensors context = model->Encode(features, dim, transform_features); - float decoder_input = 0.0; - - // set direction string - std::string dir; - switch(direction) - { - case miopen::conv::Direction::Forward: dir = "fwd"; break; - case miopen::conv::Direction::BackwardData: dir = "bwd"; break; - case miopen::conv::Direction::BackwardWeights: dir = "wrw"; break; - default: return false; - } - - // run decoder to set kernel parameters - for(size_t i = 0, num_tuning_params = 1; i < num_tuning_params; ++i) - { - - if(i == 0 && (model->metadata.predict_type == 0u)) - num_tuning_params = model->metadata.num_tuning_params[dir]; - - fdeep::tensors decoder_output = model->Decode(decoder_input, context); - auto token_scores = decoder_output[0].to_vector(); // token_scores[k] gives the - // score of the k-th token - // order tokens according to their scores - std::priority_queue> pq; - for(int j = 0; j < token_scores.size(); j++) - pq.push(std::make_pair(token_scores[j], j)); // sort by value at index - - // find a token whose value is a valid kernel parameter for the i-th position - int output_token_index = -1; - while(!pq.empty()) - { - // get the token with the highest score and look up its value - int token = pq.top().second; - std::string value = model->metadata.tuning_decodings[std::to_string(token)]; - pq.pop(); - - if(value == "-1") // if token-value is "-1", then decoding has finished - { - auto stop = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(stop - start); - MIOPEN_LOG_I2("Model ran for " << duration.count() << " micro-seconds"); - return false; - } - if(validator(i, value)) // if token-value is a valid kernel parameter, it's set - { - output_token_index = - token; // index with largest value that is valid = predicted index - if(i == 0 && model->metadata.predict_type != 0u) - num_tuning_params = model->metadata.num_tuning_params[value]; - break; - } - } - decoder_input = float(output_token_index); - context = {decoder_output.begin() + 1, decoder_output.end()}; - } - - auto stop = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(stop - start); - MIOPEN_LOG_I2("Model ran for " << duration.count() << " micro-seconds"); - return true; -} - -} // namespace tuning -#endif // MIOPEN_ENABLE_AI_KERNEL_TUNING -} // namespace ai -} // namespace miopen -#endif // MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK || MIOPEN_ENABLE_AI_KERNEL_TUNING diff --git a/src/conv/invokers/impl_gemm_dynamic.cpp b/src/conv/invokers/impl_gemm_dynamic.cpp index 7dbd8cd5d9..e69de29bb2 100644 --- a/src/conv/invokers/impl_gemm_dynamic.cpp +++ b/src/conv/invokers/impl_gemm_dynamic.cpp @@ -1,1176 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace miopen { -namespace conv { - -static inline uint32_t igemm_find_tile_size_with_upper_bound( - uint32_t out_size, size_t upper_bound, uint32_t stride, uint32_t dilation, uint32_t filter) -{ - // return tile size so that the required input tile(sec_in) is no larger than upper_bound - uint32_t n_tiles = 1; - if(n_tiles <= out_size) - { - for(; n_tiles <= out_size; n_tiles++) - { - uint32_t tile_size = (out_size + n_tiles - 1) / n_tiles; - uint32_t sec_in = (tile_size - 1) * stride + 1 + dilation * (filter - 1); - if(sec_in <= upper_bound) - break; - } - } - else - MIOPEN_THROW("out_size should not be less than one"); - - return (out_size + n_tiles - 1) / n_tiles; -} - -static float CallImplGemmDynamicForward1x1(const miopen::Handle& handle, - const ProblemDescription& problem, - ConstData_t src, - Data_t dst, - ConstData_t wei, - const std::vector& kernels) -{ - float elapsed = 0.0f; - - auto kernel = kernels[0]; - MIOPEN_LOG_I(kernel.GetName()); - - // clang-format off - int hi = problem.GetInHeight(); - int wi = problem.GetInWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetOutChannels(); - int c = problem.GetInChannels(); - int ho = problem.GetOutHeight(); - int wo = problem.GetOutWidth(); - int stride_h = problem.GetKernelStrideH(); - int stride_w = problem.GetKernelStrideW(); - int dilation_h = problem.GetDilationH(); - int dilation_w = problem.GetDilationW(); - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int gap_0 = 0; - // clang-format on - - std::vector opArgs; - opArgs.emplace_back(src); - opArgs.emplace_back(wei); - opArgs.emplace_back(dst); - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n); - opArgs.emplace_back(k); - opArgs.emplace_back(c); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_h); - opArgs.emplace_back(stride_w); - opArgs.emplace_back(dilation_h); - opArgs.emplace_back(dilation_w); - opArgs.emplace_back(pad_h); - opArgs.emplace_back(pad_w); - opArgs.emplace_back(gap_0); - - kernel(opArgs); - - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - return elapsed; -} - -InvokerFactory MakeImplGemmDynamicForward1x1InvokerFactory(const ProblemDescription& problem) -{ - return [problem](const std::vector& kernels) { - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - auto kernel = handle.Run(kernels[0]); - - std::vector ks; - std::transform(kernels.begin(), - kernels.end(), - std::back_inserter(ks), - [&](const Kernel& k) { return handle.Run(k); }); - float elapsed = 0; - elapsed = CallImplGemmDynamicForward1x1( - handle, problem, tensors.in, tensors.out, tensors.w, ks); - if(handle.IsProfilingEnabled()) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} - -InvokerFactory MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescription& problem, - const int cfg) -{ - int hi = problem.GetOutHeight(); - int wi = problem.GetOutWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetInChannels(); - int c = problem.GetOutChannels(); - int ho = problem.GetInHeight(); - int wo = problem.GetInWidth(); - int stride_h = problem.GetInHeight() > 1 ? problem.GetKernelStrideH() : 1; - int stride_w = problem.GetInWidth() > 1 ? problem.GetKernelStrideW() : 1; - int dilation_h = problem.GetWeightsHeight() > 1 ? problem.GetDilationH() : 1; - int dilation_w = problem.GetWeightsWidth() > 1 ? problem.GetDilationW() : 1; - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - - int gcd_stride_dilation_h = solver::gcd(stride_h, dilation_h); - int gcd_stride_dilation_w = solver::gcd(stride_w, dilation_w); - int y_tilda = stride_h / gcd_stride_dilation_h; - int x_tilda = stride_w / gcd_stride_dilation_w; - - int y_dot = (y + y_tilda - 1) / y_tilda; - int x_dot = (x + x_tilda - 1) / x_tilda; - - int h_tilda = ho + (dilation_h * (y - 1) + stride_h - 1) / stride_h; - int w_tilda = wo + (dilation_w * (x - 1) + stride_w - 1) / stride_w; - - int h_tilda_left = std::max(0, pad_h - dilation_h * (y_tilda - 1)) / stride_h; - int w_tilda_left = std::max(0, pad_w - dilation_w * (x_tilda - 1)) / stride_w; - - int h_tilda_right = std::min(h_tilda, (pad_h + hi - 1 + stride_h - 1) / stride_h + 1); - int w_tilda_right = std::min(w_tilda, (pad_w + wi - 1 + stride_w - 1) / stride_w + 1); - - int h_tilda_slice = h_tilda_right - h_tilda_left; - int w_tilda_slice = w_tilda_right - w_tilda_left; - - int num_of_gemms = x_tilda * y_tilda; - - int dtile_dy = dilation_h / gcd_stride_dilation_h; - int dtile_dx = dilation_w / gcd_stride_dilation_w; - int dtile_y = y_tilda; - int dtile_x = x_tilda; - int dtile_h = h_tilda; - int dtile_w = w_tilda; - int dslice_h = h_tilda_slice; - int dslice_w = w_tilda_slice; - int dslice_h_left = h_tilda_left; - int dslice_w_left = w_tilda_left; - int pack_align = cfg; - std::vector dtile_iy_gid; - std::vector dtile_ix_gid; - std::vector y_dot_slice_gid; - std::vector x_dot_slice_gid; - std::vector is_gemm_not_empty; - for(int gemm_id = 0; gemm_id < num_of_gemms; gemm_id++) - { - dtile_iy_gid.emplace_back(gemm_id / x_tilda); - dtile_ix_gid.emplace_back(gemm_id % x_tilda); - y_dot_slice_gid.emplace_back((dtile_iy_gid[gemm_id] + 1) * y_dot <= y ? y_dot : y % y_dot); - x_dot_slice_gid.emplace_back((dtile_ix_gid[gemm_id] + 1) * x_dot <= x ? x_dot : x % x_dot); - const int gemm_k_gid = k * y_dot_slice_gid[gemm_id] * x_dot_slice_gid[gemm_id]; - is_gemm_not_empty.emplace_back(gemm_k_gid > 0); - } - bool need_set_zero = false; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; - - return [=](const std::vector& kernels) { - const auto kernel = kernels[0]; - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - float elapsed = 0; - if(need_set_zero) - { - float zero = 0.f; - SetTensor(handle, tensors.outDesc, tensors.out, &zero); - - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - for(int gemm_id = 0; gemm_id < num_of_gemms; gemm_id++) - { - if(is_gemm_not_empty[gemm_id]) - { - handle.Run(kernel)(tensors.out, - tensors.w, - tensors.in, - hi, - wi, - n, - k, - c, - ho, - wo, - stride_h, - stride_w, - dilation_h, - dilation_w, - pad_h, - pad_w, - y, - x, - dtile_iy_gid[gemm_id], - dtile_ix_gid[gemm_id], - dtile_dy, - dtile_dx, - dtile_y, - dtile_x, - dtile_h, - dtile_w, - y_dot_slice_gid[gemm_id], - x_dot_slice_gid[gemm_id], - dslice_h, - dslice_w, - dslice_h_left, - dslice_w_left, - pack_align); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - } - - if(handle.IsProfilingEnabled()) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} - -InvokerFactory -MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescription& problem, - const solver::TunableImplicitGemmGTCDynamic_t& cfg) -{ - int hi = problem.GetOutHeight(); - int wi = problem.GetOutWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetInChannels(); - int c = problem.GetOutChannels(); - int ho = problem.GetInHeight(); - int wo = problem.GetInWidth(); - int stride_h = problem.GetOutHeight() > 1 ? problem.GetKernelStrideH() : 1; - int stride_w = problem.GetOutWidth() > 1 ? problem.GetKernelStrideW() : 1; - int dilation_h = problem.GetWeightsHeight() > 1 ? problem.GetDilationH() : 1; - int dilation_w = problem.GetWeightsWidth() > 1 ? problem.GetDilationW() : 1; - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int group = problem.GetGroupCount(); - - int gcd_stride_dilation_h = solver::gcd(stride_h, dilation_h); - int gcd_stride_dilation_w = solver::gcd(stride_w, dilation_w); - int y_tilda = stride_h / gcd_stride_dilation_h; - int x_tilda = stride_w / gcd_stride_dilation_w; - - int y_dot = (y + y_tilda - 1) / y_tilda; - int x_dot = (x + x_tilda - 1) / x_tilda; - - int h_tilda = ho + (dilation_h * (y - 1) + stride_h - 1) / stride_h; - int w_tilda = wo + (dilation_w * (x - 1) + stride_w - 1) / stride_w; - - int h_tilda_left = std::max(0, pad_h - dilation_h * (y_tilda - 1)) / stride_h; - int w_tilda_left = std::max(0, pad_w - dilation_w * (x_tilda - 1)) / stride_w; - - int h_tilda_right = std::min(h_tilda, (pad_h + hi - 1 + stride_h - 1) / stride_h + 1); - int w_tilda_right = std::min(w_tilda, (pad_w + wi - 1 + stride_w - 1) / stride_w + 1); - - int h_tilda_slice = h_tilda_right - h_tilda_left; - int w_tilda_slice = w_tilda_right - w_tilda_left; - - int num_of_gemms = x_tilda * y_tilda; - - int dtile_dy = dilation_h / gcd_stride_dilation_h; - int dtile_dx = dilation_w / gcd_stride_dilation_w; - int dtile_y = y_tilda; - int dtile_x = x_tilda; - int dtile_h = h_tilda; - int dtile_w = w_tilda; - int dslice_h = h_tilda_slice; - int dslice_w = w_tilda_slice; - int dslice_h_left = h_tilda_left; - int dslice_w_left = w_tilda_left; - int pack_align = 0; - std::vector dtile_iy_gid; - std::vector dtile_ix_gid; - std::vector y_dot_slice_gid; - std::vector x_dot_slice_gid; - std::vector is_gemm_not_empty; - for(int gemm_id = 0; gemm_id < num_of_gemms; gemm_id++) - { - dtile_iy_gid.emplace_back(gemm_id / x_tilda); - dtile_ix_gid.emplace_back(gemm_id % x_tilda); - y_dot_slice_gid.emplace_back((dtile_iy_gid[gemm_id] + 1) * y_dot <= y ? y_dot : y % y_dot); - x_dot_slice_gid.emplace_back((dtile_ix_gid[gemm_id] + 1) * x_dot <= x ? x_dot : x % x_dot); - const int gemm_k_gid = k * y_dot_slice_gid[gemm_id] * x_dot_slice_gid[gemm_id]; - is_gemm_not_empty.emplace_back(gemm_k_gid > 0); - } - bool need_set_zero = false; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; - - int nxb = cfg.nxb; - int b = h_tilda_slice * w_tilda_slice; - b = (cfg.nxe == 0) ? (b) : ((b + nxb - 1) / nxb) * nxb; // pad to nxb modulo when nxe != 0 - - uint32_t nb_n0 = cfg.tensor_b_cluster_lengths[2] * cfg.tensor_b_thread_lengths[2]; - uint32_t nb_n1b = cfg.tensor_b_cluster_lengths[3] * cfg.tensor_b_thread_lengths[3]; - uint32_t unmerge_sub_n = cfg.gemm_n_per_block / cfg.nxb; - uint32_t unmerge_sub_n1 = unmerge_sub_n / nb_n0; - - magic_div_u32_t mdiv_2 = - magic_div_u32_gen(((c / group) * n * b) / (cfg.gemm_m_per_block * cfg.gemm_n_per_block)); - magic_div_u32_t mdiv_3 = magic_div_u32_gen((n * b) / cfg.gemm_n_per_block); - magic_div_u32_t mdiv_4 = magic_div_u32_gen(b * unmerge_sub_n1 / nb_n1b); - magic_div_u32_t mdiv_5 = magic_div_u32_gen(b); - magic_div_u32_t mdiv_6 = magic_div_u32_gen(w_tilda_slice); - - std::vector mdiv_0_vec; - std::vector mdiv_1_vec; - std::vector shift_pack_0_vec; - uint32_t shift_pack_1; - - for(int gemm_id = 0; gemm_id < num_of_gemms; gemm_id++) - { - if(is_gemm_not_empty[gemm_id]) - { - mdiv_0_vec.push_back( - magic_div_u32_gen(y_dot_slice_gid[gemm_id] * x_dot_slice_gid[gemm_id])); - mdiv_1_vec.push_back(magic_div_u32_gen(x_dot_slice_gid[gemm_id])); - } - else - { - mdiv_0_vec.push_back(magic_div_u32_t({0, 0})); - mdiv_1_vec.push_back(magic_div_u32_t({0, 0})); - }; - - shift_pack_0_vec.push_back(magic_div_u32_pack_shift( - mdiv_0_vec[gemm_id].shift, mdiv_1_vec[gemm_id].shift, mdiv_2.shift, mdiv_3.shift)); - }; - - shift_pack_1 = magic_div_u32_pack_shift(mdiv_4.shift, mdiv_5.shift, mdiv_6.shift, 0); - - return [=](const std::vector& kernels) { - const auto kernel = kernels[0]; - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - float elapsed = 0; - if(need_set_zero) - { - float zero = 0.f; - SetTensor(handle, tensors.outDesc, tensors.out, &zero); - - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - for(int gemm_id = 0; gemm_id < num_of_gemms; gemm_id++) - { - if(is_gemm_not_empty[gemm_id]) - { - handle.Run(kernel)(tensors.out, - tensors.w, - tensors.in, - hi, - wi, - n, - k / group, - c / group, - ho, - wo, - stride_h, - stride_w, - dilation_h, - dilation_w, - pad_h, - pad_w, - y, - x, - dtile_iy_gid[gemm_id], - dtile_ix_gid[gemm_id], - dtile_dy, - dtile_dx, - dtile_y, - dtile_x, - dtile_h, - dtile_w, - y_dot_slice_gid[gemm_id], - x_dot_slice_gid[gemm_id], - dslice_h, - dslice_w, - dslice_h_left, - dslice_w_left, - group, - mdiv_0_vec[gemm_id].magic, - mdiv_1_vec[gemm_id].magic, - mdiv_2.magic, - mdiv_3.magic, - mdiv_4.magic, - mdiv_5.magic, - mdiv_6.magic, - shift_pack_0_vec[gemm_id], - shift_pack_1, - pack_align); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - } - if(handle.IsProfilingEnabled()) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} - -InvokerFactory MakeImplGemmDynamicForwardXdlopsNHWCInvokerFactory( - const ExecutionContext& ctx, - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC& config) -{ - int hi = problem.GetInHeight(); - int wi = problem.GetInWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetOutChannels(); - int c = problem.GetInChannels(); - int ho = problem.GetOutHeight(); - int wo = problem.GetOutWidth(); - int stride_h = problem.GetKernelStrideH(); - int stride_w = problem.GetKernelStrideW(); - int dilation_h = problem.GetDilationH(); - int dilation_w = problem.GetDilationW(); - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int group = problem.GetGroupCount(); - int c_karg = c / group; - int y_karg = y; - int x_karg = x; - - int splits_4G = solver::igemm_split_batch_size( - hi, wi, ho, wo, n, k, c, miopen::GetTypeSize(problem.GetInDataType())); - splits_4G = splits_4G == 0 ? n : splits_4G; - - uint32_t gemm_m = (n / splits_4G) * ho * wo; - uint32_t gemm_n = k / group; - magic_div_u32_t mdiv_0, mdiv_1, mdiv_2, mdiv_3, mdiv_4, mdiv_5; - uint32_t shift_pack_0, shift_pack_1; - uint32_t pack0 = 0; - - mdiv_0 = magic_div_u32_gen((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block); - mdiv_1 = magic_div_u32_gen(ho * wo); - mdiv_2 = magic_div_u32_gen(wo); - mdiv_3 = magic_div_u32_gen(((gemm_m + config.gemm_m_per_block - 1) / config.gemm_m_per_block) * - ((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block)); - - shift_pack_0 = magic_div_u32_pack_shift(mdiv_0.shift, mdiv_1.shift, mdiv_2.shift, mdiv_3.shift); - if(config.merge_e != 0) - { - mdiv_4 = magic_div_u32_gen(x * (c / group)); - mdiv_5 = magic_div_u32_gen(c / group); - shift_pack_1 = magic_div_u32_pack_shift(mdiv_4.shift, mdiv_5.shift, 0, 0); - - uint32_t s_move_slice_k_y = (config.gemm_k_per_block / (x * (c / group))) % y; - uint32_t s_move_slice_k_x = (config.gemm_k_per_block / (c / group)) % x; - uint32_t s_move_slice_k_c = config.gemm_k_per_block % (c / group); - y_karg = static_cast((s_move_slice_k_y << 24) | y); - x_karg = static_cast((s_move_slice_k_x << 24) | x); - c_karg = static_cast((s_move_slice_k_c << 24) | (c / group)); - } - else - { - mdiv_4 = magic_div_u32_gen(1); - mdiv_5 = magic_div_u32_gen(1); - shift_pack_1 = 0; - } - - bool need_set_zero = config.gemm_k_global_split > 0; - bool use_fp32_global_split_on_fp16 = config.vector_store == 1 && config.gemm_k_global_split > 0; - - std::vector opArgs; - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n / splits_4G); - opArgs.emplace_back(k / group); - opArgs.emplace_back(c_karg); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_h); - opArgs.emplace_back(stride_w); - opArgs.emplace_back(dilation_h); - opArgs.emplace_back(dilation_w); - opArgs.emplace_back(pad_h); - opArgs.emplace_back(pad_w); - opArgs.emplace_back(y_karg); - opArgs.emplace_back(x_karg); - opArgs.emplace_back(group); - opArgs.emplace_back(mdiv_0.magic); - opArgs.emplace_back(mdiv_1.magic); - opArgs.emplace_back(mdiv_2.magic); - opArgs.emplace_back(mdiv_3.magic); - opArgs.emplace_back(mdiv_4.magic); - opArgs.emplace_back(mdiv_5.magic); - opArgs.emplace_back(shift_pack_0); - opArgs.emplace_back(shift_pack_1); - opArgs.emplace_back(config.gemm_k_global_split); - opArgs.emplace_back(pack0); - - std::vector> opArgsTrans; - - const auto lowp_quant = problem.GetConv().lowp_quant; - const auto isGfx90aFp16altSupport = - (ctx.GetStream().GetDeviceName() == "gfx90a") && problem.IsFp16(); - - const bool need_cast = [&]() { - if(problem.GetOut().GetType() == miopenHalf) - return use_fp32_global_split_on_fp16; - if(problem.GetOut().GetType() == miopenBFloat16) - return need_set_zero; - return false; - }(); - const auto is_nchw = problem.IsLayoutDefault(); - - size_t trans_input_offset = 0; - size_t trans_input_size = 0; - - size_t trans_weight_offset = 0; - size_t trans_weight_size = 0; - - size_t trans_output_offset = 0; - size_t trans_output_size = 0; - - bool trans_input_skippable = false; - bool trans_weight_skippable = false; - bool trans_output_skippable = false; - - int trans_input_idx = -1; - int trans_weight_idx = -1; - int trans_output_idx = -1; - - if(is_nchw) - { - TransposeSolutionDefault2Nhwc trans_input(ctx, problem.GetInDataType(), n, c, hi, wi); - TransposeSolutionDefault2Nhwc trans_weight(ctx, - problem.GetWeightsDataType(), - k, - c / group, - y, - x); // group * k_per_group as batch for weight - TransposeSolutionNhwc2Default trans_output(ctx, problem.GetOutDataType(), n, k, ho, wo); - - trans_input_skippable = trans_input.IsSkippable(); - trans_weight_skippable = trans_weight.IsSkippable(); - trans_output_skippable = trans_output.IsSkippable(); - - if(!trans_input_skippable) - opArgsTrans.emplace_back(trans_input.GetKernelArg()); - if(!trans_weight_skippable) - opArgsTrans.emplace_back(trans_weight.GetKernelArg()); - if(!trans_output_skippable) - opArgsTrans.emplace_back(trans_output.GetKernelArg()); - - trans_input_size = trans_input_skippable ? 0 : trans_input.GetOutputTensorSize(); - trans_weight_size = trans_weight_skippable ? 0 : trans_weight.GetOutputTensorSize(); - trans_output_size = trans_output_skippable ? 0 : trans_output.GetOutputTensorSize(); - - int idx = 0; - if(!trans_input_skippable) - trans_input_idx = idx++; - if(!trans_weight_skippable) - trans_weight_idx = idx++; - if(!trans_output_skippable) - trans_output_idx = idx++; - } - - const size_t cast_size = need_cast ? miopen::GetTypeSize(miopenFloat) * n * k * ho * wo : 0; - - MultiBufferWorkspaceTraits wt( - {trans_input_size, trans_weight_size, trans_output_size, cast_size}); - - trans_input_offset = wt.GetOffset(0); - trans_weight_offset = wt.GetOffset(1); - trans_output_offset = wt.GetOffset(2); - - const size_t cast_offset = wt.GetOffset(3); - - const int kID_trans_start = isGfx90aFp16altSupport ? 2 : 1; - - const TensorDescriptor cast_desc( - miopenFloat, problem.GetOut().GetLengths(), problem.GetOut().GetStrides()); - auto null_buf = shared{}; - - return [=](const std::vector& kernels) mutable { - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) mutable { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - const auto& workSpace = data_ctx.workSpace; - const auto ker = - handle.Run(kernels[(isGfx90aFp16altSupport && data_ctx.gfx90aFp16alt) ? 1 : 0]); - float elapsed = 0; - - auto trans_input_buf = - trans_input_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_input_offset, trans_input_size); - auto trans_weight_buf = - trans_weight_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_weight_offset, trans_weight_size); - auto trans_output_buf = - trans_output_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_output_offset, trans_output_size); - auto cast_buf = cast_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, cast_offset, cast_size); - - if(need_set_zero) - { - auto zero_buf = need_cast - ? cast_buf.get() - : ((is_nchw && !trans_output_skippable) ? trans_output_buf.get() - : tensors.out); - auto& zero_desc = - need_cast - ? cast_desc - : tensors.outDesc; // use the same desc for NCHW/NHWC for this dense tensor - float zero = 0.f; - - SetTensor(handle, zero_desc, zero_buf, &zero); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - - if(is_nchw) - { - if(!trans_input_skippable) - { - auto& karg_input = opArgsTrans[trans_input_idx]; - karg_input[0] = OpKernelArg(trans_input_buf.get()); - karg_input[1] = OpKernelArg(tensors.in); - handle.Run(kernels[kID_trans_start + trans_input_idx])(karg_input); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - if(!trans_weight_skippable) - { - auto& karg_weight = opArgsTrans[trans_weight_idx]; - karg_weight[0] = OpKernelArg(trans_weight_buf.get()); - karg_weight[1] = OpKernelArg(tensors.w); - handle.Run(kernels[kID_trans_start + trans_weight_idx])(karg_weight); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - } - - opArgs[0] = (is_nchw && !trans_input_skippable) ? OpKernelArg(trans_input_buf.get()) - : OpKernelArg(tensors.in); - opArgs[1] = (is_nchw && !trans_weight_skippable) ? OpKernelArg(trans_weight_buf.get()) - : OpKernelArg(tensors.w); - - opArgs[2] = need_cast ? OpKernelArg(cast_buf.get()) - : ((is_nchw && !trans_output_skippable) - ? OpKernelArg(trans_output_buf.get()) - : OpKernelArg(tensors.out)); - ker(opArgs); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - - if(need_cast) - { - CastTensor(handle, - &lowp_quant, - problem.IsDirectionForward(), - cast_desc, - cast_buf.get(), - tensors.outDesc, - (is_nchw && !trans_output_skippable) ? trans_output_buf.get() - : tensors.out, - 0, - 0); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - - if(is_nchw && !trans_output_skippable) - { - auto& karg_output = opArgsTrans[trans_output_idx]; - karg_output[0] = OpKernelArg(tensors.out); - karg_output[1] = OpKernelArg(trans_output_buf.get()); - handle.Run(kernels[kID_trans_start + trans_output_idx])(karg_output); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - - if(handle.IsProfilingEnabled()) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} - -InvokerFactory MakeImplGemmDynamicBackwardDataXdlopsNHWCInvokerFactory( - const ExecutionContext& ctx, - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC& config) -{ - int hi = problem.GetOutHeight(); - int wi = problem.GetOutWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetInChannels(); - int c = problem.GetOutChannels(); - int ho = problem.GetInHeight(); - int wo = problem.GetInWidth(); - int stride_h = problem.GetOutHeight() > 1 ? problem.GetKernelStrideH() : 1; - int stride_w = problem.GetOutWidth() > 1 ? problem.GetKernelStrideW() : 1; - int dilation_h = problem.GetWeightsHeight() > 1 ? problem.GetDilationH() : 1; - int dilation_w = problem.GetWeightsWidth() > 1 ? problem.GetDilationW() : 1; - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int group = problem.GetGroupCount(); - - int gcd_stride_dilation_h = solver::gcd(stride_h, dilation_h); - int gcd_stride_dilation_w = solver::gcd(stride_w, dilation_w); - int y_tilda = stride_h / gcd_stride_dilation_h; - int x_tilda = stride_w / gcd_stride_dilation_w; - - int h_tilda = ho + (dilation_h * (y - 1) + stride_h - 1) / stride_h; - int w_tilda = wo + (dilation_w * (x - 1) + stride_w - 1) / stride_w; - - int h_tilda_left = std::max(0, pad_h - dilation_h * (y_tilda - 1)) / stride_h; - int w_tilda_left = std::max(0, pad_w - dilation_w * (x_tilda - 1)) / stride_w; - - int h_tilda_right = std::min(h_tilda, (pad_h + hi - 1 + stride_h - 1) / stride_h + 1); - int w_tilda_right = std::min(w_tilda, (pad_w + wi - 1 + stride_w - 1) / stride_w + 1); - - int h_tilda_slice = h_tilda_right - h_tilda_left; - int w_tilda_slice = w_tilda_right - w_tilda_left; - - int num_of_gemms = x_tilda * y_tilda; - - int splits_4G = solver::igemm_split_batch_size( - hi, wi, ho, wo, n, k, c, miopen::GetTypeSize(problem.GetInDataType())); - int n_in_1_block = splits_4G == 0 ? 1 : (n / splits_4G); - - uint32_t gemm_m = n_in_1_block * h_tilda_slice * w_tilda_slice; - uint32_t gemm_n = c / group; - - magic_div_u32_t mdiv_x_tilda = magic_div_u32_gen(x_tilda); - magic_div_u32_t mdiv_y_tilda = magic_div_u32_gen(y_tilda); - magic_div_u32_t mdiv_group_mn = magic_div_u32_gen( - group * ((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block) * - ((gemm_m + config.gemm_m_per_block - 1) / config.gemm_m_per_block)); - - magic_div_u32_t mdiv_0 = - magic_div_u32_gen((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block); - magic_div_u32_t mdiv_1 = - magic_div_u32_gen(((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block) * - ((gemm_m + config.gemm_m_per_block - 1) / config.gemm_m_per_block)); - magic_div_u32_t mdiv_2 = magic_div_u32_gen(config.nxe != 0 ? w_tilda_slice : wi); - magic_div_u32_t mdiv_3 = magic_div_u32_gen(h_tilda_slice * w_tilda_slice); - uint32_t shift_pack_0 = - magic_div_u32_pack_shift(mdiv_0.shift, mdiv_1.shift, mdiv_2.shift, mdiv_3.shift); - - int dtile_iy = num_of_gemms > 1 ? static_cast(mdiv_x_tilda.magic) : 0; - int dtile_ix = num_of_gemms > 1 ? static_cast(mdiv_x_tilda.shift) : 0; - int dslice_y = num_of_gemms > 1 ? static_cast(mdiv_y_tilda.magic) : y; - int dslice_x = num_of_gemms > 1 ? static_cast(mdiv_y_tilda.shift) : x; - int dtile_h = num_of_gemms > 1 ? static_cast(mdiv_group_mn.magic) : h_tilda; - int dtile_w = num_of_gemms > 1 ? static_cast(mdiv_group_mn.shift) : w_tilda; - - bool need_set_zero = false; - bool use_fp32_global_split_on_fp16 = config.vector_store == 1 && config.gemm_k_global_split > 0; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; - need_set_zero |= config.gemm_k_global_split > 0; - - std::vector opArgs; - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n_in_1_block); - opArgs.emplace_back(k / group); - opArgs.emplace_back(c / group); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_h); - opArgs.emplace_back(stride_w); - opArgs.emplace_back(dilation_h); - opArgs.emplace_back(dilation_w); - opArgs.emplace_back(pad_h); - opArgs.emplace_back(pad_w); - opArgs.emplace_back(y); - opArgs.emplace_back(x); - - opArgs.emplace_back(dtile_iy); - opArgs.emplace_back(dtile_ix); - opArgs.emplace_back(dilation_h / gcd_stride_dilation_h); - opArgs.emplace_back(dilation_w / gcd_stride_dilation_w); - opArgs.emplace_back(y_tilda); - opArgs.emplace_back(x_tilda); - opArgs.emplace_back(dtile_h); - opArgs.emplace_back(dtile_w); - opArgs.emplace_back(dslice_y); - opArgs.emplace_back(dslice_x); - - opArgs.emplace_back(h_tilda_slice); - opArgs.emplace_back(w_tilda_slice); - opArgs.emplace_back(h_tilda_left); - opArgs.emplace_back(w_tilda_left); - opArgs.emplace_back(group); - - opArgs.emplace_back(mdiv_0.magic); - opArgs.emplace_back(mdiv_1.magic); - opArgs.emplace_back(mdiv_2.magic); - opArgs.emplace_back(mdiv_3.magic); - opArgs.emplace_back(shift_pack_0); - opArgs.emplace_back(config.gemm_k_global_split); - - std::vector> opArgsTrans; - - const auto lowp_quant = problem.GetConv().lowp_quant; - const auto isGfx90aFp16altSupport = - (ctx.GetStream().GetDeviceName() == "gfx90a") && problem.IsFp16(); - const bool need_cast = [&]() { - if(problem.GetOut().GetType() == miopenHalf) - return use_fp32_global_split_on_fp16; - if(problem.GetOut().GetType() == miopenBFloat16) - return need_set_zero; - return false; - }(); - const auto is_nchw = problem.IsLayoutDefault(); - - size_t trans_input_offset = 0; - size_t trans_input_size = 0; - - size_t trans_weight_offset = 0; - size_t trans_weight_size = 0; - - size_t trans_output_offset = 0; - size_t trans_output_size = 0; - - bool trans_input_skippable = false; - bool trans_weight_skippable = false; - bool trans_output_skippable = false; - - int trans_input_idx = -1; - int trans_weight_idx = -1; - int trans_output_idx = -1; - - if(is_nchw) - { - TransposeSolutionNhwc2Default trans_input(ctx, problem.GetOutDataType(), n, c, hi, wi); - TransposeSolutionDefault2Nhwc trans_weight(ctx, - problem.GetWeightsDataType(), - k, - c / group, - y, - x); // group * k_per_group as batch for weight - TransposeSolutionDefault2Nhwc trans_output(ctx, problem.GetInDataType(), n, k, ho, wo); - - trans_input_skippable = trans_input.IsSkippable(); - trans_weight_skippable = trans_weight.IsSkippable(); - trans_output_skippable = trans_output.IsSkippable(); - - if(!trans_input_skippable) - opArgsTrans.emplace_back(trans_input.GetKernelArg()); - if(!trans_weight_skippable) - opArgsTrans.emplace_back(trans_weight.GetKernelArg()); - if(!trans_output_skippable) - opArgsTrans.emplace_back(trans_output.GetKernelArg()); - - trans_input_size = trans_input_skippable ? 0 : trans_input.GetOutputTensorSize(); - trans_weight_size = trans_weight_skippable ? 0 : trans_weight.GetOutputTensorSize(); - trans_output_size = trans_output_skippable ? 0 : trans_output.GetOutputTensorSize(); - - int idx = 0; - if(!trans_input_skippable) - trans_input_idx = idx++; - if(!trans_weight_skippable) - trans_weight_idx = idx++; - if(!trans_output_skippable) - trans_output_idx = idx++; - } - - const size_t cast_size = need_cast ? miopen::GetTypeSize(miopenFloat) * n * c * hi * wi : 0; - - MultiBufferWorkspaceTraits wt( - {trans_input_size, trans_weight_size, trans_output_size, cast_size}); - - trans_input_offset = wt.GetOffset(0); - trans_weight_offset = wt.GetOffset(1); - trans_output_offset = wt.GetOffset(2); - - const size_t cast_offset = wt.GetOffset(3); - - const int kID_trans_start = isGfx90aFp16altSupport ? 2 : 1; - - const TensorDescriptor cast_desc( - miopenFloat, problem.GetOut().GetLengths(), problem.GetOut().GetStrides()); - auto null_buf = shared{}; - - return [=](const std::vector& kernels) mutable { - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) mutable { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - const auto& workSpace = data_ctx.workSpace; - const auto ker = - handle.Run(kernels[(isGfx90aFp16altSupport && data_ctx.gfx90aFp16alt) ? 1 : 0]); - float elapsed = 0; - - auto trans_input_buf = - trans_input_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_input_offset, trans_input_size); - auto trans_weight_buf = - trans_weight_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_weight_offset, trans_weight_size); - auto trans_output_buf = - trans_output_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, trans_output_offset, trans_output_size); - auto cast_buf = cast_size == 0 - ? null_buf - : handle.CreateSubBuffer(workSpace, cast_offset, cast_size); - - if(need_set_zero) - { - auto zero_buf = need_cast - ? cast_buf.get() - : ((is_nchw && !trans_input_skippable) ? trans_input_buf.get() - : tensors.out); - auto& zero_desc = - need_cast - ? cast_desc - : tensors.outDesc; // use the same desc for NCHW/NHWC for this dense tensor - float zero = 0.f; - - SetTensor(handle, zero_desc, zero_buf, &zero); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - - if(is_nchw) - { - if(!trans_output_skippable) - { - auto& karg_output = opArgsTrans[trans_output_idx]; - karg_output[0] = OpKernelArg(trans_output_buf.get()); - karg_output[1] = OpKernelArg(tensors.in); - handle.Run(kernels[kID_trans_start + trans_output_idx])(karg_output); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - if(!trans_weight_skippable) - { - auto& karg_weight = opArgsTrans[trans_weight_idx]; - karg_weight[0] = OpKernelArg(trans_weight_buf.get()); - karg_weight[1] = OpKernelArg(tensors.w); - handle.Run(kernels[kID_trans_start + trans_weight_idx])(karg_weight); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - } - - opArgs[0] = need_cast ? OpKernelArg(cast_buf.get()) - : ((is_nchw && !trans_input_skippable) - ? OpKernelArg(trans_input_buf.get()) - : OpKernelArg(tensors.out)); - opArgs[1] = (is_nchw && !trans_weight_skippable) ? OpKernelArg(trans_weight_buf.get()) - : OpKernelArg(tensors.w); - opArgs[2] = (is_nchw && !trans_output_skippable) ? OpKernelArg(trans_output_buf.get()) - : OpKernelArg(tensors.in); - - ker(opArgs); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - - if(need_cast) - { - CastTensor(handle, - &lowp_quant, - false, - cast_desc, - cast_buf.get(), - tensors.outDesc, - (is_nchw && !trans_input_skippable) ? trans_input_buf.get() - : tensors.out, - 0, - 0); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - if((is_nchw && !trans_input_skippable)) - { - auto& karg_input = opArgsTrans[trans_input_idx]; - karg_input[0] = OpKernelArg(tensors.out); - karg_input[1] = OpKernelArg(trans_input_buf.get()); - handle.Run(kernels[kID_trans_start + trans_input_idx])(karg_input); - if(handle.IsProfilingEnabled()) - elapsed += handle.GetKernelTime(); - } - - if(handle.IsProfilingEnabled()) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} - -InvokerFactory MakeImplGemmDynamicForwardDlopsNCHWCInvokerFactory( - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC& config) -{ - int hi = problem.GetInHeight(); - int wi = problem.GetInWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetOutChannels() * config.vector_c; - int c = problem.GetInChannels(); - int ks = 1; - int ho = problem.GetOutHeight(); - int wo = problem.GetOutWidth(); - int stride_h = problem.GetKernelStrideH(); - int stride_w = problem.GetKernelStrideW(); - int dilation_h = problem.GetDilationH(); - int dilation_w = problem.GetDilationW(); - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int group = problem.GetGroupCount(); - - // Currentlly we do not tile in H/W dimension, using tile H/W as Ho/Wo, Thus Number of Tile - // equal to one - uint32_t upper_bound_h = 0xffff; // 16bit - uint32_t upper_bound_w = 0xffff; // 16bit - uint32_t tile_h = - igemm_find_tile_size_with_upper_bound(ho, upper_bound_h, stride_h, dilation_h, y); - uint32_t tile_w = - igemm_find_tile_size_with_upper_bound(wo, upper_bound_w, stride_w, dilation_w, x); - uint32_t ntile_h = 1; - uint32_t ntile_w = 1; - if(tile_h != 0 && tile_w != 0) - { - ntile_h = (ho + tile_h - 1) / tile_h; - ntile_w = (wo + tile_w - 1) / tile_w; - } - else - MIOPEN_THROW("tile_hw should not be zero"); - - int tile_hw = (tile_h << 16) | tile_w; - int ntile_hw = (ntile_h << 16) | ntile_w; - // Split K make no sense in vector format - int stride_hw = (stride_h << 16) | stride_w; - int dilation_hw = (dilation_h << 16) | dilation_w; - int pad_hw = (pad_h << 16) | pad_w; - int wei_hw = (y << 16) | x; - // Initialize here for better readibility - uint32_t s_move_slice_k_y = (config.gemm_k_per_block / config.vector_c / x) % y; - uint32_t s_move_slice_k_x = config.gemm_k_per_block / config.vector_c % x; - uint32_t s_move_slice_k_c = (config.gemm_k_per_block / config.vector_c / (x * y)) % (c / group); - int move_slice_k = (s_move_slice_k_y << 16) | (s_move_slice_k_x << 8) | s_move_slice_k_c; - - int splits_4G = solver::igemm_split_batch_size( - hi, wi, ho, wo, n, k, c, miopen::GetTypeSize(problem.GetInDataType())); - splits_4G = (splits_4G == 0 ? n : splits_4G); - uint32_t gemm_n = 1; - uint32_t gemm_m = 1; - if(splits_4G != 0) - { - gemm_n = (n / splits_4G) * tile_h * tile_w; - gemm_m = k / group; - } - else - MIOPEN_THROW("splits_4G should not be zero"); - magic_div_u32_t mdiv_0, mdiv_1, mdiv_2, mdiv_3, mdiv_4, mdiv_5, mdiv_6, mdiv_7; - uint32_t shift_pack_0, shift_pack_1; - - mdiv_0 = magic_div_u32_gen((gemm_n + config.gemm_n_per_block - 1) / config.gemm_n_per_block); - mdiv_1 = magic_div_u32_gen((gemm_m + config.gemm_m_per_block - 1) / config.gemm_m_per_block); - mdiv_2 = magic_div_u32_gen(tile_h); - mdiv_3 = magic_div_u32_gen(tile_w); - mdiv_4 = magic_div_u32_gen(y); - mdiv_5 = magic_div_u32_gen(x); - mdiv_6 = magic_div_u32_gen(ntile_h); - mdiv_7 = magic_div_u32_gen(ntile_w); - shift_pack_0 = magic_div_u32_pack_shift(mdiv_0.shift, mdiv_1.shift, mdiv_2.shift, mdiv_3.shift); - shift_pack_1 = magic_div_u32_pack_shift(mdiv_4.shift, mdiv_5.shift, mdiv_6.shift, mdiv_7.shift); - - std::vector opArgs; - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(tile_hw); - opArgs.emplace_back(ntile_hw); - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n / splits_4G); - opArgs.emplace_back(k / group); - opArgs.emplace_back(c / group); - opArgs.emplace_back(group); - opArgs.emplace_back(ks); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_hw); - opArgs.emplace_back(dilation_hw); - opArgs.emplace_back(pad_hw); - opArgs.emplace_back(wei_hw); - opArgs.emplace_back(move_slice_k); - opArgs.emplace_back(mdiv_0.magic); - opArgs.emplace_back(mdiv_1.magic); - opArgs.emplace_back(mdiv_2.magic); - opArgs.emplace_back(mdiv_3.magic); - opArgs.emplace_back(mdiv_4.magic); - opArgs.emplace_back(mdiv_5.magic); - opArgs.emplace_back(mdiv_6.magic); - opArgs.emplace_back(mdiv_7.magic); - opArgs.emplace_back(shift_pack_0); - opArgs.emplace_back(shift_pack_1); - - return [=](const std::vector& kernels) mutable { - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) mutable { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - const auto ker = handle.Run(kernels[0]); - - opArgs[0] = OpKernelArg(tensors.in); - opArgs[1] = OpKernelArg(tensors.w); - opArgs[2] = OpKernelArg(tensors.out); - ker(opArgs); - - if(handle.IsProfilingEnabled()) - { - float elapsed = 0; - elapsed += handle.GetKernelTime(); - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed); - } - }; - }; -} -} // namespace conv -} // namespace miopen diff --git a/src/conv/problem_description.cpp b/src/conv/problem_description.cpp index 46250ca19c..e69de29bb2 100644 --- a/src/conv/problem_description.cpp +++ b/src/conv/problem_description.cpp @@ -1,286 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include - -#include - -namespace miopen { - -std::string -EncodeDataTypesForKey(miopenDataType_t in, miopenDataType_t weights, miopenDataType_t out) -{ - if(in == weights && in == out) - return GetDataTypeName(in); - return GetDataTypeName(in) + GetDataTypeName(weights) + GetDataTypeName(out); -} - -namespace conv { -namespace { - -std::function -PrintDHW(char sep, unsigned spatial_dims, int64_t depth, int64_t height, int64_t width) -{ - return [=](std::ostream& stream) { - if(spatial_dims > 2) - stream << depth << sep; - stream << height << sep << width; - }; -} - -std::ostream& operator<<(std::ostream& stream, std::function&& manipulator) -{ - manipulator(stream); - return stream; -} - -} // namespace - -miopenAlphaBetaCase_t ClassifyAlphaBeta(const Scalar& alpha, const Scalar& beta) -{ - // double since we are comparing - double alpha_val = alpha.GetAsDouble(); - double beta_val = beta.GetAsDouble(); - - bool alpha_one = (alpha_val == 1.0); - bool alpha_zero = (alpha_val == 0.0); - bool beta_zero = (beta_val == 0.0); - - if(alpha_one && beta_zero) - { - return DEFAULT; - } - - if(!alpha_one && beta_zero) - { - return SCALE; - } - - if(!alpha_zero && !beta_zero) - { - return BILINEAR; - } - - return ERROR_STATE; -} - -std::string ProblemDescription::GetDirectionStr() const -{ - std::string s; - - switch(GetDirection()) - { - case Direction::Forward: s = "F"; break; - case Direction::BackwardData: s = "B"; break; - case Direction::BackwardWeights: s = "W"; break; - default: assert(false); - } - - return s; -} - -std::string ProblemDescription::GetAlphaBetaCaseStr() const -{ - switch(GetAlphaBetaCase()) - { - case BILINEAR: return "Bilinear"; - case SCALE: return "Scale"; - case DEFAULT: return "Default"; - default: MIOPEN_THROW(miopenStatusInvalidValue, "Alpha Beta Case in ERROR_STATE"); - } -} - -void ProblemDescription::MakeNetworkConfig(std::string& conf_key) const -{ - std::ostringstream ss; - - ss << GetInChannels(); - ss << 'x' << PrintDHW('x', GetSpatialDims(), GetInDepth(), GetInHeight(), GetInWidth()); - ss << 'x' - << PrintDHW('x', GetSpatialDims(), GetWeightsDepth(), GetWeightsHeight(), GetWeightsWidth()); - ss << 'x' << GetOutChannels(); - ss << 'x' << PrintDHW('x', GetSpatialDims(), GetOutDepth(), GetOutHeight(), GetOutWidth()); - ss << 'x' << GetInBatchSize(); - if((GetInLayout() == "NCHW" && GetWeightsLayout() == "NCHW" && GetOutLayout() == "NCHW") || - (GetInLayout() == "NCDHW" && GetWeightsLayout() == "NCDHW" && GetOutLayout() == "NCDHW")) - { - ss << 'x' << GetInLayout(); - } - else - { - ss << 'x' << GetInLayout(); - ss << 'x' << GetWeightsLayout(); - ss << 'x' << GetOutLayout(); - } - ss << 'x' << EncodeDataTypesForKey(GetInDataType(), GetWeightsDataType(), GetOutDataType()); - - std::ostringstream optional; - if(const auto ct = GetInCastType()) - optional << "ci" << GetDataTypeName(*ct); - if(const auto ct = GetWeightsCastType()) - optional << "cw" << GetDataTypeName(*ct); - if(const auto ct = GetOutCastType()) - optional << "co" << GetDataTypeName(*ct); - if(!optional.str().empty()) - { - ss << 'x' << optional.str(); - } - - ss << 'x' << PrintDHW('x', GetSpatialDims(), GetPadD(), GetPadH(), GetPadW()); - ss << 'x' - << PrintDHW( - 'x', GetSpatialDims(), GetKernelStrideD(), GetKernelStrideH(), GetKernelStrideW()); - ss << 'x' << PrintDHW('x', GetSpatialDims(), GetDilationD(), GetDilationH(), GetDilationW()); - ss << 'x' << GetGroupCount(); - ss << 'x' << GetDirectionStr(); - ss << 'x' << GetAlphaBetaCaseStr(); - - conf_key = ss.str(); -} - -void ProblemDescription::Serialize(std::ostream& stream) const -{ - const auto sep = '-'; - // Problem description with default layout - // 576-4-4-1x1-192-4-4-8-1x1-2x2-3x3-0-NCHW-FP32-F - // Problem description with non-default layout - // 576-4-4-1x1-192-4-4-8-1x1-2x2-3x3-0-NHWC-NCHW-NCHW-FP32-F - // clang-format off - stream << GetInChannels(); - stream << sep << PrintDHW(sep, GetSpatialDims(), GetInDepth(), GetInHeight(), GetInWidth()); - stream << sep << PrintDHW('x', GetSpatialDims(), GetWeightsDepth(), GetWeightsHeight(), GetWeightsWidth()); - stream << sep << GetOutChannels(); - stream << sep << PrintDHW(sep, GetSpatialDims(), GetOutDepth(), GetOutHeight(), GetOutWidth()); - stream << sep << GetInBatchSize(); - stream << sep << PrintDHW('x', GetSpatialDims(), GetPadD(), GetPadH(), GetPadW()); - stream << sep << PrintDHW('x', GetSpatialDims(), GetKernelStrideD(), GetKernelStrideH(), GetKernelStrideW()); - stream << sep << PrintDHW('x', GetSpatialDims(), GetDilationD(), GetDilationH(), GetDilationW()); - stream << sep << GetBias(); - if ((GetInLayout() == "NCHW" && GetWeightsLayout() == "NCHW" && GetOutLayout() == "NCHW") - || (GetInLayout() == "NCDHW" && GetWeightsLayout() == "NCDHW" && GetOutLayout() == "NCDHW")) - { - stream << sep << GetInLayout(); - } else { - stream << sep << GetInLayout(); - stream << sep << GetWeightsLayout(); - stream << sep << GetOutLayout(); - } - stream << sep << EncodeDataTypesForKey(GetInDataType(), GetWeightsDataType(), GetOutDataType()); - stream << sep << GetDirectionStr(); - - // clang-format on - // New performance config entries shall come into variable/optional part of db key. - // This is to support backward compatibility with previous versions of databases. - std::ostringstream optional; - { - // Group count > 1 identifies Group/Depthwise modes. - if(GetGroupCount() != 1) - optional << "_g" << GetGroupCount(); - - if(const auto ct = GetInCastType()) - optional << "_ci" << GetDataTypeName(*ct); - if(const auto ct = GetWeightsCastType()) - optional << "_cw" << GetDataTypeName(*ct); - if(const auto ct = GetOutCastType()) - optional << "_co" << GetDataTypeName(*ct); - } - if(!optional.str().empty()) - { - stream << optional.str(); - } -} - -bool ProblemDescription::IsLayoutDefault() const -{ - if(GetSpatialDims() == 2) - { - return (in_layout == "NCHW") && (out_layout == "NCHW") && (weights_layout == "NCHW"); - } - else - { - return (in_layout == "NCDHW") && (out_layout == "NCDHW") && (weights_layout == "NCDHW"); - } -} - -bool ProblemDescription::IsLayoutNHWC() const -{ - if(GetSpatialDims() == 2) - { - return (in_layout == "NHWC") && (out_layout == "NHWC") && (weights_layout == "NHWC"); - } - else - { - return (in_layout == "NDHWC") && (out_layout == "NDHWC") && (weights_layout == "NDHWC"); - } -} - -bool ProblemDescription::IsLayoutNCHWc() const -{ - return GetSpatialDims() == 2 && (IsNCHWc_NCHWc() || IsNCHWc_CHWNc()); -} - -bool ProblemDescription::IsNCHWc_NCHWc() const -{ - return GetInLayout() == "NCHWc" && GetWeightsLayout() == "NCHWc" && GetOutLayout() == "NCHWc"; -} - -bool ProblemDescription::IsNCHWc_CHWNc() const -{ - return GetInLayout() == "NCHWc" && GetWeightsLayout() == "CHWNc" && GetOutLayout() == "NCHWc"; -} - -void ProblemDescription::SetupFloats(ExecutionContext& ctx) const -{ - if(IsFp32() || IsFp16() || IsBfp16() || IsInt8() || IsFp8() || IsBfp8()) - { - ctx.general_compile_options += GetDataTypeKernelParams(GetInDataType()); - return; - } - - MIOPEN_LOG_W("Unsupported data types configuration: " - << GetDataTypeName(GetInDataType()) << "x" << GetDataTypeName(GetWeightsDataType()) - << "x" << GetDataTypeName(GetOutDataType())); -} - -std::string ProblemDescription::ComputeLayout(const TensorDescriptor& td) const -{ - return td.GetLayout_str(); -} - -std::string ProblemDescription::ComputeInLayout() const { return ComputeLayout(in); } - -std::string ProblemDescription::ComputeOutLayout() const { return ComputeLayout(out); } - -std::string ProblemDescription::ComputeWeightsLayout() const { return ComputeLayout(weights); } - -} // namespace conv -} // namespace miopen diff --git a/src/conv/solver_finders.cpp b/src/conv/solver_finders.cpp index e79df1b723..e69de29bb2 100644 --- a/src/conv/solver_finders.cpp +++ b/src/conv/solver_finders.cpp @@ -1,435 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include -#include - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_GEMM) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_DIRECT) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_WINOGRAD) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_FFT) -MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DEVICE_ARCH) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_COMPILE_ONLY) - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_FIND_CONV_INSUFFICIENT_WORKSPACE_ALLOW_FINDDB_UPDATE) - -namespace miopen { - -namespace conv { -namespace { - -class DirectSolverFinder : public SolversFinderMixin -{ -protected: - AlgorithmName GetAlgorithmName(const ProblemDescription& problem) const override - { - return AlgorithmName{ConvolutionAlgoToDirectionalString(miopenConvolutionAlgoDirect, - problem.GetDirection())}; - } - - bool IsEnabled(const ExecutionContext& /*ctx*/, - const ProblemDescription& /*problem*/, - const ConvFindParameters& parameters) const override - { - return !parameters.use_winograd_only && !env::disabled(MIOPEN_DEBUG_CONV_DIRECT); - } - - std::vector FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const ConvFindParameters&, - const std::optional&) const override - { - /// \todo: actually use FindOptions - return problem.GetDirection() != conv::Direction::BackwardWeights - ? FindAllDirectSolutions(ctx, problem, invoke_ctx) - : FindAllBwdWrW2DSolutions(ctx, problem, invoke_ctx); - } -}; - -class ImplicitGemmSolverFinder : public SolversFinderMixin -{ -protected: - AlgorithmName GetAlgorithmName(const ProblemDescription& problem) const override - { - return AlgorithmName{ConvolutionAlgoToDirectionalString(miopenConvolutionAlgoImplicitGEMM, - problem.GetDirection())}; - } - - bool IsEnabled(const ExecutionContext& /*ctx*/, - const ProblemDescription& /*problem*/, - const ConvFindParameters& parameters) const override - { - return !parameters.use_winograd_only && !env::disabled(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM); - } - - std::vector FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const ConvFindParameters&, - const std::optional&) const override - { - /// \todo: actually use FindOptions - return problem.GetDirection() != conv::Direction::BackwardWeights - ? FindAllImplicitGemmSolutions(ctx, problem, invoke_ctx) - : FindImplicitGemmWrWAllSolutions(ctx, problem, invoke_ctx); - } -}; - -class FftSolverFinder : public SolversFinderMixin -{ -protected: - AlgorithmName GetAlgorithmName(const ProblemDescription& problem) const override - { - return AlgorithmName{ - ConvolutionAlgoToDirectionalString(miopenConvolutionAlgoFFT, problem.GetDirection())}; - } - - bool IsEnabled(const ExecutionContext& /*ctx*/, - const ProblemDescription& problem, - const ConvFindParameters& parameters) const override - { - return !parameters.use_winograd_only && - problem.GetDirection() != conv::Direction::BackwardWeights && - !env::disabled(MIOPEN_DEBUG_CONV_FFT); - } - - std::vector FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const ConvFindParameters&, - const std::optional&) const override - { - /// \todo: actually use FindOptions - return FindAllFFTSolutions(ctx, problem, invoke_ctx); - } -}; - -class GemmSolverFinder : public SolversFinderMixin -{ -protected: - AlgorithmName GetAlgorithmName(const ProblemDescription& problem) const override - { - return AlgorithmName{ - ConvolutionAlgoToDirectionalString(miopenConvolutionAlgoGEMM, problem.GetDirection())}; - } - - bool IsEnabled(const ExecutionContext& /*ctx*/, - const ProblemDescription& /*problem*/, - const ConvFindParameters& parameters) const override - { - return !parameters.use_winograd_only && !env::disabled(MIOPEN_DEBUG_CONV_GEMM); - } - - std::vector FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const ConvFindParameters&, - const std::optional&) const override - { - /// \todo: actually use FindOptions - return FindAllGemmSolutions(ctx, problem, invoke_ctx); - } -}; - -class WinogradSolverFinder : public SolversFinderMixin -{ -protected: - AlgorithmName GetAlgorithmName(const ProblemDescription& problem) const override - { - return AlgorithmName{ConvolutionAlgoToDirectionalString(miopenConvolutionAlgoWinograd, - problem.GetDirection())}; - } - - bool IsEnabled(const ExecutionContext& /*ctx*/, - const ProblemDescription& /*problem*/, - const ConvFindParameters& /*parameters*/) const override - { - return !env::disabled(MIOPEN_DEBUG_CONV_WINOGRAD); - } - - std::vector FindImpl(const ExecutionContext& ctx, - const ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - const ConvFindParameters& parameters, - const std::optional&) const override - { - /// \todo: actually use FindOptions - auto ctx_copy = ctx; - if(parameters.use_winograd_only) - ctx_copy.use_dynamic_solutions_only = true; - return problem.GetDirection() != conv::Direction::BackwardWeights - ? FindAllWinogradSolutions(ctx_copy, problem, invoke_ctx) - : FindWinogradWrWAllSolutions(ctx_copy, problem, invoke_ctx); - } -}; - -} // namespace - -const std::vector>& GetConvSolverFinders() -{ - static const auto finders = []() { - auto tmp = std::vector>{}; - tmp.emplace_back(std::make_unique()); - tmp.emplace_back(std::make_unique()); - tmp.emplace_back(std::make_unique()); - tmp.emplace_back(std::make_unique()); - tmp.emplace_back(std::make_unique()); - return tmp; - }(); - - return finders; -} - -} // namespace conv - -/// Register invoker only for the best solution within algorithm. -static std::vector EvaluateInvokers(Handle& handle, - const std::vector& solutions, - const AlgorithmName& algorithm_name, - const NetworkConfig& network_config, - const AnyInvokeParams& invoke_ctx, - bool& is_result_optimal, - bool force_attach_binary) -{ - const auto arch = env::value(MIOPEN_DEVICE_ARCH); - if(!arch.empty()) - return {}; - - auto selected = miopen::solver::ConvSolution{miopenStatusUnknownError}; - auto best = std::numeric_limits::max(); - auto best_invoker = Invoker{}; - auto ret = std::vector{}; - - for(const auto& sol : solutions) - { - if(!conv::IsEnoughWorkspace( - "EvaluateInvokers", solver::Id{sol.solver_id}, sol.workspace_sz, &invoke_ctx)) - { - // Providing smaller workspace may result in the selection of a slow convolution - // algorithm, and therefore affect library performance. Moreover, sub-optimal data may - // be cached in the user's find-db. This means that the performance drop will become - // persistent, i.e. even providing sufficient workspace won't restore the performance. - // To get rid of this problem, the user will need to either remove the user's find-db, - // or repeat miopenFindConvolution*() with affected convolution configs in Normal Find - // Mode (the latter will overwrite sub-optimal user's find-db records). - // - // That is why we do not write sub-optimal results into persistent find-db (on disk) - // unless this is explicitly enabled via environment setting. - if(!env::enabled(MIOPEN_FIND_CONV_INSUFFICIENT_WORKSPACE_ALLOW_FINDDB_UPDATE)) - is_result_optimal = false; - continue; - } - - if(!sol.invoker_factory) - MIOPEN_THROW("Invoker is not provided by solver " + sol.solver_id); - - std::vector programs; - const auto invoker = handle.PrepareInvoker(*sol.invoker_factory, - sol.construction_params, - force_attach_binary ? &programs : nullptr); - - try - { - // Run invoker max 6 times, with ~5 sec time limit. - using elapsed_t = decltype(handle.GetKernelTime()); - constexpr elapsed_t TIME_MS_MAX = 5000.0; - constexpr int N_RUNS_MAX = 8; - constexpr int N_RUNS_DISCARD = 3; - auto elapsed = static_cast(0); - auto first_elapsed = static_cast(0); - int i = 0; - while(i < N_RUNS_MAX && elapsed < TIME_MS_MAX) - { - invoker(handle, invoke_ctx); - elapsed += handle.GetKernelTime(); - if(i < N_RUNS_DISCARD) - first_elapsed += elapsed; - ++i; - } - // If the execution time was not too long, - // then the 1st run is not counted (assume it's warm-up): - if(i > 1) - elapsed = (elapsed - first_elapsed) / static_cast(i - N_RUNS_DISCARD); - - MIOPEN_LOG_I(sol << ": " << elapsed << (elapsed < best ? " < " : " >= ") << best); - if(elapsed < best) - { - best = elapsed; - selected = sol; - best_invoker = invoker; - } - - auto solution = Solution{solver::Id{selected.solver_id}, best, selected.workspace_sz}; - if(force_attach_binary) - solution.SetInvoker(invoker, programs, selected.construction_params); - else - solution.SetInvoker(invoker, {}, {}); - ret.emplace_back(std::move(solution)); - } - catch(const miopen::Exception& ex) - { - MIOPEN_LOG_E(ex.what()); - } - } - - if(!selected.Succeeded()) - return {}; - - handle.RegisterInvoker(best_invoker, network_config, selected.solver_id, algorithm_name); - MIOPEN_LOG_I("Selected: " << selected << ": " << best - << ", workspace_sz = " << selected.workspace_sz); - - return ret; -} - -FindCoreResult FindCore(const AnyInvokeParams& invoke_ctx, - const ExecutionContext& ctx, - const ProblemDescriptionBase& problem, - const PrimitiveFindParameters& parameters, - const std::vector>& finders, - const std::optional& options, - bool force_attach_binary) -{ - auto& handle = ctx.GetStream(); - - // Find - auto solutions = std::map>{}; - std::transform( - finders.begin(), finders.end(), std::inserter(solutions, solutions.end()), [&](auto&& f) { - return std::make_pair(f->GetAlgorithmName(problem), - f->Find(ctx, problem, invoke_ctx, parameters, options)); - }); - - std::size_t total = 0; - - for(auto it = solutions.begin(); it != solutions.end();) - { - if(it->second.empty()) - { - it = solutions.erase(it); - continue; - } - - total += it->second.size(); - ++it; - } - - // Precompile - { - auto all = std::vector{}; - all.reserve(total); - for(const auto& ss : solutions) - std::transform(ss.second.begin(), - ss.second.end(), - std::back_inserter(all), - [](auto&& s) { return &s; }); - PrecompileSolutions(handle, all, force_attach_binary); - } - - if(env::enabled((MIOPEN_DEBUG_COMPILE_ONLY))) - MIOPEN_THROW( - miopenStatusGpuOperationsSkipped, - "MIOPEN_DEBUG_COMPILE_ONLY is enabled, escaping forward convolution. Search skipped."); - - // Evaluate Invokers - AutoEnableProfiling enableProfiling{handle}; - const auto network_config = problem.MakeNetworkConfig(); - auto ret = FindCoreResult(); - ret.is_optimal = true; - - ret.solutions.reserve(total); - - for(const auto& ss : solutions) - { - auto evaluated = EvaluateInvokers(handle, - ss.second, - ss.first, - network_config, - invoke_ctx, - ret.is_optimal, - force_attach_binary); - - ret.solutions.insert(ret.solutions.end(), - std::make_move_iterator(evaluated.begin()), - std::make_move_iterator(evaluated.end())); - } - - return ret; -} - -namespace conv { - -bool IsAlgorithmDisabled(miopenConvAlgorithm_t algo) -{ - switch(algo) - { // clang-format off -#if MIOPEN_USE_GEMM - case miopenConvolutionAlgoGEMM: - return env::disabled(MIOPEN_DEBUG_CONV_GEMM); -#endif - case miopenConvolutionAlgoDirect: - return env::disabled(MIOPEN_DEBUG_CONV_DIRECT); - case miopenConvolutionAlgoFFT: - return env::disabled(MIOPEN_DEBUG_CONV_FFT); - case miopenConvolutionAlgoWinograd: - return env::disabled(MIOPEN_DEBUG_CONV_WINOGRAD); - case miopenConvolutionAlgoImplicitGEMM: - return env::disabled(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM); - default: // Disable future algos by default to enforce explicit handling: - return true; - } // clang-format on -} - -bool IsEnoughWorkspace(std::string_view where, - const miopen::solver::Id& solver_id, - const std::size_t required_size, - const miopen::AnyInvokeParams* const invokeParams) -{ - if(invokeParams != nullptr && required_size > 0) - { - const auto provided_size = invokeParams->GetWorkspaceSize(); - const auto provided_ptr = invokeParams->GetWorkspace(); - if(provided_ptr == nullptr || provided_size < required_size) - { - MIOPEN_LOG_W("[" << where << "] Solver <" << solver_id.ToString() << ">" - << ", workspace required: " << required_size - << ", provided ptr: " << provided_ptr << " size: " << provided_size); - return false; - } - } - return true; -} - -} // namespace conv -} // namespace miopen diff --git a/src/convolution.cpp b/src/convolution.cpp index 51d94ece7b..e69de29bb2 100644 --- a/src/convolution.cpp +++ b/src/convolution.cpp @@ -1,603 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include -#include - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_DIRECT) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_WINOGRAD) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_GEMM) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_FFT) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK) - -namespace miopen { - -namespace { - -std::size_t GetMaxWorkSpaceSize(const std::vector>& values) -{ - std::size_t sz = 0; - for(const auto& pr : values) - { - if(sz < pr.second) - { - MIOPEN_LOG_I2(sz << " < " << pr.second); - sz = pr.second; - } - } - return sz; -} - -std::size_t GetWorkSpaceSizeGEMM(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ -#if MIOPEN_USE_GEMM - if(env::disabled(MIOPEN_DEBUG_CONV_GEMM) || - miopen::any_of(problem.GetConv().GetConvDilations(), [](auto v) { return v > 1; })) - return 0; - - return GetMaxWorkSpaceSize(AllGemmWorkspaceSize(ctx, problem)); -#else - std::ignore = ctx; - std::ignore = problem; - return 0; -#endif -} - -std::size_t GetWorkSpaceSizeImplicitGemm(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM)) - return 0; - return GetMaxWorkSpaceSize(FindAllImplicitGemmWorkspaceSizes(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeDirect(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_DIRECT)) - return 0; - return GetMaxWorkSpaceSize(AllDirectForwardBackwardDataWorkspaceSize(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeFFT(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_FFT)) - return 0; - return GetMaxWorkSpaceSize(AllFFTForwardBackwardDataWorkspaceSize(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeWinograd(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_WINOGRAD)) - return 0; - return GetMaxWorkSpaceSize(FindAllWinogradWorkspaceSizes(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeDirectWrW(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_DIRECT)) - return 0; - return GetMaxWorkSpaceSize(AllDirectBwdWrW2DWorkspaceSize(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeWinogradWrW(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_WINOGRAD)) - return 0; - return GetMaxWorkSpaceSize(FindWinogradWrWWorkspaceSizes(ctx, problem)); -} - -std::size_t GetWorkSpaceSizeImplicitGemmWrW(const miopen::ExecutionContext& ctx, - const conv::ProblemDescription& problem) -{ - if(env::disabled(MIOPEN_DEBUG_CONV_IMPLICIT_GEMM)) - return 0; - return GetMaxWorkSpaceSize(FindImplicitGemmWrWWorkspaceSizes(ctx, problem)); -} - -} // namespace - -ConvolutionDescriptor::ConvolutionDescriptor(std::size_t spatial_dim, - miopenConvolutionMode_t c_mode, - miopenPaddingMode_t p_mode, - const std::vector& p_pads, - const std::vector& p_strides, - const std::vector& p_dilations, - const std::vector& p_trans_output_pads, - int p_group_count, - float p_lowp_quant) - : spatialDim(spatial_dim), - mode(c_mode), - paddingMode(p_mode), - pads(p_pads), - strides(p_strides), - dilations(p_dilations), - trans_output_pads(p_trans_output_pads), - group_count(p_group_count), - lowp_quant(p_lowp_quant) -{ - if(pads.size() != spatial_dim || strides.size() != spatial_dim || - dilations.size() != spatial_dim || trans_output_pads.size() != spatial_dim || - miopen::any_of(pads, [](auto v) { return v < 0; }) || - miopen::any_of(strides, [](auto v) { return v < 1; }) || - miopen::any_of(dilations, [](auto v) { return v < 1; })) - { - MIOPEN_THROW(miopenStatusBadParm, - "Invalid parameters, check usage. MIOPEN expects padding " - ">= 0, stride >= 1, dilation >= 1 and the same dilation " - "factor for horizontal and vertical direction"); - } - if(!(mode == miopenConvolution || mode == miopenTranspose)) - { - if(mode == miopenGroupConv || mode == miopenDepthwise) - { - mode = miopenConvolution; - } - else - { - MIOPEN_THROW(miopenStatusBadParm, "Convolution mode not supported"); - } - } - if(!(paddingMode == miopenPaddingSame || paddingMode == miopenPaddingValid || - paddingMode == miopenPaddingDefault)) - { - MIOPEN_THROW(miopenStatusBadParm, "Padding mode not supported"); - } -} - -ConvolutionDescriptor::ConvolutionDescriptor(const std::vector& p_pads, - const std::vector& p_strides, - const std::vector& p_dilations, - const std::vector& p_trans_output_pads, - int p_group_count, - float p_lowp_quant) - : ConvolutionDescriptor{p_pads.size(), - miopenConvolution, - miopenPaddingDefault, - p_pads, - p_strides, - p_dilations, - p_trans_output_pads, - p_group_count, - p_lowp_quant} -{ -} - -std::size_t ConvolutionDescriptor::GetSpatialDimension() const { return spatialDim; } - -const std::vector& ConvolutionDescriptor::GetConvPads() const { return pads; } - -const std::vector& ConvolutionDescriptor::GetConvStrides() const { return strides; } - -const std::vector& ConvolutionDescriptor::GetConvDilations() const { return dilations; } - -const std::vector& ConvolutionDescriptor::GetTransposeConvPads() const -{ - return trans_output_pads; -} - -int ConvolutionDescriptor::GetGroupCount() const { return group_count; } - -TensorDescriptor -ConvolutionDescriptor::GetForwardOutputTensorWithLayout(const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - const std::string& yLayout, - miopenDataType_t yType) const -{ - const std::size_t spatial_dim = GetSpatialDimension(); - - assert(xDesc.GetLengths().size() == spatial_dim + 2); - assert(wDesc.GetLengths().size() == spatial_dim + 2); - - if(xDesc.GetType() != wDesc.GetType()) - { - MIOPEN_THROW(miopenStatusBadParm, "Types do not match for the filter"); - } - - std::size_t in_n, in_c; - std::tie(in_n, in_c) = miopen::tie_pick<0, 1>{}(xDesc.GetLengths()); - - auto in_spatial = boost::adaptors::slice(xDesc.GetLengths(), 2, 2 + spatial_dim); - - std::size_t wei_k, wei_c; - std::tie(wei_k, wei_c) = miopen::tie_pick<0, 1>{}(wDesc.GetLengths()); - - auto wei_spatial = boost::adaptors::slice(wDesc.GetLengths(), 2, 2 + spatial_dim); - - if(wDesc.GetLayout_str() == "CHWNc") - { - std::tie(wei_k, wei_c) = miopen::tie_pick<3, 0>{}(wDesc.GetLengths()); - wei_spatial = boost::adaptors::slice(wDesc.GetLengths(), 1, 1 + spatial_dim); - } - - if(mode == miopenConvolution) - { - // for depthwise conv wei_c must be 1 while group_count must be wei_c - if((group_count == 1 && in_c != wei_c) || - (group_count > 1 && (in_c % wei_c != 0 || wei_k % (in_c / wei_c) != 0))) - { - MIOPEN_THROW(miopenStatusBadParm, "Channels do not match for the filter"); - } - } - else if(mode == miopenTranspose) - { - if(in_c != wei_k || (group_count > 1 && (wei_k % group_count != 0))) - { - MIOPEN_THROW(miopenStatusBadParm, "Channels do not match for the filter"); - } - - if(miopen::any_of(boost::combine(GetTransposeConvPads(), GetConvStrides()), [](auto v) { - auto trans_conv_pad = boost::get<0>(v); - auto stride = boost::get<1>(v); - return trans_conv_pad >= stride; - })) - { - MIOPEN_THROW(miopenStatusBadParm, - "Output shape doesn't match due to invalid output padding"); - } - } - - std::size_t out_c = 0; - std::vector out_lens(spatial_dim + 2); - - auto out_spatial = boost::adaptors::slice(out_lens, 2, 2 + spatial_dim); - - if(paddingMode == miopenPaddingSame && mode == miopenConvolution && - miopen::all_of(GetConvDilations(), [](auto v) { return v == 1; })) - { - out_c = wei_k; - - for(int i = 0; i < spatial_dim; ++i) - { - out_spatial[i] = miopen::integer_division_ceil(in_spatial[i], GetConvStrides()[i]); - } - } - else if(paddingMode == miopenPaddingValid && mode == miopenConvolution && - miopen::all_of(GetConvDilations(), [](auto v) { return v == 1; })) - { - out_c = wei_k; - - for(int i = 0; i < spatial_dim; ++i) - { - out_spatial[i] = miopen::integer_division_ceil( - std::ptrdiff_t(in_spatial[i]) - wei_spatial[i] + 1, GetConvStrides()[i]); - } - } - else if(paddingMode == miopenPaddingDefault || paddingMode == miopenPaddingSame || - paddingMode == miopenPaddingValid) - { - if(mode == miopenTranspose) - { - out_c = wei_c * group_count; - - for(int i = 0; i < spatial_dim; ++i) - { - out_spatial[i] = std::max( - 1, - GetConvStrides()[i] * (std::ptrdiff_t(in_spatial[i]) - 1) + 1 + - GetConvDilations()[i] * (std::ptrdiff_t(wei_spatial[i]) - 1) - - 2 * static_cast(GetConvPads()[i]) + - GetTransposeConvPads()[i]); - } - } - else - { - out_c = wei_k / wDesc.GetVectorLength(); - - for(int i = 0; i < spatial_dim; ++i) - { - out_spatial[i] = std::max( - 1, - (ptrdiff_t(in_spatial[i]) - - (1 + GetConvDilations()[i] * (std::ptrdiff_t(wei_spatial[i]) - 1)) + - 2 * static_cast(GetConvPads()[i])) / - GetConvStrides()[i] + - 1); - } - } - } - else - MIOPEN_THROW(miopenStatusInvalidValue, "Invalid Padding Mode!"); - - out_lens[0] = in_n; - out_lens[1] = out_c; - - const std::string default_layout = tensor_layout_get_default(xDesc.GetNumDims()); - std::vector out_strides; - tensor_layout_to_strides( - out_lens, default_layout, yLayout, xDesc.GetVectorLength(), out_strides); - return {(xDesc.GetType() == miopenInt8 - ? (yType) - : xDesc.GetType()), // TODO: This function overrides the output type with - // essentially the input which is incorrect. - xDesc.GetLayout_t(), - out_lens, - out_strides}; -} - -TensorDescriptor ConvolutionDescriptor::GetForwardOutputTensor(const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - miopenDataType_t yType) const -{ - // output layout same as input - const std::string in_layout = xDesc.GetLayout_str(); - return GetForwardOutputTensorWithLayout(xDesc, wDesc, in_layout, yType); -} - -/// There is assumption that if Winograd is applicable and granularity loss is low, then there is no -/// advantage in trying other algorithms as those either slower or use more workspace. This allows -/// for some related host-side optimizations. -/// -/// These optimizations are kind of cutting corners, but advantages are quite high. -bool ConvolutionDescriptor::IsWinograd3x3SupportedAndFast( - const miopen::ExecutionContext& ctx, const conv::ProblemDescription& problem) const -{ - if(env::disabled(MIOPEN_DEBUG_CONV_WINOGRAD)) - return false; - - // Disable this performance optimization when we want to run some specific Solver. - // Other Solvers will be skipped anyway. - if(GetEnvFindOnlySolver()) - return false; - - // Filter out configs where 3x3 Winograd does not have high WTI. - if(!(problem.GetOutChannels() >= 16 && problem.GetOutChannels() % 2 == 0)) - return false; - - return solver::conv::ConvBinWinograd3x3U{}.IsApplicable(ctx, problem); -} - -std::size_t ConvolutionDescriptor::GetWorkSpaceSize(ExecutionContext ctx, - const conv::ProblemDescription& problem) const -{ - MIOPEN_LOG_I2(""); - - ctx.do_search = false; - ctx.disable_perfdb_access = true; - - while(findMode.IsFast(ctx) || findMode.IsHybrid(ctx)) - { - /// \section ffind_gwss_why_not_0 - /// Basically we can return 0 here because - /// * (A) Find() emulated by Immediate mode does not execute kernels. - /// * (B) We expect that applications read output of Find() and - /// allocate WS for Run phase as indicated there - /// (in miopenConvAlgoPerf_t::memory). - /// - /// However there are some known apps that allocate WS once - /// (using size returned by *this* call) and then re-use - /// the same workspace for Run phase. That is why we shall return - /// actually required workspace here. - auto fallback = bool{}; - const auto solutions = GetSolutions(ctx, problem, 1, &fallback); - if(solutions.empty() || ((findMode.IsHybrid(ctx) && fallback) && - !env::enabled(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK))) - { - ctx.use_dynamic_solutions_only = findMode.IsDynamicHybrid(ctx); - break; // Fall down to Normal Find. - } - const auto id = solver::Id{solutions.front().solution_id}; - const auto& s = id.GetSolver(); - const auto workspace_size = s.GetWorkspaceSize(ctx, problem); - - MIOPEN_LOG_I(workspace_size); - return workspace_size; - } - - size_t workspace_size; - - if(problem.GetDirection() != conv::Direction::BackwardWeights) - { - if(IsWinograd3x3SupportedAndFast(ctx, problem)) - { - ctx.use_dynamic_solutions_only = true; - workspace_size = GetWorkSpaceSizeWinograd(ctx, problem); - } - else - { - workspace_size = std::max({GetWorkSpaceSizeFFT(ctx, problem), - GetWorkSpaceSizeGEMM(ctx, problem), - GetWorkSpaceSizeDirect(ctx, problem), - GetWorkSpaceSizeImplicitGemm(ctx, problem), - GetWorkSpaceSizeWinograd(ctx, problem)}); - } - } - else - { - workspace_size = std::max({GetWorkSpaceSizeGEMM(ctx, problem), - GetWorkSpaceSizeDirectWrW(ctx, problem), - GetWorkSpaceSizeImplicitGemmWrW(ctx, problem), - GetWorkSpaceSizeWinogradWrW(ctx, problem)}); - } - - MIOPEN_LOG_I(workspace_size); - return workspace_size; -} - -std::ostream& operator<<(std::ostream& stream, const ConvolutionDescriptor& c) -{ - stream << "conv" << c.spatialDim << "d, "; - MIOPEN_LOG_ENUM(stream, c.mode, miopenConvolution, miopenTranspose) << ", "; - MIOPEN_LOG_ENUM( - stream, c.paddingMode, miopenPaddingDefault, miopenPaddingSame, miopenPaddingValid) - << ", "; - - LogRange(stream << "{", c.GetConvPads(), ", ") << "}, "; - LogRange(stream << "{", c.GetConvStrides(), ", ") << "}, "; - LogRange(stream << "{", c.GetConvDilations(), ", ") << "}, "; - - if(c.group_count > 1) - { - stream << c.group_count << ", "; - } - - if(c.mode == miopenTranspose) - { - LogRange(stream << "{", c.GetTransposeConvPads(), ", ") << "}, "; - } - - return stream; -} - -void to_json(nlohmann::json& json, const ConvolutionAttribute::Gfx90aFp16alt& attribute) -{ - json = {{"value", attribute.value}}; -} - -void from_json(const nlohmann::json& json, ConvolutionAttribute::Gfx90aFp16alt& attribute) -{ - json.at("value").get_to(attribute.value); -} - -void ConvolutionAttribute::Set(miopenConvolutionAttrib_t attr, int value) -{ - if(attr == MIOPEN_CONVOLUTION_ATTRIB_FP16_ALT_IMPL) - { - if(value < -1 || value > 1) - { - MIOPEN_THROW(miopenStatusBadParm, - "[Set conv attribute] Error: Attempt to set invalid value of " - "MIOPEN_CONVOLUTION_ATTRIB_FP16_ALT_IMPL: " + - std::to_string(value)); - } - gfx90aFp16alt.value = value; - } - else if(attr == MIOPEN_CONVOLUTION_ATTRIB_DETERMINISTIC) - { - if(value < 0 || value > 1) - { - MIOPEN_THROW(miopenStatusBadParm, - "[Set conv attribute] Error: Attemp to set invalid value for " - "MIOPEN_CONVOLUTION_ATTRIB_DETERMINISTIC: " + - std::to_string(value)); - } - deterministic.value = value; - } - else if(attr == MIOPEN_CONVOLUTION_ATTRIB_FP8_ROUNDING_MODE) - { - const auto rounding_mode = static_cast(value); - if(rounding_mode != miopenF8RoundingModeStochastic && - rounding_mode != miopenF8RoundingModeStandard) - { - MIOPEN_THROW(miopenStatusBadParm, - "[Set conv attribute] Error: Attempt to set invalid value for " - "MIOPEN_CONVOLUTION_ATTRIB_FP8_ROUNDING_MODE" + - std::to_string(value)); - } - fp8rounding_mode.rounding_mode = rounding_mode; - } - else - { - MIOPEN_THROW(miopenStatusBadParm, - "[Set conv attribute] Error: Attribute [" + - std::to_string(static_cast(attr)) + "] does not exist."); - } -} - -int ConvolutionAttribute::Get(miopenConvolutionAttrib_t attr) const -{ - if(attr == MIOPEN_CONVOLUTION_ATTRIB_FP16_ALT_IMPL) - return gfx90aFp16alt.value; - else if(attr == MIOPEN_CONVOLUTION_ATTRIB_FP8_ROUNDING_MODE) - return static_cast(fp8rounding_mode.rounding_mode); - else if(attr == MIOPEN_CONVOLUTION_ATTRIB_DETERMINISTIC) - return deterministic.value; - MIOPEN_THROW(miopenStatusBadParm, - "[Get conv attribute] Error: Attribute [" + - std::to_string(static_cast(attr)) + "] does not exist."); -} - -void to_json(nlohmann::json& json, const ConvolutionAttribute& conv) -{ - json = {{"gfx90aFp16alt", conv.gfx90aFp16alt}}; -} - -void from_json(const nlohmann::json& json, ConvolutionAttribute& conv) -{ - json.at("gfx90aFp16alt").get_to(conv.gfx90aFp16alt); -} - -void to_json(nlohmann::json& json, const ConvolutionDescriptor& conv) -{ - json = nlohmann::json{ - {"spatialDim", conv.spatialDim}, - {"mode", conv.mode}, - {"paddingMode", conv.paddingMode}, - {"pads", conv.pads}, - {"strides", conv.strides}, - {"dilations", conv.dilations}, - {"transOutputPads", conv.trans_output_pads}, - {"groupCount", conv.group_count}, - {"lowpQuant", conv.lowp_quant}, - {"attribute", conv.attribute}, - }; -} - -void from_json(const nlohmann::json& json, ConvolutionDescriptor& conv) -{ - json.at("spatialDim").get_to(conv.spatialDim); - json.at("mode").get_to(conv.mode); - json.at("paddingMode").get_to(conv.paddingMode); - json.at("pads").get_to(conv.pads); - json.at("strides").get_to(conv.strides); - json.at("dilations").get_to(conv.dilations); - json.at("transOutputPads").get_to(conv.trans_output_pads); - json.at("groupCount").get_to(conv.group_count); - json.at("lowpQuant").get_to(conv.lowp_quant); - json.at("attribute").get_to(conv.attribute); -} - -} // namespace miopen diff --git a/src/cumulative_reduction.cpp b/src/cumulative_reduction.cpp new file mode 100644 index 0000000000..eab11a404f --- /dev/null +++ b/src/cumulative_reduction.cpp @@ -0,0 +1,76 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include +#include +#include + +namespace miopen { + +miopenStatus_t CumulativeReductionForward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputDesc, + Data_t output, + const TensorDescriptor& indicesDesc, + Data_t indices, + const int dim, + const bool exclusive, + const bool reverse, + const miopenCumOp_t cumOp) +{ + const auto problem = cumulative_reduction::ForwardProblemDescription{ + inputDesc, outputDesc, indicesDesc, dim, cumOp}; + + const auto invoke_params = [&]() { + auto tmp = cumulative_reduction::InvokeParams{}; + tmp.type = InvokeType::Run; + tmp.inputDesc = &inputDesc; + tmp.outputDesc = &outputDesc; + tmp.indicesDesc = &indicesDesc; + tmp.input = input; + tmp.output = output; + tmp.indices = indices; + + tmp.dim = dim; + tmp.exclusive = exclusive; + tmp.reverse = reverse; + + return tmp; + }(); + + const auto algo = AlgorithmName{"CumulativeReductionForward"}; + const auto solvers = + solver::SolverContainer{}; + + solvers.ExecutePrimitive(handle, problem, algo, invoke_params); + + return miopenStatusSuccess; +} + +} // namespace miopen diff --git a/src/cumulative_reduction/problem_description.cpp b/src/cumulative_reduction/problem_description.cpp new file mode 100644 index 0000000000..4ef94ef138 --- /dev/null +++ b/src/cumulative_reduction/problem_description.cpp @@ -0,0 +1,72 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace miopen { + +namespace cumulative_reduction { + +bool checkSameLength(const TensorDescriptor& x, const TensorDescriptor& y) +{ + if(x.GetNumDims() != y.GetNumDims()) + return false; + for(int i = 0; i < x.GetNumDims(); ++i) + { + if(x.GetLengths()[i] != y.GetLengths()[i]) + return false; + } + return true; +} + +NetworkConfig ForwardProblemDescription::MakeNetworkConfig() const +{ + auto input_dtype = inputDesc.GetType(); + auto output_dtype = outputDesc.GetType(); + auto size = inputDesc.GetElementSize(); + auto inner_size = inputDesc.GetLengths()[dim]; + auto outer_size = size / inner_size; + + std::ostringstream ss; + + ss << "cum_reduc_fwd"; + ss << "idtype" << input_dtype; + ss << "odtype" << output_dtype; + ss << "outer" << outer_size; + ss << "inner" << inner_size; + ss << "op" << cumOp; + ss << "packed" << IsAllPacked(); + ss << "dimstride1" << IsAllDimStride1(); + + return NetworkConfig{ss.str()}; +} + +} // namespace cumulative_reduction + +} // namespace miopen diff --git a/src/cumulative_reduction_api.cpp b/src/cumulative_reduction_api.cpp new file mode 100644 index 0000000000..d344f6e0a7 --- /dev/null +++ b/src/cumulative_reduction_api.cpp @@ -0,0 +1,108 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include + +static void LogCmdCumulativeReduction(const miopenTensorDescriptor_t inputDesc, + const miopenTensorDescriptor_t outputDesc, + const miopenTensorDescriptor_t indicesDesc, + const int dim, + const bool exclusive, + const bool reverse, + const miopenCumOp_t cumOp, + const bool is_fwd) +{ + if(miopen::IsLoggingCmd()) + { + std::stringstream ss; + auto dtype = miopen::deref(inputDesc).GetType(); + if(dtype == miopenHalf) + { + ss << "cumulative_reductionfp16"; + } + else if(dtype == miopenFloat) + { + ss << "cumulative_reductionfp32"; + } + else if(dtype == miopenBFloat16) + { + ss << "cumulative_reductionbfp16"; + } + + MIOPEN_LOG_FUNCTION(inputDesc, outputDesc, indicesDesc); + ss << " -d " << dim; + ss << " --excl " << exclusive; + ss << " --rev " << reverse; + ss << " --op " << cumOp; + ss << " -F " << ((is_fwd) ? "1" : "2"); + + MIOPEN_LOG_DRIVER_CMD(ss.str()); + } +} + +extern "C" miopenStatus_t +miopenCumulativeReductionForward(miopenHandle_t handle, + const miopenTensorDescriptor_t inputDesc, + const void* input, + const miopenTensorDescriptor_t outputDesc, + void* output, + const miopenTensorDescriptor_t indicesDesc, + void* indices, + const int dim, + const bool exclusive, + const bool reverse, + const miopenCumOp_t cumOp) +{ + MIOPEN_LOG_FUNCTION(handle, + inputDesc, + input, + outputDesc, + output, + indicesDesc, + indices, + dim, + exclusive, + reverse, + cumOp); + + LogCmdCumulativeReduction( + inputDesc, outputDesc, indicesDesc, dim, exclusive, reverse, cumOp, true); + return miopen::try_([&] { + miopen::CumulativeReductionForward(miopen::deref(handle), + miopen::deref(inputDesc), + DataCast(input), + miopen::deref(outputDesc), + DataCast(output), + miopen::deref(indicesDesc), + DataCast(indices), + dim, + exclusive, + reverse, + cumOp); + }); +} diff --git a/src/driver_arguments.cpp b/src/driver_arguments.cpp index c105996d57..e69de29bb2 100644 --- a/src/driver_arguments.cpp +++ b/src/driver_arguments.cpp @@ -1,315 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include -#include - -namespace miopen { -namespace debug { - -miopenProblemDirection_t CmdArgToDirection(ConvDirection direction) -{ - switch(direction) - { - case ConvDirection::Fwd: return miopenProblemDirectionForward; - case ConvDirection::Bwd: return miopenProblemDirectionBackward; - case ConvDirection::WrW: return miopenProblemDirectionBackwardWeights; - }; - MIOPEN_THROW(miopenStatusInternalError); -} - -void ConvDataType(std::stringstream& ss, const miopen::TensorDescriptor& desc) -{ - if(desc.GetType() == miopenHalf) - { - ss << "convfp16"; - } - else if(desc.GetType() == miopenBFloat16) - { - ss << "convbfp16"; - } - else if(desc.GetType() == miopenInt8) - { - ss << "convint8"; - } - else - { - ss << "conv"; - } -} - -void BnDataType(std::stringstream& ss, const miopen::TensorDescriptor& desc) -{ - if(desc.GetType() == miopenHalf) - { - ss << "bnormfp16"; - } - else - { - ss << "bnorm"; - } -} - -void BnDriverInfo(std::stringstream& ss, - const BatchNormDirection_t& dir, - const void* resultRunningMean, - const void* resultRunningVariance, - const void* resultSaveMean, - const void* resultSaveInvVariance) -{ - if(dir != Backward) - { - ss << " --forw " << (dir == ForwardInference ? "2" : "1") << " -b 0"; - } - else - { - ss << " --forw 0 -b 1"; - } - if((resultRunningMean != nullptr) && (resultRunningVariance != nullptr)) - { - ss << " -s 1"; - } - if((resultSaveMean != nullptr) && (resultSaveInvVariance != nullptr)) - { - ss << " -r 1"; - } -} - -std::string ConvArgsForMIOpenDriver(const miopen::TensorDescriptor& xDesc, - const miopen::TensorDescriptor& wDesc, - const miopen::ConvolutionDescriptor& convDesc, - const miopen::TensorDescriptor& yDesc, - const miopenProblemDirection_t& dir, - std::optional immediate_mode_solver_id, - bool print_for_conv_driver) -{ - const auto conv_dir = [&]() { - switch(dir) - { - case miopenProblemDirectionForward: return ConvDirection::Fwd; - case miopenProblemDirectionBackward: return ConvDirection::Bwd; - case miopenProblemDirectionBackwardWeights: return ConvDirection::WrW; - case miopenProblemDirectionInference: - MIOPEN_THROW(miopenStatusInternalError); - return ConvDirection::Fwd; - } - }(); - - std::stringstream ss; - if(print_for_conv_driver) - ConvDataType(ss, xDesc); - - /// \todo Dimensions (N, C, H, W, K..) are always parsed as if layout is NC(D)HW. - /// For other layouts, invalid values are printed. - - if(convDesc.GetSpatialDimension() == 2) - { - ss << " -n " << xDesc.GetLengths()[0] // - << " -c " << xDesc.GetLengths()[1] // - << " -H " << xDesc.GetLengths()[2] // - << " -W " << xDesc.GetLengths()[3] // - << " -k " - << (convDesc.mode == miopenTranspose // - ? wDesc.GetLengths()[1] // - : wDesc.GetLengths()[0]) // - << " -y " << wDesc.GetLengths()[2] // - << " -x " << wDesc.GetLengths()[3] // - << " -p " << convDesc.GetConvPads()[0] // - << " -q " << convDesc.GetConvPads()[1] // - << " -u " << convDesc.GetConvStrides()[0] // - << " -v " << convDesc.GetConvStrides()[1] // - << " -l " << convDesc.GetConvDilations()[0] // - << " -j " << convDesc.GetConvDilations()[1]; - std::string x_layout = xDesc.GetLayout_str(); - std::string w_layout = wDesc.GetLayout_str(); - std::string y_layout = yDesc.GetLayout_str(); - if(x_layout != "NCHW") - { - ss << " --in_layout " << x_layout; - } - if(w_layout != "NCHW") - { - ss << " --fil_layout " << w_layout; - } - if(y_layout != "NCHW") - { - ss << " --out_layout " << y_layout; - } - } - else if(convDesc.GetSpatialDimension() == 3) - { - ss << " -n " << xDesc.GetLengths()[0] // - << " -c " << xDesc.GetLengths()[1] // - << " --in_d " << xDesc.GetLengths()[2] // - << " -H " << xDesc.GetLengths()[3] // - << " -W " << xDesc.GetLengths()[4] // - << " -k " - << (convDesc.mode == miopenTranspose // - ? wDesc.GetLengths()[1] // - : wDesc.GetLengths()[0]) // - << " --fil_d " << wDesc.GetLengths()[2] // - << " -y " << wDesc.GetLengths()[3] // - << " -x " << wDesc.GetLengths()[4] // - << " --pad_d " << convDesc.GetConvPads()[0] // - << " -p " << convDesc.GetConvPads()[1] // - << " -q " << convDesc.GetConvPads()[2] // - << " --conv_stride_d " << convDesc.GetConvStrides()[0] // - << " -u " << convDesc.GetConvStrides()[1] // - << " -v " << convDesc.GetConvStrides()[2] // - << " --dilation_d " << convDesc.GetConvDilations()[0] // - << " -l " << convDesc.GetConvDilations()[1] // - << " -j " << convDesc.GetConvDilations()[2] // - << " --spatial_dim 3"; - std::string x_layout = xDesc.GetLayout_str(); - std::string w_layout = wDesc.GetLayout_str(); - std::string y_layout = yDesc.GetLayout_str(); - if(x_layout != "NCDHW") - { - ss << " --in_layout " << x_layout; - } - if(w_layout != "NCDHW") - { - ss << " --fil_layout " << w_layout; - } - if(y_layout != "NCDHW") - { - ss << " --out_layout " << y_layout; - } - } - if(print_for_conv_driver) - ss << " -m " << (convDesc.mode == 1 ? "trans" : "conv"); // clang-format off - ss << " -g " << convDesc.group_count; - if(print_for_conv_driver) - ss << " -F " << std::to_string(static_cast(conv_dir)) << " -t 1"; // clang-format on - if(immediate_mode_solver_id.has_value()) - { - ss << " -S " << *immediate_mode_solver_id; - } - - return ss.str(); -} - -std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, - miopenBatchNormMode_t bn_mode, - const void* resultRunningMean, - const void* resultRunningVariance, - const void* resultSaveMean, - const void* resultSaveInvVariance, - const BatchNormDirection_t& dir, - bool print_for_bn_driver) -{ - int size = {0}; - miopenGetTensorDescriptorSize(xDesc, &size); - std::stringstream ss; - if(print_for_bn_driver) - BnDataType(ss, miopen::deref(xDesc)); - - ss << " -n " << miopen::deref(xDesc).GetLengths()[0] // clang-format off - << " -c " << miopen::deref(xDesc).GetLengths()[1]; - if(size == 5) - { - ss << " -D " << miopen::deref(xDesc).GetLengths()[2] - << " -H " << miopen::deref(xDesc).GetLengths()[3] - << " -W " << miopen::deref(xDesc).GetLengths()[4]; - } - else - { - ss << " -H " << miopen::deref(xDesc).GetLengths()[2] - << " -W " << miopen::deref(xDesc).GetLengths()[3]; - } - ss << " -m " << bn_mode; // clang-format on - if(print_for_bn_driver) - { - BnDriverInfo(ss, - dir, - resultRunningMean, - resultRunningVariance, - resultSaveMean, - resultSaveInvVariance); - } - return ss.str(); -} - -int GetFusionMode(const miopenFusionPlanDescriptor_t& fusePlanDesc) -{ - int fusion_mode = -1; - - if(miopen::deref(fusePlanDesc).op_map.size() == 4 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpBiasForward) && - (miopen::deref(fusePlanDesc).op_map[2]->kind() == miopenFusionOpBatchNormInference) && - (miopen::deref(fusePlanDesc).op_map[3]->kind() == miopenFusionOpActivForward)) - { - fusion_mode = 0; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 3 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpBatchNormInference) && - (miopen::deref(fusePlanDesc).op_map[2]->kind() == miopenFusionOpActivForward)) - { - fusion_mode = 1; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 2 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpBatchNormInference) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpActivForward)) - { - fusion_mode = 2; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 2 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpBatchNormInference)) - { - fusion_mode = 3; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 3 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpBiasForward) && - (miopen::deref(fusePlanDesc).op_map[2]->kind() == miopenFusionOpActivForward)) - { - fusion_mode = 4; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 2 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpActivForward)) - { - fusion_mode = 5; - } - else if(miopen::deref(fusePlanDesc).op_map.size() == 2 && - (miopen::deref(fusePlanDesc).op_map[0]->kind() == miopenFusionOpConvForward) && - (miopen::deref(fusePlanDesc).op_map[1]->kind() == miopenFusionOpBiasForward)) - { - fusion_mode = 6; - } - - if(fusion_mode < 0) - { - MIOPEN_LOG_E("Unknown fusion plan : " << fusion_mode); - } - - return fusion_mode; -} - -} // namespace debug -} // namespace miopen diff --git a/src/dropout_api.cpp b/src/dropout_api.cpp index 712fab97c5..e69de29bb2 100644 --- a/src/dropout_api.cpp +++ b/src/dropout_api.cpp @@ -1,225 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include -#include -#include - -// disable __device__ qualifiers -#ifdef FQUALIFIERS -#error rocrand FQUALIFIERS defined externally, probably one of rocrand device header included prior to this -#endif -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-macros" -#define FQUALIFIERS inline -#pragma clang diagnostic pop -#include "kernels/miopen_rocrand.hpp" - -extern "C" miopenStatus_t miopenCreateDropoutDescriptor(miopenDropoutDescriptor_t* dropoutDesc) -{ - - MIOPEN_LOG_FUNCTION(dropoutDesc); - return miopen::try_([&] { - auto& desc = miopen::deref(dropoutDesc); - desc = new miopen::DropoutDescriptor(); - }); -} - -extern "C" miopenStatus_t miopenDestroyDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc) -{ - MIOPEN_LOG_FUNCTION(dropoutDesc); - return miopen::try_([&] { miopen_destroy_object(dropoutDesc); }); -} - -extern "C" miopenStatus_t miopenDropoutGetReserveSpaceSize(const miopenTensorDescriptor_t xDesc, - size_t* reserveSpaceSizeInBytes) -{ - MIOPEN_LOG_FUNCTION(xDesc); - return miopen::try_([&] { - miopen::deref(reserveSpaceSizeInBytes) = - miopen::deref(xDesc).GetElementSize() * sizeof(bool); - }); -} - -extern "C" miopenStatus_t miopenDropoutGetStatesSize(miopenHandle_t handle, - size_t* stateSizeInBytes) -{ - MIOPEN_LOG_FUNCTION(handle); - return miopen::try_([&] { - miopen::deref(stateSizeInBytes) = - std::min(size_t(MAX_PRNG_STATE), miopen::deref(handle).GetImage3dMaxWidth()) * - sizeof(rocrand_state_xorwow); - }); -} - -extern "C" miopenStatus_t miopenGetDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t /* handle */, - float* dropout, - void** states, - unsigned long long* seed, - bool* use_mask, - bool* state_evo, - miopenRNGType_t* rng_mode) -{ - MIOPEN_LOG_FUNCTION(dropoutDesc); - return miopen::try_([&] { - miopen::deref(dropout) = miopen::deref(dropoutDesc).dropout; - miopen::deref(states) = &(miopen::deref(dropoutDesc).pstates); - miopen::deref(seed) = miopen::deref(dropoutDesc).seed; - miopen::deref(use_mask) = miopen::deref(dropoutDesc).use_mask; - miopen::deref(state_evo) = miopen::deref(dropoutDesc).state_evo; - miopen::deref(rng_mode) = miopen::deref(dropoutDesc).rng_mode; - }); -} - -extern "C" miopenStatus_t miopenRestoreDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t /* handle */, - float dropout, - void* states, - size_t stateSizeInBytes, - unsigned long long seed, - bool use_mask, - bool state_evo, - miopenRNGType_t rng_mode) -{ - - MIOPEN_LOG_FUNCTION(dropoutDesc, dropout, states, stateSizeInBytes, seed, use_mask, state_evo); - return miopen::try_([&] { - miopen::deref(dropoutDesc).dropout = dropout; - miopen::deref(dropoutDesc).pstates = DataCast(states); - miopen::deref(dropoutDesc).stateSizeInBytes = stateSizeInBytes; - miopen::deref(dropoutDesc).seed = seed; - miopen::deref(dropoutDesc).use_mask = use_mask; - miopen::deref(dropoutDesc).state_evo = state_evo; - miopen::deref(dropoutDesc).rng_mode = rng_mode; - }); -} - -extern "C" miopenStatus_t miopenSetDropoutDescriptor(miopenDropoutDescriptor_t dropoutDesc, - miopenHandle_t handle, - float dropout, - void* states, - size_t stateSizeInBytes, - unsigned long long seed, - bool use_mask, - bool state_evo, - miopenRNGType_t rng_mode) -{ - - MIOPEN_LOG_FUNCTION(dropoutDesc, dropout, states, stateSizeInBytes, seed, use_mask, state_evo); - return miopen::try_([&] { - miopen::deref(dropoutDesc).dropout = dropout; - miopen::deref(dropoutDesc).pstates = DataCast(states); - miopen::deref(dropoutDesc).stateSizeInBytes = stateSizeInBytes; - miopen::deref(dropoutDesc).seed = seed; - miopen::deref(dropoutDesc).use_mask = use_mask; - miopen::deref(dropoutDesc).state_evo = state_evo; - miopen::deref(dropoutDesc).rng_mode = rng_mode; - miopen::deref(dropoutDesc) - .InitPRNGState(miopen::deref(handle), DataCast(states), stateSizeInBytes, seed); - }); -} - -static void LogCmdDropout(const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t xDesc, - bool is_fwd) -{ - if(miopen::IsLoggingCmd()) - { - std::stringstream ss; - if(miopen::deref(xDesc).GetType() == miopenFloat) - ss << "dropout"; - else if(miopen::deref(xDesc).GetType() == miopenHalf) - ss << "dropoutfp16"; - - if(is_fwd) - ss << " -F 1"; - else - ss << " -F 2"; - // clang-format off - ss << " -d " << miopen::deref(xDesc).GetLengths().size() - << " -e " << std::to_string(static_cast(miopen::deref(dropoutDesc).use_mask)) - << " -l " << (miopen::deref(dropoutDesc).seed & 0xFFFFFFFF) - << " -m " << ((miopen::deref(dropoutDesc).seed >> 32) & 0xFFFFFFFF) - << " -p " << std::to_string(miopen::deref(dropoutDesc).dropout); - // clang-format on - MIOPEN_LOG_DRIVER_CMD(ss.str()); - } -} -extern "C" miopenStatus_t miopenDropoutForward(miopenHandle_t handle, - const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t noise_shape, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t yDesc, - void* y, - void* reserveSpace, - size_t reserveSpaceSizeInBytes) -{ - - MIOPEN_LOG_FUNCTION( - dropoutDesc, noise_shape, xDesc, x, yDesc, y, reserveSpace, reserveSpaceSizeInBytes); - LogCmdDropout(dropoutDesc, xDesc, true); - return miopen::try_([&] { - miopen::deref(dropoutDesc) - .DropoutForward(miopen::deref(handle), - miopen::deref(noise_shape), - miopen::deref(xDesc), - DataCast(x), - miopen::deref(yDesc), - DataCast(y), - DataCast(reserveSpace), - reserveSpaceSizeInBytes); - }); -} - -extern "C" miopenStatus_t miopenDropoutBackward(miopenHandle_t handle, - const miopenDropoutDescriptor_t dropoutDesc, - const miopenTensorDescriptor_t noise_shape, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t dxDesc, - void* dx, - void* reserveSpace, - size_t reserveSpaceSizeInBytes) -{ - - MIOPEN_LOG_FUNCTION(dropoutDesc, dyDesc, dy, dxDesc, dx, reserveSpace, reserveSpaceSizeInBytes); - LogCmdDropout(dropoutDesc, dxDesc, false); - return miopen::try_([&] { - miopen::deref(dropoutDesc) - .DropoutBackward(miopen::deref(handle), - miopen::deref(noise_shape), - miopen::deref(dyDesc), - DataCast(dy), - miopen::deref(dxDesc), - DataCast(dx), - DataCast(reserveSpace), - reserveSpaceSizeInBytes); - }); -} diff --git a/src/expanduser.cpp b/src/expanduser.cpp index c87d5d10cb..e69de29bb2 100644 --- a/src/expanduser.cpp +++ b/src/expanduser.cpp @@ -1,253 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include -#include - -#include -#ifdef _WIN32 -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include -#include -#include - -#ifndef LL_SUPER_MAGIC -#define LL_SUPER_MAGIC \ - 0x0BD00BD0 // LUSTRE - // https://github.com/whamcloud/lustre/blob/a336d7c7c1cd62a5a5213835aa85b8eaa87b076a/lustre/include/uapi/linux/lustre/lustre_user.h#L252 -#endif -#ifndef CEPH_SUPER_MAGIC -#define CEPH_SUPER_MAGIC 0x00c36400 -#endif -#ifndef NFS_SUPER_MAGIC -#define NFS_SUPER_MAGIC 0x6969 -#endif -#ifndef SMB_SUPER_MAGIC -#define SMB_SUPER_MAGIC 0x517b -#endif -#ifndef SMB2_MAGIC_NUMBER -#define SMB2_MAGIC_NUMBER 0xfe534d42 -#endif -#ifndef CIFS_MAGIC_NUMBER -#define CIFS_MAGIC_NUMBER 0xff534d42 -#endif -#ifndef CODA_SUPER_MAGIC -#define CODA_SUPER_MAGIC 0x73757245 -#endif -#ifndef OCFS2_SUPER_MAGIC -#define OCFS2_SUPER_MAGIC 0x7461636f -#endif -#ifndef AFS_SUPER_MAGIC -#define AFS_SUPER_MAGIC 0x5346414f -#endif -#ifndef EXT2_OLD_SUPER_MAGIC -#define EXT2_OLD_SUPER_MAGIC 0xef51 -#endif -#ifndef EXT4_SUPER_MAGIC -#define EXT4_SUPER_MAGIC 0xef53 -#endif -#ifndef TMPFS_MAGIC -#define TMPFS_MAGIC 0x01021994 -#endif -#ifndef OVERLAYFS_SUPER_MAGIC -#define OVERLAYFS_SUPER_MAGIC 0x794c7630 -#endif -#endif // __linux__ - -MIOPEN_DECLARE_ENV_VAR_STR(HOME) - -#ifdef _WIN32 -MIOPEN_DECLARE_ENV_VAR_STR(USERPROFILE, miopen::fs::temp_directory_path().string()) -MIOPEN_DECLARE_ENV_VAR_STR(HOMEPATH, miopen::fs::temp_directory_path().string()) -MIOPEN_DECLARE_ENV_VAR_STR(HOMEDRIVE) -#endif - -namespace miopen { - -#ifdef __linux__ - -#define CASE_RET_STRING(macro) \ - case macro: return #macro; - -namespace { -const char* Stringize(unsigned long ft) -{ - switch(ft) - { - CASE_RET_STRING(NFS_SUPER_MAGIC) - CASE_RET_STRING(SMB_SUPER_MAGIC) - CASE_RET_STRING(SMB2_MAGIC_NUMBER) - CASE_RET_STRING(CIFS_MAGIC_NUMBER) - CASE_RET_STRING(CODA_SUPER_MAGIC) - CASE_RET_STRING(OCFS2_SUPER_MAGIC) - CASE_RET_STRING(AFS_SUPER_MAGIC) - CASE_RET_STRING(LL_SUPER_MAGIC) - CASE_RET_STRING(CEPH_SUPER_MAGIC) - CASE_RET_STRING(TMPFS_MAGIC) - CASE_RET_STRING(OVERLAYFS_SUPER_MAGIC) - CASE_RET_STRING(EXT2_OLD_SUPER_MAGIC) - case EXT4_SUPER_MAGIC: return "EXT2/3/4_SUPER_MAGIC"; - default: return ""; - } -} - -bool IsNetworked(unsigned long ft) -{ - switch(ft) - { - case NFS_SUPER_MAGIC: // fall through - case SMB_SUPER_MAGIC: // fall through - case SMB2_MAGIC_NUMBER: // fall through - case CIFS_MAGIC_NUMBER: // fall through - case CODA_SUPER_MAGIC: // fall through - case OCFS2_SUPER_MAGIC: // fall through - case AFS_SUPER_MAGIC: // fall through - case LL_SUPER_MAGIC: // fall through - case CEPH_SUPER_MAGIC: return true; - default: return false; - } -} -} // namespace - -#undef CASE_RET_STRING - -bool IsNetworkedFilesystem(const fs::path& path_) -{ - // Non-DEV builds put user databases in ~/.config/miopen by default; the binary cache is placed - // in ~/.cache/miopen. If these directories do not exist, this is not a problem, because the - // library creates them as needed. - // - // The problem is that statfs doesn't work in this case, and we need to determine the type of FS - // _before_ the databases are created. - // - // Solution (A): Just create_directories if path doesn't exist. It looks like a hack: if the - // path is on NFS, then the library will not use it, but will create directories (which is not - // good). And this function should not have any side effects on the file system. - // - // Solution (B): Traverse the input path up to the first existing directory, then check that - // directory. Stop after some fixed number of iterations to protect against possible file system - // problems. Let's use this solution for now. - auto path = path_; - for(int i = 0; i < 32; ++i) - { - if(fs::exists(path)) - break; - MIOPEN_LOG_NQI2("Path does not exist: '" << path << '\''); - path = path.parent_path(); - if(path.empty()) - break; - } - struct statfs stat; - const int rc = statfs(path.c_str(), &stat); - if(rc != 0) - { - // NOLINTNEXTLINE (concurrency-mt-unsafe) - MIOPEN_LOG_NQE("statfs('" << path << "') rc = " << rc << ", '" << strerror(errno) << "'"); - return false; - } - MIOPEN_LOG_NQI("Filesystem type at '" << path << "' is: 0x" << std::hex << stat.f_type << " '" - << Stringize(stat.f_type) << '\''); - return IsNetworked(stat.f_type); -} - -namespace { -std::string GetHomeDir() -{ - const auto p = env::value(HOME); - if(!(p.empty() || p == std::string("/"))) - { - return p; - } - // todo: - // need to figure out what is the correct thing to do here - // in tensoflow unit tests run via bazel, $HOME is not set, so this can happen - // setting home_dir to the /tmp for now - return fs::temp_directory_path().string(); -} -} // namespace - -fs::path ExpandUser(const fs::path& path) -{ - static const auto home_dir = GetHomeDir(); - return {ReplaceString(path.string(), "~", home_dir)}; -} - -#else - -namespace { -std::optional> ReplaceVariable( - std::string_view path, const env::detail::EnvVar& t, std::size_t offset = 0) -{ - std::vector variables{"$" + std::string{t.name()}, - "$env:" + std::string{t.name()}, - "%" + std::string{t.name()} + "%"}; - for(auto& variable : variables) - { - auto pos{path.find(variable, offset)}; - if(pos != std::string::npos) - { - std::string result{path}; - result.replace(pos, variable.length(), t.value()); - return {{pos, result}}; - } - } - return std::nullopt; -} -} // namespace - -fs::path ExpandUser(const fs::path& path) -{ - auto result{ReplaceVariable(path.string(), USERPROFILE)}; - if(!result) - { - result = ReplaceVariable(path.string(), HOME); - if(!result) - { - result = ReplaceVariable(path.string(), HOMEDRIVE); - if(result) - { - result = ReplaceVariable(result->second, HOMEPATH, result->first); - // TODO: if (not result): log warning message that - // HOMEDRIVE and HOMEPATH work in conjunction, respectively. - } - } - } - return {!result ? path : std::get<1>(*result)}; -} - -bool IsNetworkedFilesystem(const fs::path&) { return false; } - -#endif - -} // namespace miopen diff --git a/src/fused_api.cpp b/src/fused_api.cpp index 6a4ac9a866..e69de29bb2 100644 --- a/src/fused_api.cpp +++ b/src/fused_api.cpp @@ -1,512 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Return an error code that is "NotImplemented", if it exists then return success -// This function should: -// set up the place descriptor with expected input and ouput edges. -// Set up the internal datastructures for the fused kernel. -extern "C" miopenStatus_t miopenCreateFusionPlan(miopenFusionPlanDescriptor_t* fusePlanDesc, - const miopenFusionDirection_t fuseDirection, - const miopenTensorDescriptor_t inputDesc) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, fuseDirection, inputDesc); - return miopen::try_([&] { - auto& desc = miopen::deref(fusePlanDesc); - desc = new miopen::FusionPlanDescriptor(fuseDirection, miopen::deref(inputDesc)); - }); -} - -extern "C" miopenStatus_t miopenDestroyFusionPlan(miopenFusionPlanDescriptor_t fusePlanDesc) -{ - - MIOPEN_LOG_FUNCTION(fusePlanDesc); - return miopen::try_([&] { miopen_destroy_object(fusePlanDesc); }); -} - -extern "C" miopenStatus_t miopenFusionPlanGetOp(miopenFusionPlanDescriptor_t fusePlanDesc, - const int op_idx, - miopenFusionOpDescriptor_t* op) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, op_idx); - miopenStatus_t res = miopenStatusBadParm; - miopen::try_([&] { - std::shared_ptr desc; - res = miopen::deref(fusePlanDesc).GetOp(op_idx, desc); - miopen::deref(op) = desc.get(); - }); - return res; -} - -// Return an error code that is "NotImplemented", if it exists then return success -extern "C" miopenStatus_t miopenCompileFusionPlan(miopenHandle_t handle, - miopenFusionPlanDescriptor_t fusePlanDesc) -{ - MIOPEN_LOG_FUNCTION(handle, fusePlanDesc); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { res = miopen::deref(fusePlanDesc).Compile(miopen::deref(handle)); }); - return res; -} - -extern "C" miopenStatus_t -miopenFusionPlanGetWorkSpaceSize(miopenHandle_t handle, - miopenFusionPlanDescriptor_t fusePlanDesc, - size_t* workSpaceSize, - miopenConvFwdAlgorithm_t algo) -{ - MIOPEN_LOG_FUNCTION(handle, fusePlanDesc, algo); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - size_t sz; - res = miopen::deref(fusePlanDesc).GetWorkspaceSizeImmed(miopen::deref(handle), sz, algo); - miopen::deref(workSpaceSize) = sz; - }); - return res; -} - -extern "C" miopenStatus_t -miopenFusionPlanConvolutionGetAlgo(miopenFusionPlanDescriptor_t fusePlanDesc, - const int requestAlgoCount, - int* returnedAlgoCount, - miopenConvFwdAlgorithm_t* returnedAlgos) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, requestAlgoCount); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - int cnt = 0; - res = miopen::deref(fusePlanDesc).GetConvAlgos(requestAlgoCount, cnt, returnedAlgos); - miopen::deref(returnedAlgoCount) = cnt; - }); - return res; -} - -extern "C" miopenStatus_t -miopenFusionPlanConvolutionSetAlgo(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenConvFwdAlgorithm_t algo) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, algo); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { res = miopen::deref(fusePlanDesc).SetConvAlgo(algo); }); - return res; -} - -// Create convolution ops with unknown algorithms -extern "C" miopenStatus_t miopenCreateOpConvForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* convOp, - miopenConvolutionDescriptor_t convDesc, - const miopenTensorDescriptor_t wDesc) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, convOp, convDesc, wDesc); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto fod = std::make_shared(miopen::deref(convDesc), - miopen::deref(wDesc)); - miopen::deref(convOp) = fod.get(); - res = miopen::deref(fusePlanDesc).AddOp(fod); - }); - return res; -} -// Activation create ops -extern "C" miopenStatus_t miopenCreateOpActivationForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* activOp, - miopenActivationMode_t mode) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, activOp, mode); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto fod = std::make_shared(mode); - miopen::deref(activOp) = fod.get(); - res = miopen::deref(fusePlanDesc).AddOp(fod); - }); - return res; -} - -extern "C" miopenStatus_t -miopenCreateOpActivationBackward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* activOp, - miopenActivationMode_t mode) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, activOp, mode); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto fod = std::make_shared(mode); - miopen::deref(activOp) = fod.get(); - res = miopen::deref(fusePlanDesc).AddOp(fod); - }); - return res; -} -//--- - -extern "C" miopenStatus_t miopenCreateOpBiasForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* biasOp, - const miopenTensorDescriptor_t bDesc) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, biasOp, bDesc); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto bod = std::make_shared(miopen::deref(bDesc)); - miopen::deref(biasOp) = bod.get(); - res = miopen::deref(fusePlanDesc).AddOp(bod); - }); - return res; -} - -// Batch normalization create op -extern "C" miopenStatus_t -miopenCreateOpBatchNormInference(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnOp, - const miopenBatchNormMode_t bn_mode, - const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, bnOp, bn_mode, bnScaleBiasMeanVarDesc); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto bod = std::make_shared( - bn_mode, miopen::deref(bnScaleBiasMeanVarDesc)); - miopen::deref(bnOp) = bod.get(); - res = miopen::deref(fusePlanDesc).AddOp(bod); - }); - return res; -} - -extern "C" miopenStatus_t miopenCreateOpBatchNormForward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnOp, - const miopenBatchNormMode_t bn_mode, - bool runningMeanVariance) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, bnOp, bn_mode, runningMeanVariance); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto bod = std::make_shared( - bn_mode, runningMeanVariance); - miopen::deref(bnOp) = bod.get(); - res = miopen::deref(fusePlanDesc).AddOp(bod); - }); - return res; -} - -extern "C" miopenStatus_t miopenCreateOpBatchNormBackward(miopenFusionPlanDescriptor_t fusePlanDesc, - miopenFusionOpDescriptor_t* bnOp, - const miopenBatchNormMode_t bn_mode) -{ - MIOPEN_LOG_FUNCTION(fusePlanDesc, bnOp, bn_mode); - miopenStatus_t res = miopenStatusUnknownError; - miopen::try_([&] { - auto bod = std::make_shared(bn_mode); - miopen::deref(bnOp) = bod.get(); - res = miopen::deref(fusePlanDesc).AddOp(bod); - }); - return res; -} -//--- - -extern "C" miopenStatus_t miopenCreateOperatorArgs(miopenOperatorArgs_t* args) -{ - MIOPEN_LOG_FUNCTION(args); - return miopen::try_([&] { - auto& theArgs = miopen::deref(args); - theArgs = new miopen::OperatorArgs(); - }); -} - -extern "C" miopenStatus_t miopenDestroyOperatorArgs(miopenOperatorArgs_t args) -{ - MIOPEN_LOG_FUNCTION(args); - return miopen::try_([&] { miopen_destroy_object(args); }); -} -extern "C" miopenStatus_t miopenSetOpArgsConvForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t convOp, - const void* alpha, - const void* beta, - const void* w) -{ - MIOPEN_LOG_FUNCTION(args, alpha, beta, convOp, w); - return miopen::try_([&] { - auto&& op = dynamic_cast(miopen::deref(convOp)); - auto tmp = DataCast(w); - op.SetArgs(miopen::deref(args), alpha, beta, tmp); - }); -} - -extern "C" miopenStatus_t miopenSetOpArgsBiasForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t biasOp, - const void* alpha, - const void* beta, - const void* bias) -{ - - MIOPEN_LOG_FUNCTION(args, biasOp, alpha, beta, bias); - return miopen::try_([&] { - auto&& op = dynamic_cast(miopen::deref(biasOp)); - op.SetArgs(miopen::deref(args), alpha, beta, DataCast(bias)); - }); -} - -extern "C" miopenStatus_t miopenSetOpArgsActivForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t activFwdOp, - const void* alpha, - const void* beta, - double activAlpha, - double activBeta, - double activGamma) -{ - - MIOPEN_LOG_FUNCTION(args, activFwdOp, alpha, beta, activAlpha, activBeta, activGamma); - return miopen::try_([&] { - auto&& op = dynamic_cast(miopen::deref(activFwdOp)); - op.SetArgs(miopen::deref(args), alpha, beta, activAlpha, activBeta, activGamma); - }); -} - -extern "C" miopenStatus_t miopenSetOpArgsActivBackward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t activBwdOp, - const void* alpha, - const void* beta, - const void* y, - const void* /*reserved*/, - double activAlpha, - double activBeta, - double activGamma) -{ - MIOPEN_LOG_FUNCTION(args, activBwdOp, alpha, beta, y, activAlpha, activBeta, activGamma); - return miopen::try_([&] { - auto&& op = dynamic_cast(miopen::deref(activBwdOp)); - op.SetArgs(miopen::deref(args), - alpha, - beta, - DataCast(y), - nullptr, - activAlpha, - activBeta, - activGamma); - }); -} - -// Fusion op args for Batch Normalization -extern "C" miopenStatus_t miopenSetOpArgsBatchNormInference(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnOp, - const void* alpha, - const void* beta, - const void* bnScale, - const void* bnBias, - const void* estimatedMean, - const void* estimatedVariance, - double epsilon) -{ - MIOPEN_LOG_FUNCTION( - args, bnOp, alpha, beta, bnScale, bnBias, estimatedMean, estimatedVariance, epsilon); - return miopen::try_([&] { - auto&& op = - dynamic_cast(miopen::deref(bnOp)); - op.SetArgs(miopen::deref(args), - alpha, - beta, - DataCast(bnScale), - DataCast(bnBias), - DataCast(estimatedMean), - DataCast(estimatedVariance), - epsilon); - }); -} - -extern "C" miopenStatus_t miopenSetOpArgsBatchNormForward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnFwdOp, - const void* alpha, - const void* beta, - const void* bnScale, - const void* bnBias, - void* savedMean, - void* savedInvVariance, - void* runningMean, - void* runningVariance, - double expAvgFactor, - double epsilon) -{ - MIOPEN_LOG_FUNCTION(args, - bnFwdOp, - alpha, - beta, - bnScale, - bnBias, - savedMean, - savedInvVariance, - runningMean, - runningVariance, - expAvgFactor, - epsilon); - return miopen::try_([&] { - auto&& op = - dynamic_cast(miopen::deref(bnFwdOp)); - op.SetArgs(miopen::deref(args), - alpha, - beta, - DataCast(runningMean), - DataCast(runningVariance), - DataCast(savedMean), - DataCast(savedInvVariance), - DataCast(bnScale), - DataCast(bnBias), - expAvgFactor, - epsilon); - }); -} - -extern "C" miopenStatus_t miopenSetOpArgsBatchNormBackward(miopenOperatorArgs_t args, - const miopenFusionOpDescriptor_t bnBwdOp, - const void* alpha, - const void* beta, - const void* x, - const void* bnScale, - const void* bnBias, - void* resultBnScaleDiff, - void* resultBnBiasDiff, - const void* savedMean, - const void* savedInvVariance) -{ - MIOPEN_LOG_FUNCTION(args, - bnBwdOp, - alpha, - beta, - x, - bnScale, - bnBias, - resultBnScaleDiff, - resultBnBiasDiff, - savedMean, - savedInvVariance); - return miopen::try_([&] { - auto&& op = - dynamic_cast(miopen::deref(bnBwdOp)); - op.SetArgs(miopen::deref(args), - alpha, - beta, - DataCast(x), - DataCast(bnScale), - DataCast(bnBias), - DataCast(resultBnScaleDiff), - DataCast(resultBnBiasDiff), - DataCast(savedMean), - DataCast(savedInvVariance)); - }); -} -//--- - -// Return an error code that is "NotImplemented", if it exists then return success -extern "C" miopenStatus_t miopenExecuteFusionPlan(const miopenHandle_t handle, - const miopenFusionPlanDescriptor_t fusePlanDesc, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputDesc, - void* output, - miopenOperatorArgs_t args) -{ - MIOPEN_LOG_FUNCTION(handle, fusePlanDesc, inputDesc, input, outputDesc, output, args); - return miopen::try_([&] { - miopen::deref(fusePlanDesc) - .Execute(miopen::deref(handle), - miopen::deref(inputDesc), - DataCast(input), - miopen::deref(outputDesc), - DataCast(output), - miopen::deref(args)); - }); -} - -extern "C" miopenStatus_t -miopenConvolutionBiasActivationForward(miopenHandle_t handle, - const void* alpha1, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t wDesc, - const void* w, - const miopenConvolutionDescriptor_t conv_desc, - miopenConvFwdAlgorithm_t algo, - void* workspace, - size_t workspaceSizeInBytes, - const void* alpha2, - const miopenTensorDescriptor_t zDesc, - const void* z, - const miopenTensorDescriptor_t biasDesc, - const void* bias, - const miopenActivationDescriptor_t activationDesc, - const miopenTensorDescriptor_t yDesc, - void* y) -{ - - MIOPEN_LOG_FUNCTION(handle, - alpha1, - xDesc, - x, - wDesc, - w, - conv_desc, - algo, - workspace, - workspaceSizeInBytes, - alpha2, - zDesc, - z, - biasDesc, - bias, - activationDesc, - ydesc, - y); - miopenStatus_t res = miopenStatusUnknownError; - const auto try_res = miopen::try_([&] { - res = ConvBiasActivFusion(miopen::deref(handle), - alpha1, - miopen::deref(xDesc), - DataCast(x), - miopen::deref(wDesc), - DataCast(w), - miopen::deref(conv_desc), - algo, - DataCast(workspace), - workspaceSizeInBytes, - alpha2, - miopen::deref(zDesc), - DataCast(z), - miopen::deref(biasDesc), - DataCast(bias), - miopen::deref(activationDesc), - miopen::deref(yDesc), - DataCast(y)); - }); - if(try_res == miopenStatusSuccess) - return res; - return try_res; -} diff --git a/src/glu.cpp b/src/glu.cpp index 9d1181d32e..e69de29bb2 100644 --- a/src/glu.cpp +++ b/src/glu.cpp @@ -1,102 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include -#include - -namespace miopen { - -namespace glu { - -miopenStatus_t GLUForward(Handle& handle, - const TensorDescriptor& inputDesc, - ConstData_t input, - const TensorDescriptor& outputDesc, - Data_t output, - uint32_t dim) -{ - const auto problem = glu::ProblemDescription{inputDesc, outputDesc, dim}; - - const auto invoke_params = [&]() { - auto tmp = glu::FwdInvokeParams{}; - tmp.type = InvokeType::Run; - tmp.inputDesc = &inputDesc; - tmp.outputDesc = &outputDesc; - tmp.input = input; - tmp.output = output; - tmp.dim = dim; - return tmp; - }(); - - const auto algo = AlgorithmName{"GLUForward"}; - const auto solvers = solver::SolverContainer{}; - - solvers.ExecutePrimitive(handle, problem, algo, invoke_params); - - return miopenStatusSuccess; -} - -miopenStatus_t GLUBackward(Handle& handle, - const TensorDescriptor& inputDesc, - ConstData_t input, - const TensorDescriptor& outputGradDesc, - ConstData_t outputGrad, - const TensorDescriptor& inputGradDesc, - Data_t inputGrad, - uint32_t dim) -{ - const auto problem = glu::ProblemDescription{inputDesc, outputGradDesc, inputGradDesc, dim}; - - const auto invoke_params = [&]() { - auto tmp = glu::BwdInvokeParams{}; - tmp.type = InvokeType::Run; - tmp.inputDesc = &inputDesc; - tmp.input = input; - tmp.inputGradDesc = &inputGradDesc; - tmp.inputGrad = inputGrad; - tmp.outputGradDesc = &outputGradDesc; - tmp.outputGrad = outputGrad; - tmp.dim = dim; - return tmp; - }(); - - const auto algo = AlgorithmName{"GLUBackward"}; - const auto solvers = solver::SolverContainer{}; - - solvers.ExecutePrimitive(handle, problem, algo, invoke_params); - - return miopenStatusSuccess; -} - -} // namespace glu - -} // namespace miopen diff --git a/src/glu/problem_description.cpp b/src/glu/problem_description.cpp index 53caaf1fc7..e69de29bb2 100644 --- a/src/glu/problem_description.cpp +++ b/src/glu/problem_description.cpp @@ -1,208 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include - -#include - -namespace miopen { - -namespace glu { - -ProblemDescription::ProblemDescription(const TensorDescriptor& inputDesc_, - const TensorDescriptor& outputDesc_, - uint32_t dim_) - : direction(Direction::Forward), inputDesc(inputDesc_), outputDesc(outputDesc_), dim(dim_) -{ - if(inputDesc.GetNumDims() != outputDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Number of dimensions between input and output " - "tensor do not match."); - } - - if(dim >= inputDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Dimension is out of range."); - } - - if(inputDesc.GetLengths()[dim] % 2 != 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: The split dimension size of input tensor should " - "be divisible by 2."); - } - - for(auto i = 0; i < inputDesc.GetNumDims(); i++) - { - if(i == dim) - { - if(inputDesc.GetLengths()[i] / 2 != outputDesc.GetLengths()[i]) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Dimension sizes don't match between " - "input tensor and output tensor."); - } - } - else - { - if(inputDesc.GetLengths()[i] != outputDesc.GetLengths()[i]) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Dimension sizes don't match between " - "input tensor and output tensor."); - } - } - } - - if(!IsSameType()) - { - MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Tensor types do not match."); - } -} - -ProblemDescription::ProblemDescription(const TensorDescriptor& inputDesc_, - const TensorDescriptor& outputGradDesc_, - const TensorDescriptor& inputGradDesc_, - uint32_t dim_) - : direction(Direction::Backward), - inputDesc(inputDesc_), - outputGradDesc(outputGradDesc_), - inputGradDesc(inputGradDesc_), - dim(dim_) -{ - if(inputDesc.GetNumDims() != inputGradDesc.GetNumDims() || - inputDesc.GetNumDims() != outputGradDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Number of tensor dimensions between input and " - "output tensor do not match."); - } - - if(dim >= inputDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Dimension is out of range."); - } - - if(inputDesc.GetLengths()[dim] % 2 != 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: The split dimension size of input tensor should " - "be divisible by 2."); - } - - for(auto i = 0; i < inputDesc.GetNumDims(); i++) - { - if(i == dim) - { - if(inputDesc.GetLengths()[i] / 2 != outputGradDesc.GetLengths()[i] || - inputDesc.GetLengths()[i] != inputGradDesc.GetLengths()[i]) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Dimension sizes don't match between " - "input tensor and output tensor."); - } - } - else - { - if(inputDesc.GetLengths()[i] != inputGradDesc.GetLengths()[i] || - inputDesc.GetLengths()[i] != outputGradDesc.GetLengths()[i]) - { - MIOPEN_THROW(miopenStatusBadParm, - "GLU::ProblemDescription: Dimension sizes don't match between " - "input tensor and output tensor."); - } - } - } - - if(!IsSameType()) - { - MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Tensor types do not match."); - } -} - -bool ProblemDescription::IsSameType() const -{ - if(direction == Direction::Forward) - { - if(inputDesc.GetType() != outputDesc.GetType()) - { - return false; - } - } - else - { - if(inputDesc.GetType() != inputGradDesc.GetType() || - inputGradDesc.GetType() != outputGradDesc.GetType()) - { - return false; - } - } - - return true; -} - -bool ProblemDescription::IsAllContiguous() const -{ - if(direction == Direction::Forward) - { - if(!(inputDesc.IsContiguous() && outputDesc.IsContiguous())) - { - return false; - } - } - else - { - if(!(inputDesc.IsContiguous() && inputGradDesc.IsContiguous() && - outputGradDesc.IsContiguous())) - { - return false; - } - } - - return true; -} - -NetworkConfig ProblemDescription::MakeNetworkConfig() const -{ - auto input_numel = inputDesc.GetElementSize(); - auto io_dtype = miopen::GetDataType(inputDesc.GetType()); - - std::ostringstream ss; - - ss << "io_dtype" << io_dtype; - ss << "dim" << dim; - ss << "input_numel" << input_numel; - ss << IsAllContiguous(); - - return NetworkConfig{ss.str()}; -} - -} // namespace glu - -} // namespace miopen diff --git a/src/glu_api.cpp b/src/glu_api.cpp index 856eeb6920..e69de29bb2 100644 --- a/src/glu_api.cpp +++ b/src/glu_api.cpp @@ -1,74 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include -#include -#include -#include - -extern "C" miopenStatus_t miopenGLUForward(miopenHandle_t handle, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputDesc, - void* output, - const uint32_t dim) -{ - MIOPEN_LOG_FUNCTION(handle, inputDesc, input, outputDesc, output, dim); - - return miopen::try_([&] { - miopen::glu::GLUForward(miopen::deref(handle), - miopen::deref(inputDesc), - DataCast(input), - miopen::deref(outputDesc), - DataCast(output), - dim); - }); -} - -extern "C" miopenStatus_t miopenGLUBackward(miopenHandle_t handle, - const miopenTensorDescriptor_t inputDesc, - const void* input, - const miopenTensorDescriptor_t outputGradDesc, - const void* outputGrad, - const miopenTensorDescriptor_t inputGradDesc, - void* inputGrad, - const uint32_t dim) -{ - MIOPEN_LOG_FUNCTION( - handle, inputDesc, input, outputGradDesc, outputGrad, inputGradDesc, inputGrad, dim); - return miopen::try_([&] { - miopen::glu::GLUBackward(miopen::deref(handle), - miopen::deref(inputDesc), - DataCast(input), - miopen::deref(outputGradDesc), - DataCast(outputGrad), - miopen::deref(inputGradDesc), - DataCast(inputGrad), - dim); - }); -} diff --git a/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp b/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp index dbbfcd18d5..e69de29bb2 100644 --- a/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp +++ b/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp @@ -1,100 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include -#include -#include - -namespace miopen { - -namespace graphapi { - -namespace { -std::vector Convert(const std::vector& values) -{ - std::vector converted(values.size()); - std::transform(values.begin(), values.end(), converted.begin(), [](int64_t value) { - assert(value <= std::numeric_limits::max() && - value >= std::numeric_limits::min()); - return static_cast(value); - }); - - return converted; -} - -ConvolutionDescriptor Convert(const Convolution& conv, int groupCount) -{ - return {conv.getSpatialDims(), - conv.getMode(), - miopenPaddingMode_t::miopenPaddingDefault, - Convert(conv.getPrePaddings()), - Convert(conv.getFilterStrides()), - Convert(conv.getDilations()), - Convert(conv.getPostPaddings()), - groupCount}; -} -} // namespace - -void ConvBiasResAddActivForwardExecutor::execute(miopenHandle_t handle, const VariantPack& vpk) -{ - auto convDesc = Convert(*mConvolution, mGroupCount); - - ActivationDescriptor activDesc{miopenActivationRELU, mActivationAlpha, 1.0, 1.0}; - - auto* xData = vpk.getDataPointer(mXTensor->getId()); - auto* wData = vpk.getDataPointer(mWTensor->getId()); - auto* zData = vpk.getDataPointer(mZTensor->getId()); - auto* biasData = vpk.getDataPointer(mBiasTensor->getId()); - auto* yData = vpk.getDataPointer(mYTensor->getId()); - - auto status = - ConvBiasActivFusion(miopen::deref(handle), - &mAlpha1, - *mXTensor, - xData, - *mWTensor, - wData, - convDesc, - miopenConvFwdAlgorithm_t::miopenConvolutionFwdAlgoImplicitGEMM, - nullptr, - 0, - &mAlpha2, - *mZTensor, - zData, - *mBiasTensor, - biasData, - activDesc, - *mYTensor, - yData); - - MIOPEN_THROW_IF(status != miopenStatusSuccess, "execute failed"); -} - -} // namespace graphapi - -} // namespace miopen diff --git a/src/include/miopen/activ.hpp b/src/include/miopen/activ.hpp index 26ce91112f..e69de29bb2 100644 --- a/src/include/miopen/activ.hpp +++ b/src/include/miopen/activ.hpp @@ -1,92 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef MIOPEN_ACTIV_HPP_ -#define MIOPEN_ACTIV_HPP_ - -#include -#include -#include - -#include - -#include - -namespace miopen { - -struct Handle; -struct TensorDescriptor; - -struct MIOPEN_INTERNALS_EXPORT ActivationDescriptor : miopenActivationDescriptor -{ - ActivationDescriptor(); - ActivationDescriptor(miopenActivationMode_t m, const double* pparms); - ActivationDescriptor(miopenActivationMode_t m, double alpha, double beta, double gamma); - - miopenActivationMode_t GetMode() const; - double GetAlpha() const; - double GetBeta() const; - double GetGamma() const; - - miopenStatus_t Forward(Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - size_t xOffset = 0, - size_t yOffset = 0) const; - - miopenStatus_t Backward(Handle& handle, - const void* alpha, - const TensorDescriptor& yDesc, - ConstData_t y, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - size_t yOffset = 0, - size_t dyOffset = 0, - size_t xOffset = 0, - size_t dxOffset = 0) const; - - friend std::ostream& operator<<(std::ostream& stream, const ActivationDescriptor& x); - - friend void to_json(nlohmann::json& json, const ActivationDescriptor& descriptor); - friend void from_json(const nlohmann::json& json, ActivationDescriptor& descriptor); - -private: - std::vector parms; - - miopenActivationMode_t mode = miopenActivationPASTHRU; -}; - -} // namespace miopen -MIOPEN_DEFINE_OBJECT(miopenActivationDescriptor, miopen::ActivationDescriptor); -#endif // _MIOPEN_ACTIV_HPP_ diff --git a/src/include/miopen/any_solver.hpp b/src/include/miopen/any_solver.hpp index 6305893fd9..e69de29bb2 100644 --- a/src/include/miopen/any_solver.hpp +++ b/src/include/miopen/any_solver.hpp @@ -1,388 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MIOPEN_GUARD_MLOPEN_ANY_SOLVER_HPP -#define MIOPEN_GUARD_MLOPEN_ANY_SOLVER_HPP - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -namespace miopen { -namespace solver { - -struct AnySolver -{ - AnySolver() : ptr_value(nullptr){}; - template - AnySolver(U src) : ptr_value(new AnySolver_tmpl(std::forward(src))){}; - bool IsApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const - { - assert(ptr_value != nullptr); - return ptr_value->IsApplicable(ctx, problem); - }; - bool IsTunable() const - { - assert(ptr_value != nullptr); - return ptr_value->IsTunable(); - }; - bool TestPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& params) const - { - assert(ptr_value != nullptr); - return ptr_value->TestPerfCfgParams(ctx, problem, params); - }; - std::vector GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const - { - assert(ptr_value != nullptr); - return ptr_value->GetAllSolutions(ctx, problem); - }; - bool IsDynamic() const - { - assert(ptr_value != nullptr); - return ptr_value->IsDynamic(); - }; - float GetWti(const ExecutionContext& ctx, const miopen::conv::ProblemDescription& problem) const - { - assert(ptr_value != nullptr); - return ptr_value->GetWti(ctx, problem); - }; - const std::type_info& Type() const - { - assert(ptr_value != nullptr); - return ptr_value->Type(); - }; - bool IsEmpty() const { return ptr_value == nullptr; }; - ConvSolution FindSolution(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db, - const miopen::AnyInvokeParams& invoke_ctx, - const std::string& perf_cfg = "") const - { - assert(ptr_value != nullptr); - return ptr_value->FindSolution(ctx, problem, db, invoke_ctx, perf_cfg); - }; - InvokerFactory GetInvokeFactory(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& perf_cfg = "") const - { - assert(ptr_value != nullptr); - return ptr_value->GetInvokeFactory(ctx, problem, perf_cfg); - }; - std::string GetPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db) const - { - assert(ptr_value != nullptr); - return ptr_value->GetPerfCfgParams(ctx, problem, db); - }; - std::string GetSolverDbId() const - { - assert(ptr_value != nullptr); - return ptr_value->GetSolverDbId(); - } - - size_t GetWorkspaceSize(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const - { - assert(ptr_value != nullptr); - return ptr_value->GetWorkspaceSize(ctx, problem); - } - - bool MayNeedWorkspace() const - { - assert(ptr_value != nullptr); - return ptr_value->MayNeedWorkspace(); - } - - // virtual base class - struct AnySolver_base - { - using ptr = std::shared_ptr; - - virtual ~AnySolver_base(){}; - virtual bool IsApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const = 0; - virtual bool IsTunable() const = 0; - virtual bool TestPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& params) const = 0; - virtual std::vector - GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const = 0; - virtual bool IsDynamic() const = 0; - virtual float GetWti(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const = 0; - virtual const std::type_info& Type() const = 0; - virtual std::string GetSolverDbId() const = 0; - virtual ConvSolution FindSolution(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db, - const miopen::AnyInvokeParams& invoke_ctx, - const std::string& perf_cfg) const = 0; - virtual InvokerFactory GetInvokeFactory(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& perf_cfg) const = 0; - virtual std::string GetPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db) const = 0; - virtual size_t GetWorkspaceSize(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const = 0; - virtual bool MayNeedWorkspace() const = 0; - }; - - // templated derived class - template - struct AnySolver_tmpl : AnySolver_base - { - struct TunableSolver - { - template - static constexpr auto Test(U*) -> - typename std::is_class().GetDefaultPerformanceConfig( - std::declval(), - std::declval()))>::type; - - template - static constexpr std::false_type Test(...); - - using type = decltype(Test(nullptr)); - static constexpr bool Is = type::value; - }; - - struct LegacySolver - { - template - static constexpr auto Test(U*) -> typename std::is_same< - LegacyPerformanceConfig, - decltype(std::declval().GetDefaultPerformanceConfig( - std::declval(), - std::declval()))>::type; - - template - static constexpr std::false_type Test(...); - - using type = decltype(Test(nullptr)); - static constexpr bool Is = type::value; - }; - - bool TestPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& params, - std::true_type) const - { - using PerformanceConfig = decltype(value.GetDefaultPerformanceConfig( - std::declval(), - std::declval())); - PerformanceConfig config{}; - - bool success = config.Deserialize(params); - if(!success) - { - MIOPEN_LOG_WE("Perf params are obsolete or corrupt: " - << params << ". Performance may degrade."); - return false; - } - - success = value.IsValidPerformanceConfig(ctx, problem, config); - - return success; - } - bool TestPerfCfgParams(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const std::string&, - std::false_type) const - { - return false; - } - - bool TestPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& params) const override - { - return TestPerfCfgParams( - ctx, problem, params, std::integral_constant()); - } - - // tunable legacy solver - std::vector GetAllSolutions(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - std::true_type, - std::true_type) const - { - MIOPEN_THROW("No solutions returned for Legacy Solvers."); - } - - // tunable solver, not legacy - std::vector GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - std::true_type, - std::false_type) const - { - return miopen::solver::GetAllSolutions(value, ctx, problem); - } - - // non tunable solver - std::vector GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - std::false_type, - std::true_type) const - { - std::vector solutions; - solutions.push_back(value.GetSolution(ctx, problem)); - return solutions; - } - std::vector GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - std::false_type, - std::false_type) const - { - std::vector solutions; - solutions.push_back(value.GetSolution(ctx, problem)); - return solutions; - } - - std::vector - GetAllSolutions(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - return GetAllSolutions(ctx, - problem, - std::integral_constant(), - std::integral_constant()); - } - - AnySolver_tmpl(T obj) : value(std::move(obj)){}; - - bool IsApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - return value.IsApplicable(ctx, problem); - } - bool IsTunable() const override { return TunableSolver::Is; } - bool IsDynamic() const override { return value.IsDynamic(); } - float GetWti(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - return value.GetWti(ctx, problem); - } - - ConvSolution FindSolution(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db, - const miopen::AnyInvokeParams& invoke_ctx, - const std::string& perf_cfg) const override - { - return miopen::solver::FindSolution(value, ctx, problem, db, invoke_ctx, perf_cfg); - }; - - InvokerFactory GetInvokeFactory(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& perf_cfg) const override - { - return miopen::solver::GetInvokeFactory(value, ctx, problem, perf_cfg); - } - - std::string GetPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db, - std::true_type) const - { - using PerformanceConfig = decltype(value.GetDefaultPerformanceConfig(ctx, problem)); - PerformanceConfig config{}; - if(db.Load(problem, value.SolverDbId(), config)) - { - MIOPEN_LOG_I2("PerformanceDb: Record Loaded: " << value.SolverDbId()); - if(value.IsValidPerformanceConfig(ctx, problem, config)) - { - return config.ToString(); - } - MIOPEN_LOG_I2("PerformanceDb: Invalid Config: " << value.SolverDbId()); - } - else if(!value.AltSolverDbId().empty() && - db.Load(problem, value.AltSolverDbId(), config)) - { - MIOPEN_LOG_I("PerformanceDb: alternate record loaded: " << value.AltSolverDbId()); - if(value.IsValidPerformanceConfig(ctx, problem, config)) - { - return config.ToString(); - } - MIOPEN_LOG_I2("PerformanceDb: Invalid alternate record: " << value.AltSolverDbId() - << ": " << config); - } - - MIOPEN_LOG_I2("PerformanceDb: Failed Loading, Using Default: " << value.SolverDbId()); - config = value.GetDefaultPerformanceConfig(ctx, problem); - return config.ToString(); - } - std::string GetPerfCfgParams(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceDb&, - std::false_type) const - { - MIOPEN_LOG_I2("PerformanceDb: No Config: " << value.SolverDbId()); - return ""; - } - - std::string GetPerfCfgParams(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - PerformanceDb& db) const override - { - return GetPerfCfgParams( - ctx, problem, db, std::integral_constant()); - } - - size_t GetWorkspaceSize(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - return value.GetWorkspaceSize(ctx, problem); - } - bool MayNeedWorkspace() const override { return value.MayNeedWorkspace(); } - const std::type_info& Type() const override { return typeid(T); }; - std::string GetSolverDbId() const override { return value.SolverDbId(); } - - private: - T value; - }; - - AnySolver_base::ptr ptr_value; -}; - -} // namespace solver -} // namespace miopen - -#endif diff --git a/src/include/miopen/batchnorm/problem_description.hpp b/src/include/miopen/batchnorm/problem_description.hpp index b87494b725..e69de29bb2 100644 --- a/src/include/miopen/batchnorm/problem_description.hpp +++ b/src/include/miopen/batchnorm/problem_description.hpp @@ -1,252 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include - -#include -#include - -namespace miopen { - -struct NetworkConfig; -struct ExecutionContext; - -namespace batchnorm { - -enum class Direction -{ - ForwardTraining, - ForwardInference, - Backward, -}; - -struct ProblemDescriptionTag -{ -}; - -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, ProblemDescriptionTag -{ - // Forward Training - ProblemDescription(miopenBatchNormMode_t bn_mode_, - const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - const TensorDescriptor& bnScaleBiasMeanVarDesc_, - double expAvgFactor_, - double epsilon_, - bool resultsave_, - bool resultrunning_) - : direction(Direction::ForwardTraining), - bn_mode(bn_mode_), - xDesc(xDesc_), - yOrDyDesc(yDesc_), - scaleBiasDesc(bnScaleBiasMeanVarDesc_), - expAvgFactor(expAvgFactor_), - epsilon(epsilon_), - resultsave(resultsave_), - resultrunning(resultrunning_) - { - SetSpatialDims(); - in_layout = ComputeInLayout(); - out_layout = ComputeOutLayout(); - } - - // Forward Inference - ProblemDescription(miopenBatchNormMode_t bn_mode_, - const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - const TensorDescriptor& bnScaleBiasMeanVarDesc_, - double epsilon_) - : direction(Direction::ForwardInference), - bn_mode(bn_mode_), - xDesc(xDesc_), - yOrDyDesc(yDesc_), - scaleBiasDesc(bnScaleBiasMeanVarDesc_), - epsilon(epsilon_) - { - SetSpatialDims(); - in_layout = ComputeInLayout(); - out_layout = ComputeOutLayout(); - } - - // Backward - ProblemDescription(miopenBatchNormMode_t bn_mode_, - const TensorDescriptor& xDesc_, - const TensorDescriptor& dyDesc_, - const TensorDescriptor& dxDesc_, - const TensorDescriptor& bnScaleBiasDiffDesc_, - double epsilon_, - bool useSaved_) - : direction(Direction::Backward), - bn_mode(bn_mode_), - xDesc(xDesc_), - yOrDyDesc(dyDesc_), - dxDesc(dxDesc_), - scaleBiasDesc(bnScaleBiasDiffDesc_), - epsilon(epsilon_), - useSaved(useSaved_) - { - SetSpatialDims(); - in_layout = ComputeInLayout(); - out_layout = ComputeOutLayout(); - din_layout = ComputeDinLayout(); - } - - void SetSpatialDims() - { - if(Is2D()) - spatial_dim = 2; - else if(Is3D()) - spatial_dim = 3; - else - MIOPEN_THROW("Unknown spatial dim!"); - } - Direction GetDirection() const { return direction; } - miopenBatchNormMode_t GetMode() const { return bn_mode; } - const TensorDescriptor& GetXDesc() const { return xDesc; } - - const TensorDescriptor& GetYDesc() const - { - assert(direction == Direction::ForwardTraining || direction == Direction::ForwardInference); - return yOrDyDesc; - } - - const TensorDescriptor& GetDYDesc() const - { - assert(direction == Direction::Backward); - return yOrDyDesc; - } - - const TensorDescriptor& GetDXDesc() const - { - assert(direction == Direction::Backward); - return dxDesc; - } - - const TensorDescriptor& GetBnScaleBiasMeanVarDesc() const - { - assert(direction == Direction::ForwardTraining || direction == Direction::ForwardInference); - return scaleBiasDesc; - } - - const TensorDescriptor& GetScaleBiasDiffDesc() const - { - assert(direction == Direction::Backward); - return scaleBiasDesc; - } - - bool GetResultSave() const - { - assert(direction == Direction::ForwardTraining); - return resultsave; - } - - bool GetResultRunning() const - { - assert(direction == Direction::ForwardTraining); - return resultrunning; - } - - bool UseSaved() const - { - assert(direction == Direction::Backward); - return useSaved; - } - - bool IsLayoutNHWC() const - { - if(direction == Direction::Backward) - { - return xDesc.GetLengths().size() == 4 - ? ((in_layout == "NHWC") && (out_layout == "NHWC") && (din_layout == "NHWC")) - : ((in_layout == "NDHWC") && (out_layout == "NDHWC") && - (din_layout == "NDHWC")); - } - - return xDesc.GetLengths().size() == 4 ? ((in_layout == "NHWC") && (out_layout == "NHWC")) - : ((in_layout == "NDHWC") && (out_layout == "NDHWC")); - } - - bool Is2D() const { return xDesc.GetLengths().size() == 4; } - bool Is3D() const { return xDesc.GetLengths().size() == 5; } - - bool IsFp64() const { return xDesc.GetType() == miopenDouble; } - bool IsFp32() const { return xDesc.GetType() == miopenFloat; } - bool IsFp16() const { return xDesc.GetType() == miopenHalf; } - bool IsBfp16() const { return xDesc.GetType() == miopenBFloat16; } - - NetworkConfig MakeNetworkConfig() const override; - - // This declaration marks batchnorm as a primitive with tuning enabled. - // Any tunable solver would be able pick it and fetch a db instance in ExecutePrimitive. - // It has to be discoverable via ADL from problem description. - friend auto GetDb(const ExecutionContext& ctx, const ProblemDescriptionTag&) -> PerformanceDb; - -private: - Direction direction; - miopenBatchNormMode_t bn_mode; - TensorDescriptor xDesc; // input - TensorDescriptor yOrDyDesc; // output - TensorDescriptor dxDesc; - TensorDescriptor scaleBiasDesc; - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-private-field" -#endif - - double expAvgFactor = 0; - double epsilon; - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - - bool resultsave = false; - bool resultrunning = false; - bool useSaved = false; - std::string in_layout = "NCHW"; - std::string out_layout = "NCHW"; - std::string din_layout = "NCHW"; - std::size_t spatial_dim = 2; - - NetworkConfig MakeForwardTrainingNetworkConfig() const; - NetworkConfig MakeForwardInferenceNetworkConfig() const; - NetworkConfig MakeBackwardNetworkConfig() const; - - std::string ComputeLayout(const TensorDescriptor& td) const { return td.GetLayout_str(); } - std::string ComputeInLayout() const { return ComputeLayout(xDesc); } - std::string ComputeOutLayout() const { return ComputeLayout(yOrDyDesc); } - std::string ComputeDinLayout() const { return ComputeLayout(dxDesc); } -}; - -} // namespace batchnorm - -} // namespace miopen diff --git a/src/include/miopen/conv/heuristics/ai_heuristics.hpp b/src/include/miopen/conv/heuristics/ai_heuristics.hpp index 2d240c0814..e69de29bb2 100644 --- a/src/include/miopen/conv/heuristics/ai_heuristics.hpp +++ b/src/include/miopen/conv/heuristics/ai_heuristics.hpp @@ -1,101 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef GAURD_MIOPEN_AI_HEURISTICS_HPP_ -#define GAURD_MIOPEN_AI_HEURISTICS_HPP_ - -#include -#if MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK || MIOPEN_ENABLE_AI_KERNEL_TUNING -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace miopen { -namespace ai { -#if MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK -namespace immed_mode { -struct Metadata -{ -private: - nlohmann::json json; - const std::unordered_map direction_encodings; - const std::unordered_map precision_encodings; - const std::unordered_map layout_encodings; - -public: - const std::vector features; - const size_t num_inputs; - const size_t num_outputs; - const size_t num_solvers; - const std::unordered_map solver_map; - const std::vector features_mean; - const std::vector features_std; - const std::vector test_features_mean; - const std::vector test_features_std; - Metadata(const std::string& arch); - size_t EncodeDirection(miopen::conv::Direction dir) const; - size_t EncodePrecision(miopenDataType_t data_type) const; - size_t EncodeLayout(const std::string& layout) const; -}; -class Model; -MIOPEN_INTERNALS_EXPORT std::vector PredictSolver(const conv::ProblemDescription& problem, - const ExecutionContext& ctx, - const std::string& device); -} // namespace immed_mode - -#endif // MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK -#if MIOPEN_ENABLE_AI_KERNEL_TUNING -namespace tuning { -struct Metadata -{ - std::size_t predict_type; - std::unordered_map num_tuning_params; - std::unordered_map tuning_decodings; - Metadata(const std::string& arch, const std::string& solver); -}; - -bool ModelSetParams(const std::string& arch, - const std::string& solver, - conv::Direction direction, - const std::vector& features, - bool transform_features, - std::function validator); -} // namespace tuning -#endif // MIOPEN_ENABLE_AI_KERNEL_TUNING -} // namespace ai -} // namespace miopen -#endif -#endif // GAURD_MIOPEN_AI_HEURISTICS_HPP_ diff --git a/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp b/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp index 3248989c4b..e69de29bb2 100644 --- a/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp +++ b/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp @@ -1,228 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace miopen { -namespace conv { - -template -inline std::vector -ComputeDynamicIGemmForwardKernelArgs(const ProblemDescription& problem, const T& cfg); - -template <> -inline std::vector -ComputeDynamicIGemmForwardKernelArgs(const ProblemDescription& problem, const int& cfg) -{ - std::vector opArgs; - // clang-format off - int hi = problem.GetInHeight(); - int wi = problem.GetInWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetOutChannels(); - int c = problem.GetInChannels(); - int ho = problem.GetOutHeight(); - int wo = problem.GetOutWidth(); - int stride_h = problem.GetKernelStrideH(); - int stride_w = problem.GetKernelStrideW(); - int dilation_h = problem.GetDilationH(); - int dilation_w = problem.GetDilationW(); - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int pack0 = cfg; - // clang-format on - - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n); - opArgs.emplace_back(k); - opArgs.emplace_back(c); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_h); - opArgs.emplace_back(stride_w); - opArgs.emplace_back(dilation_h); - opArgs.emplace_back(dilation_w); - opArgs.emplace_back(pad_h); - opArgs.emplace_back(pad_w); - opArgs.emplace_back(y); - opArgs.emplace_back(x); - opArgs.emplace_back(pack0); - - return opArgs; -} - -template <> -inline std::vector -ComputeDynamicIGemmForwardKernelArgs( - const ProblemDescription& problem, const solver::TunableImplicitGemmGTCDynamic_t& cfg) -{ - std::vector opArgs; - // clang-format off - int hi = problem.GetInHeight(); - int wi = problem.GetInWidth(); - int n = problem.GetInBatchSize(); - int k = problem.GetOutChannels(); - int c = problem.GetInChannels(); - int ho = problem.GetOutHeight(); - int wo = problem.GetOutWidth(); - int stride_h = problem.GetKernelStrideH(); - int stride_w = problem.GetKernelStrideW(); - int dilation_h = problem.GetDilationH(); - int dilation_w = problem.GetDilationW(); - int pad_h = problem.GetPadH(); - int pad_w = problem.GetPadW(); - int y = problem.GetWeightsHeight(); - int x = problem.GetWeightsWidth(); - int group = problem.GetGroupCount(); - int pack0 = 0; - // clang-format on - - int gemm_m = - ((k / group + cfg.gemm_m_per_block - 1) / cfg.gemm_m_per_block) * cfg.gemm_m_per_block; - int nxe = cfg.nxe; - int nxb = cfg.nxb; - int b = - nxe == 0 ? (ho * wo) : ((ho * wo + nxb - 1) / nxb) * nxb; // pad to nxb modulo when nxe != 0 - - // init magic division parameters - uint32_t nb_n0 = cfg.tensor_b_cluster_lengths[2] * cfg.tensor_b_thread_lengths[2]; - uint32_t nb_n1b = cfg.tensor_b_cluster_lengths[3] * cfg.tensor_b_thread_lengths[3]; - uint32_t unmerge_sub_n = cfg.gemm_n_per_block / nxb; - uint32_t unmerge_sub_n1 = unmerge_sub_n / nb_n0; - - magic_div_u32_t mdiv_0 = magic_div_u32_gen(gemm_m / cfg.gemm_m_per_block); - magic_div_u32_t mdiv_1 = magic_div_u32_gen(b * unmerge_sub_n1 / nb_n1b); - magic_div_u32_t mdiv_2 = magic_div_u32_gen(y * x); - magic_div_u32_t mdiv_3 = magic_div_u32_gen(x); - magic_div_u32_t mdiv_4 = magic_div_u32_gen(b); - magic_div_u32_t mdiv_5 = magic_div_u32_gen(wo); - magic_div_u32_t mdiv_6 = - magic_div_u32_gen((n * b * (gemm_m)) / (cfg.gemm_m_per_block * cfg.gemm_n_per_block)); - - uint32_t magic_0 = mdiv_0.magic; - uint32_t magic_1 = mdiv_1.magic; - uint32_t magic_2 = mdiv_2.magic; - uint32_t magic_3 = mdiv_3.magic; - uint32_t magic_4 = mdiv_4.magic; - uint32_t magic_5 = mdiv_5.magic; - uint32_t magic_6 = mdiv_6.magic; - uint32_t shift_pack_0 = - magic_div_u32_pack_shift(mdiv_0.shift, mdiv_1.shift, mdiv_2.shift, mdiv_3.shift); - uint32_t shift_pack_1 = magic_div_u32_pack_shift(mdiv_4.shift, mdiv_5.shift, mdiv_6.shift, 0); - - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(0); // placeholder - opArgs.emplace_back(hi); - opArgs.emplace_back(wi); - opArgs.emplace_back(n); - opArgs.emplace_back(k); - opArgs.emplace_back(c); - opArgs.emplace_back(ho); - opArgs.emplace_back(wo); - opArgs.emplace_back(stride_h); - opArgs.emplace_back(stride_w); - opArgs.emplace_back(dilation_h); - opArgs.emplace_back(dilation_w); - opArgs.emplace_back(pad_h); - opArgs.emplace_back(pad_w); - opArgs.emplace_back(y); - opArgs.emplace_back(x); - opArgs.emplace_back(group); - opArgs.emplace_back(magic_0); - opArgs.emplace_back(magic_1); - opArgs.emplace_back(magic_2); - opArgs.emplace_back(magic_3); - opArgs.emplace_back(magic_4); - opArgs.emplace_back(magic_5); - opArgs.emplace_back(magic_6); - opArgs.emplace_back(shift_pack_0); - opArgs.emplace_back(shift_pack_1); - - opArgs.emplace_back(pack0); - - return opArgs; -} - -template -static inline InvokerFactory -MakeImplGemmDynamicForwardInvokerFactory(const ProblemDescription& problem, const T& cfg) -{ - auto opArgs = ComputeDynamicIGemmForwardKernelArgs(problem, cfg); - return [opArgs](const std::vector& kernels) mutable { - return [=](const Handle& handle, const AnyInvokeParams& primitive_parameters) mutable { - decltype(auto) data_ctx = primitive_parameters.CastTo(); - const auto& tensors = data_ctx.tensors; - const auto k = handle.Run(kernels[0]); - - opArgs[0] = OpKernelArg(tensors.in); - opArgs[1] = OpKernelArg(tensors.w); - opArgs[2] = OpKernelArg(tensors.out); - - k(opArgs); - }; - }; -} - -InvokerFactory MakeImplGemmDynamicForward1x1InvokerFactory(const ProblemDescription& problem); - -InvokerFactory MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescription& problem, - int cfg); - -InvokerFactory -MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescription& problem, - const solver::TunableImplicitGemmGTCDynamic_t& cfg); - -InvokerFactory MakeImplGemmDynamicForwardXdlopsNHWCInvokerFactory( - const ExecutionContext& ctx, - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC& config); -InvokerFactory MakeImplGemmDynamicBackwardDataXdlopsNHWCInvokerFactory( - const ExecutionContext& ctx, - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC& config); -InvokerFactory MakeImplGemmDynamicForwardDlopsNCHWCInvokerFactory( - const ProblemDescription& problem, - const solver::conv::PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC& config); - -} // namespace conv -} // namespace miopen diff --git a/src/include/miopen/conv/problem_description.hpp b/src/include/miopen/conv/problem_description.hpp index 5d1d89ce23..e69de29bb2 100644 --- a/src/include/miopen/conv/problem_description.hpp +++ b/src/include/miopen/conv/problem_description.hpp @@ -1,463 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#if MIOPEN_ENABLE_SQLITE -#include -#endif - -namespace miopen { - -struct ExecutionContext; - -MIOPEN_INTERNALS_EXPORT std::string -EncodeDataTypesForKey(miopenDataType_t in, miopenDataType_t weights, miopenDataType_t out); - -template -constexpr auto GetDHW(unsigned spatial_dims, const std::vector& data) -{ - if(spatial_dims == 2) - return std::make_tuple(0, data[0], data[1]); - return std::make_tuple(data[0], data[1], data[2]); -} - -template -constexpr TElement GetD3(unsigned spatial_dims, const std::vector& data) -{ - return std::get<0>(GetDHW(spatial_dims, data)); -} - -template -constexpr TElement GetH3(unsigned spatial_dims, const std::vector& data) -{ - return std::get<1>(GetDHW(spatial_dims, data)); -} - -template -constexpr TElement GetW3(unsigned spatial_dims, const std::vector& data) -{ - return std::get<2>(GetDHW(spatial_dims, data)); -} -template -constexpr auto GetCHWN(const std::vector& data) -{ - return miopen::tien<4>(data, 1); -} - -template -constexpr TElement GetNofCHWN(const std::vector& data) -{ - return std::get<3>(GetCHWN(data)); -} - -template -constexpr TElement GetCofCHWN(const std::vector& data) -{ - return std::get<0>(GetCHWN(data)); -} - -template -constexpr TElement GetHofCHWN(const std::vector& data) -{ - return std::get<1>(GetCHWN(data)); -} - -template -constexpr TElement GetWofCHWN(const std::vector& data) -{ - return std::get<2>(GetCHWN(data)); -} - -template -constexpr TElement GetN5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<0>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetC5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<1>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetD5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<2>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetH5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<3>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetW5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<4>(GetNCDHW(spatial_dims, data)); -} - -namespace conv { - -MIOPEN_INTERNALS_EXPORT miopenAlphaBetaCase_t ClassifyAlphaBeta(const Scalar& alpha, - const Scalar& beta); - -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase -#if MIOPEN_ENABLE_SQLITE - , - SQLiteSerializable -#endif -{ - ProblemDescription() = default; - - /// \todo Get rid of the swapping of x and y. - ProblemDescription(const TensorDescriptor& in_, // x for Forward, y for Backward* - const TensorDescriptor& weights_, - const TensorDescriptor& out_, // y for Forward, x for Backward* - const ConvolutionDescriptor& conv_, - Direction direction_, - int bias_ = 0, - const Scalar& alpha_ = Scalar(1.0), - const Scalar& beta_ = Scalar(0.0)) - : in(in_), - weights(weights_), - out(out_), - conv(conv_), - in_layout(ComputeInLayout()), - weights_layout(ComputeWeightsLayout()), - out_layout(ComputeOutLayout()), - direction(direction_), - bias(bias_), - alpha(alpha_), - beta(beta_), - alpha_beta_case(ClassifyAlphaBeta(alpha, beta)) - { - } - - // Conv descriptor getters - unsigned GetSpatialDims() const { return conv.GetSpatialDimension(); } - int GetPadD() const { return GetD3(GetSpatialDims(), conv.GetConvPads()); } - int GetPadH() const { return GetH3(GetSpatialDims(), conv.GetConvPads()); } - int GetPadW() const { return GetW3(GetSpatialDims(), conv.GetConvPads()); } - int GetKernelStrideD() const { return GetD3(GetSpatialDims(), conv.GetConvStrides()); } - int GetKernelStrideH() const { return GetH3(GetSpatialDims(), conv.GetConvStrides()); } - int GetKernelStrideW() const { return GetW3(GetSpatialDims(), conv.GetConvStrides()); } - int GetDilationD() const { return GetD3(GetSpatialDims(), conv.GetConvDilations()); } - int GetDilationH() const { return GetH3(GetSpatialDims(), conv.GetConvDilations()); } - int GetDilationW() const { return GetW3(GetSpatialDims(), conv.GetConvDilations()); } - int GetGroupCount() const { return conv.GetGroupCount(); } - int GetVectorLength() const { return in.GetVectorLength(); } - - // In getters - miopenDataType_t GetInDataType() const { return in.GetType(); } - std::optional GetInCastType() const { return in.GetCastType(); } - std::size_t GetInBatchSize() const { return GetN5(GetSpatialDims(), in.GetLengths()); } - std::size_t GetBatchSize() const { return GetInBatchSize(); } // alias of GetInBatchSize() - std::size_t GetInChannels() const { return GetC5(GetSpatialDims(), in.GetLengths()); } - std::size_t GetInDepth() const { return GetD5(GetSpatialDims(), in.GetLengths()); } - std::size_t GetInHeight() const { return GetH5(GetSpatialDims(), in.GetLengths()); } - std::size_t GetInWidth() const { return GetW5(GetSpatialDims(), in.GetLengths()); } - std::size_t GetInBatchStride() const { return GetN5(GetSpatialDims(), in.GetStrides()); } - std::size_t GetInChannelStride() const { return GetC5(GetSpatialDims(), in.GetStrides()); } - std::size_t GetInStrideD() const { return GetD5(GetSpatialDims(), in.GetStrides()); } - std::size_t GetInStrideH() const { return GetH5(GetSpatialDims(), in.GetStrides()); } - std::size_t GetInStrideW() const { return GetW5(GetSpatialDims(), in.GetStrides()); } - std::string GetInLayout() const { return in_layout; } - std::size_t GetInElementSize() const { return GetTypeSize(GetInDataType()); } - std::size_t GetInSize() const { return in.GetNumBytes(); } - - // Out getters - miopenDataType_t GetOutDataType() const { return out.GetType(); } - std::optional GetOutCastType() const { return out.GetCastType(); } - std::size_t GetOutBatchSize() const { return GetN5(GetSpatialDims(), out.GetLengths()); } - std::size_t GetOutChannels() const { return GetC5(GetSpatialDims(), out.GetLengths()); } - std::size_t GetOutDepth() const { return GetD5(GetSpatialDims(), out.GetLengths()); } - std::size_t GetOutHeight() const { return GetH5(GetSpatialDims(), out.GetLengths()); } - std::size_t GetOutWidth() const { return GetW5(GetSpatialDims(), out.GetLengths()); } - std::size_t GetOutBatchStride() const { return GetN5(GetSpatialDims(), out.GetStrides()); } - std::size_t GetOutChannelStride() const { return GetC5(GetSpatialDims(), out.GetStrides()); } - std::size_t GetOutStrideD() const { return GetD5(GetSpatialDims(), out.GetStrides()); } - std::size_t GetOutStrideH() const { return GetH5(GetSpatialDims(), out.GetStrides()); } - std::size_t GetOutStrideW() const { return GetW5(GetSpatialDims(), out.GetStrides()); } - std::string GetOutLayout() const { return out_layout; } - std::size_t GetOutElementSize() const { return GetTypeSize(GetOutDataType()); } - std::size_t GetOutSize() const { return out.GetNumBytes(); } - - // Weights getters - miopenDataType_t GetWeightsDataType() const { return weights.GetType(); } - std::optional GetWeightsCastType() const { return weights.GetCastType(); } - std::size_t GetWeightsDepth() const { return GetD5(GetSpatialDims(), weights.GetLengths()); } - std::size_t GetWeightsHeight() const - { - if(weights_layout == "CHWNc") - return GetHofCHWN(weights.GetLengths()); - else - return GetH5(GetSpatialDims(), weights.GetLengths()); - } - std::size_t GetWeightsWidth() const - { - if(weights_layout == "CHWNc") - return GetWofCHWN(weights.GetLengths()); - else - return GetW5(GetSpatialDims(), weights.GetLengths()); - } - std::size_t GetWeightsStrideK() const { return GetN5(GetSpatialDims(), weights.GetStrides()); } - std::size_t GetWeightsStrideC() const { return GetC5(GetSpatialDims(), weights.GetStrides()); } - std::size_t GetWeightsStrideD() const { return GetD5(GetSpatialDims(), weights.GetStrides()); } - std::size_t GetWeightsStrideH() const { return GetH5(GetSpatialDims(), weights.GetStrides()); } - std::size_t GetWeightsStrideW() const { return GetW5(GetSpatialDims(), weights.GetStrides()); } - std::string GetWeightsLayout() const { return weights_layout; } - std::size_t GetWeightsElementSize() const { return GetTypeSize(GetWeightsDataType()); } - std::size_t GetWeightsSize() const { return weights.GetNumBytes(); } - - const TensorDescriptor& GetIn() const { return in; } - const TensorDescriptor& GetWeights() const { return weights; } - const TensorDescriptor& GetOut() const { return out; } - const ConvolutionDescriptor& GetConv() const { return conv; } - - Direction GetDirection() const { return direction; } - bool IsDirectionForward() const { return direction == conv::Direction::Forward; } - bool IsDirectionBackwardData() const { return direction == conv::Direction::BackwardData; } - bool IsDirectionBackwardWrW() const { return direction == conv::Direction::BackwardWeights; } - std::string GetDirectionStr() const; - - const Scalar& GetAlpha() const { return alpha; } - const Scalar& GetBeta() const { return beta; } - - miopenAlphaBetaCase_t GetAlphaBetaCase() const { return alpha_beta_case; } - - std::string GetAlphaBetaCaseStr() const; - - int GetBias() const { return bias; } - - std::size_t GetBiasSize() const - { - return (GetBias() != 0) ? (GetOutChannels() * GetOutElementSize()) : 0; - } - - int64_t GetBackwardPadW() const - { - return static_cast(GetWeightsWidth()) - GetPadW() - 1; - } - int64_t GetBackwardPadH() const - { - return static_cast(GetWeightsHeight()) - GetPadH() - 1; - } - - bool IsAsymmetricPadH() const - { - return conv.paddingMode == miopenPaddingSame && (GetWeightsHeight() % 2) == 0; - } - bool IsAsymmetricPadW() const - { - return conv.paddingMode == miopenPaddingSame && (GetWeightsWidth() % 2) == 0; - } - - bool Is2d() const { return GetSpatialDims() == 2; } - bool Is3d() const { return GetSpatialDims() == 3; } - - bool IsFp32() const - { - return GetInDataType() == miopenFloat && GetWeightsDataType() == miopenFloat && - GetOutDataType() == miopenFloat; - } - bool IsFp16() const - { - return GetInDataType() == miopenHalf && GetWeightsDataType() == miopenHalf && - GetOutDataType() == miopenHalf; - } - bool IsBfp16() const - { - return GetInDataType() == miopenBFloat16 && GetWeightsDataType() == miopenBFloat16 && - GetOutDataType() == miopenBFloat16; - } - bool IsInt8() const - { - return GetInDataType() == miopenInt8 && GetWeightsDataType() == miopenInt8 && - (GetOutDataType() == miopenInt32 || GetOutDataType() == miopenFloat); - } - bool IsFp8() const - { - return GetInDataType() == miopenFloat8 || GetWeightsDataType() == miopenFloat8 || - GetOutDataType() == miopenFloat8; - } - bool IsBfp8() const - { - return GetInDataType() == miopenBFloat8 || GetWeightsDataType() == miopenBFloat8 || - GetOutDataType() == miopenBFloat8; - } - bool IsTensorsCasted() const - { - return GetInCastType() || GetWeightsCastType() || GetOutCastType(); - } - - // To be used in Solvers that do not implement ALT FP16 kernels. - // Those Solvers must be non-applicable for gfx90a when this function returns true. - bool IsGfx90aFp16altRequired() const - { - if(!IsFp16()) - return false; - if(direction == conv::Direction::Forward) - return conv.attribute.gfx90aFp16alt.GetFwd(); - if(direction == conv::Direction::BackwardData) - return conv.attribute.gfx90aFp16alt.GetBwd(); - if(direction == conv::Direction::BackwardWeights) - return conv.attribute.gfx90aFp16alt.GetWrW(); - MIOPEN_THROW("Direction must be known!"); - } - - bool IsLayoutDefault() const; - bool IsLayoutNHWC() const; - bool IsLayoutNCHWc() const; - bool IsNCHWc_NCHWc() const; - bool IsNCHWc_CHWNc() const; - - bool HasNonPackedTensors() const - { - return !(in.IsPacked() && weights.IsPacked() && out.IsPacked()); - } - - bool HasMixedDataTypes() const - { - return !(GetInDataType() == GetWeightsDataType() && - GetWeightsDataType() == GetOutDataType()); - } - - bool AllTensorsDimsFitIntoInt() const - { - return in.AllDimsFitIntoInt() && weights.AllDimsFitIntoInt() && out.AllDimsFitIntoInt(); - } - - bool AllTensorsLengthsFitIntoInt() const - { - return in.AllLengthsFitIntoInt() && weights.AllLengthsFitIntoInt() && - out.AllLengthsFitIntoInt(); - } - - void MakeNetworkConfig(std::string& conf_key) const; - - NetworkConfig MakeNetworkConfig() const override - { - std::string ret; - MakeNetworkConfig(ret); - return NetworkConfig{ret}; - } - - // Todo: remove after fixing fin - [[deprecated]] NetworkConfig BuildConfKey() const { return MakeNetworkConfig(); } - - void Serialize(std::ostream& stream) const; - - friend std::ostream& operator<<(std::ostream& os, const ProblemDescription& obj) - { - obj.Serialize(os); - return os; - } - -#if MIOPEN_ENABLE_SQLITE - static std::string table_name() { return "config"; } -#endif - - template - static void Visit(Self&& self, std::function f) - { - // The column names match the driver command line argument names - f(self.GetSpatialDims(), "spatial_dim"); - f(self.GetInChannels(), "in_channels"); - f(self.GetInHeight(), "in_h"); - f(self.GetInWidth(), "in_w"); - f(self.GetInDepth(), "in_d"); - f(self.GetWeightsHeight(), "fil_h"); - f(self.GetWeightsWidth(), "fil_w"); - f(self.GetWeightsDepth(), "fil_d"); - f(self.GetOutChannels(), "out_channels"); - f(self.GetBatchSize(), "batchsize"); - f(self.GetPadH(), "pad_h"); - f(self.GetPadW(), "pad_w"); - f(self.GetPadD(), "pad_d"); - f(self.GetKernelStrideH(), "conv_stride_h"); - f(self.GetKernelStrideW(), "conv_stride_w"); - f(self.GetKernelStrideD(), "conv_stride_d"); - f(self.GetDilationH(), "dilation_h"); - f(self.GetDilationW(), "dilation_w"); - f(self.GetDilationD(), "dilation_d"); - f(self.GetBias(), "bias"); - f(self.GetGroupCount(), "group_count"); - } - - template - static void Visit(Self&& self, std::function f) - { - f(self.GetInLayout(), "layout"); - std::string data_type = EncodeDataTypesForKey( - self.GetInDataType(), self.GetWeightsDataType(), self.GetOutDataType()); - f(data_type, "data_type"); - f(self.GetDirectionStr(), "direction"); - } - - template - static void VisitAll(Self&& self, const Visitor& f) - { - Visit(std::forward(self), [&](int64_t value, std::string name) { f(value, name); }); - Visit(std::forward(self), - [&](std::string value, std::string name) { f(value, name); }); - } - - void SetupFloats(ExecutionContext& ctx) const; - -private: - std::string ComputeLayout(const TensorDescriptor& td) const; - std::string ComputeInLayout() const; - std::string ComputeOutLayout() const; - std::string ComputeWeightsLayout() const; - - TensorDescriptor in; - TensorDescriptor weights; - TensorDescriptor out; - ConvolutionDescriptor conv; - std::string in_layout; - std::string weights_layout; - std::string out_layout; - Direction direction = Direction::Forward; - int bias = 0; - Scalar alpha = Scalar(1.0); - Scalar beta = Scalar(0.0); - miopenAlphaBetaCase_t alpha_beta_case = DEFAULT; -}; - -} // namespace conv -} // namespace miopen diff --git a/src/include/miopen/conv/solvers.hpp b/src/include/miopen/conv/solvers.hpp index 699f267b99..e69de29bb2 100644 --- a/src/include/miopen/conv/solvers.hpp +++ b/src/include/miopen/conv/solvers.hpp @@ -1,5122 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace miopen { - -namespace debug { - -/// If set to true, then always enable ConvDirectNaive* solver, regardless of environment value -/// MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_* that control enable/disable of these solvers. -/// Currently used during driver using naive kernel as gpu reference. -MIOPEN_EXPORT extern bool - AlwaysEnableConvDirectNaive; // NOLINT (cppcoreguidelines-avoid-non-const-global-variables) - -} // namespace debug - -struct AnyInvokeParams; - -namespace solver { -/// \todo Move wave_size into abstraction wich represent GPU information -const int wave_size = 64; - -namespace conv { - -/// Base class for convolution tunable and non-tunable solvers -using ConvSolverBase = SolverMixin; - -/// Typedef for convolution non-tunable solvers -using ConvSolver = NonTunableSolverBase; - -/// Typedef for convolution tunable solvers -template -using ConvTunableSolver = - TunableSolverMixin; - -struct PerformanceConfigConvAsm3x3U : PerfConfigBase -{ - int limit_wave_cnt; // [0..9] - int filters_per_wave; // [1..8] - int output_lines_per_wave; // [1..8] - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U(int lwc, int fpw, int olpw); - PerformanceConfigConvAsm3x3U() : PerformanceConfigConvAsm3x3U(-1, -1, -1) {} - PerformanceConfigConvAsm3x3U(bool) : PerformanceConfigConvAsm3x3U(0, 1, 1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.limit_wave_cnt, "limit_wave_cnt"); - f(self.filters_per_wave, "filters_per_wave"); - f(self.output_lines_per_wave, "output_lines_per_wave"); - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm3x3U& other) const; -}; - -struct ConvAsm3x3U final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm3x3U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm3x3U&) const override; -}; - -struct PerformanceConfigConvAsm1x1U : PerfConfigBase -{ - // ----------------- // Full set Optimized Spare - // ---------------------------------------------------------------------------- - int read_size; // [1..4] - int k_mult; // 1,[4,8,12..32] 2^n[8..32] 1,4 - int chunks_per_wave; // [1..16] [1..8] - int chunk_size; // 2^n[1..64] 2^n[16..64] 1,4 - int n_mult; // [1..8] [1..4] - int c_mult; // 2^n[1..32] 2^n[1..4] - int waves_c_in_group; // [1..8] [1..4] - int waves_k_in_group; // 1,[2,4,8] 1,[2,4,8] - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigConvAsm1x1U(int, int, int, int, int, int, int, int, bool); - PerformanceConfigConvAsm1x1U() - : PerformanceConfigConvAsm1x1U(-1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.read_size, "read_size"); - f(self.k_mult, "k_mult"); - f(self.chunks_per_wave, "chunks_per_wave"); - f(self.chunk_size, "chunk_size"); - f(self.n_mult, "n_mult"); - f(self.c_mult, "c_mult"); - f(self.waves_c_in_group, "waves_c_in_group"); - f(self.waves_k_in_group, "waves_k_in_group"); - } - - // clang-format off - int GetReadSize() const { return read_size; } - int GetKMult() const { return k_mult; } - int GetChunksPerWave() const { return chunks_per_wave; } - int GetChunkSize() const { return chunk_size; } - int GetNMult() const { return n_mult; } - int GetCMult() const { return c_mult; } - int GetWavesCInGroup() const { return waves_c_in_group; } - int GetWavesKInGroup() const { return waves_k_in_group; } - int GetNPerGpr() const { assert(chunk_size); return 64 / chunk_size; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void StaticHeuristic(const miopen::conv::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - bool IsValidValue() const { return IsValidValueImpl(8); } - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - bool IsValid(const miopen::conv::ProblemDescription& problem) const - { - return IsValidImpl(problem, 8); - } - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1U& other) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - bool IsPartiallyValid(const miopen::conv::ProblemDescription& problem, - int sequence_length) const - { - return IsValidImpl(problem, sequence_length); - } - bool IsPartiallyValidValue(int sequence_length) const - { - return IsValidValueImpl(sequence_length); - } - bool RunParameterPredictionModel(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - bool ModelApplyToken(int index, std::string value, const miopen::conv::ProblemDescription&); -#endif - bool IsValidImpl(const miopen::conv::ProblemDescription& problem, int sequence_length) const; - bool IsValidValueImpl(int sequence_length) const; -}; - -struct ConvAsm1x1U final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1U&) const override; -}; - -struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase -{ - // ----------------- // Full set Optimized Spare - // ---------------------------------------------------------------------------- - int chunk_size; // 2^n[1..64] 2^n[16..64] - int dwords_per_ld; // [1..4] 1,2,3 - int k_mult; // [1..32] 8,16 1,2,3,4 - int c_mult; // [1..32] 2^n[1..4] - int n_mult; // [1..32] 1,2 - int w_mult; // [1..32] 1,2 - int h_mult; // [1..32] 1,2 - int h_per_chunk; // 2^n[1..64] [2,4,8] - int waves_k_in_group; // [1..8] 2,4 - int waves_c_in_group; // [1..8] 1,2 - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigConvAsm1x1UV2(int, int, int, int, int, int, int, int, int, int, bool); - PerformanceConfigConvAsm1x1UV2() - : PerformanceConfigConvAsm1x1UV2(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.chunk_size, "chunk_size"); - f(self.dwords_per_ld, "dwords_per_ld"); - f(self.k_mult, "k_mult"); - f(self.c_mult, "c_mult"); - f(self.n_mult, "n_mult"); - f(self.w_mult, "w_mult"); - f(self.h_mult, "h_mult"); - f(self.h_per_chunk, "h_per_chunk"); - f(self.waves_k_in_group, "waves_k_in_group"); - f(self.waves_c_in_group, "waves_c_in_group"); - } - - // clang-format off - int GetChunkSize() const { return chunk_size; } - int GetDwordsPerLd() const { return dwords_per_ld; } - int GetCMult() const { return c_mult; } - int GetKMult() const { return k_mult; } - int GetNMult() const { return n_mult; } - int GetWMult() const { return w_mult; } - int GetHMult() const { return h_mult; } - int GetHPerChunk() const { return h_per_chunk; } - int GetWavesCInGroup() const { return waves_c_in_group; } - int GetWavesKInGroup() const { return waves_k_in_group; } - int GetNPerGpr() const { assert(chunk_size); return 64 / chunk_size; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1UV2& other) const; -}; - -struct ConvAsm1x1UV2 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1UV2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1UV2&) const override; -}; - -struct ConvAsm5x10u2v2f1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsm5x10u2v2b1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsm7x7c3h224w224k64u2v2p3q3f1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclDirectFwd11x11 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclDirectFwdGen final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceImplicitGemm : PerfConfigBase -{ - int BPerBlock; // 2^n[8..16] - int KPerBlock; // 2^n[32..128] - int EPerBlock; // 2^n[4..16] - - int GemmNRepeat; // == 2 - - int GemmMPerThreadSubC; // 2^n[2..4] - int GemmNPerThreadSubC; // 2^n[2..4] - - int GemmMLevel0Cluster; // 2^n[1..4] - int GemmNLevel0Cluster; // 2^n[1..4] - int GemmMLevel1Cluster; // 2^n[1..4] - int GemmNLevel1Cluster; // 2^n[1..4] - - int InBlockCopyClusterLengths_E; // 2^n[4..16] - int InBlockCopyClusterLengths_B; // 2^n[8..16] - int InBlockCopyClusterLengths_N1; // 2^n[1..2] - int InBlockCopyClusterLengths_N2; // 2^n[1..4] - - int WeiBlockCopyClusterLengths_E; // 2^n[1..4] - int WeiBlockCopyClusterLengths_K; // 2^n[16..128] - - bool use_spare_set; - - PerformanceImplicitGemm( - int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, bool); - - PerformanceImplicitGemm() - : PerformanceImplicitGemm( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemm(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.BPerBlock, "BPerBlock"); - f(self.KPerBlock, "KPerBlock"); - f(self.EPerBlock, "EPerBlock"); - f(self.GemmNRepeat, "GemmNRepeat"); - f(self.GemmMPerThreadSubC, "GemmMPerThreadSubC"); - f(self.GemmNPerThreadSubC, "GemmNPerThreadSubC"); - f(self.GemmMLevel0Cluster, "GemmMLevel0Cluster"); - f(self.GemmNLevel0Cluster, "GemmNLevel0Cluster"); - f(self.GemmMLevel1Cluster, "GemmMLevel1Cluster"); - f(self.GemmNLevel1Cluster, "GemmNLevel1Cluster"); - f(self.InBlockCopyClusterLengths_E, "InBlockCopyClusterLengths_E"); - f(self.InBlockCopyClusterLengths_N1, "InBlockCopyClusterLengths_N1"); - f(self.InBlockCopyClusterLengths_B, "InBlockCopyClusterLengths_B"); - f(self.InBlockCopyClusterLengths_N2, "InBlockCopyClusterLengths_N2"); - f(self.WeiBlockCopyClusterLengths_E, "WeiBlockCopyClusterLengths_E"); - f(self.WeiBlockCopyClusterLengths_K, "WeiBlockCopyClusterLengths_K"); - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemm& other) const; -}; - -struct PerformanceImplicitGemmV4R1 : public PerformanceImplicitGemm -{ - PerformanceImplicitGemmV4R1(int a, - int b, - int c, - int d, - int e, - int f, - int g, - int h, - int i, - int j, - int k, - int l, - int m, - int n, - int o, - int p, - bool q) - : PerformanceImplicitGemm(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) - { - } - - PerformanceImplicitGemmV4R1() - : PerformanceImplicitGemmV4R1( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmV4R1(bool spare) : PerformanceImplicitGemm(spare) {} - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmV4R4Fwd(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmV4R4Fwd(a, b, c, d, e, f, false) - { - } - - PerformanceImplicitGemmV4R4Fwd() : PerformanceImplicitGemmV4R4Fwd(-1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4Fwd& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmV4R4WrW(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmV4R4WrW(a, b, c, d, e, f, false) - { - } - - PerformanceImplicitGemmV4R4WrW() : PerformanceImplicitGemmV4R4WrW(-1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4WrW& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmBwdDataV1R1() - : PerformanceImplicitGemmBwdDataV1R1(-1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmBwdDataV1R1(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmBwdDataV1R1(a, b, c, d, e, f, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV1R1& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmBwdDataV4R1() - : PerformanceImplicitGemmBwdDataV4R1(-1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmBwdDataV4R1(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmBwdDataV4R1(a, b, c, d, e, f, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV4R1& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple MIOPEN_INTERNALS_EXPORT - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV4R1Xdlops - : PerfConfigBase -{ - int GemmNPerBlock; // 2^n[8..16] - int GemmMPerBlock; // 2^n[32..128] - int GemmKPerBlock; // 2^n[4..16] - - int GemmKPACKSize; // 2^[1..4] - - int GemmMPerWave; - int GemmNPerWave; - - // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - bool use_spare_set; - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmBwdDataV4R1Xdlops(int, int, int, int, int, int, bool, bool, bool); - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(bool spare); - PerformanceImplicitGemmBwdDataV4R1Xdlops( - int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceImplicitGemmBwdDataV4R1Xdlops(a, b, c, d, e, f, g, h, false) - { - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmBwdDataV4R1Xdlops& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmKPACKSize, "GemmKPACKSize"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct ConvHipImplicitGemmV4R1Fwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmV4R4Fwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4Fwd&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4Fwd&) const override; - -private: - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmV4R4Fwd; -}; - -struct PerformanceConvMlirIgemm : PerfConfigBase -{ - int BlockSize; - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerThread; - int GemmNPerThread; - bool use_spare_set; - - /// \ref https://github.com/ROCm/MIOpen/issues/1154 - static PerformanceConvMlirIgemm& MlirHeuristicInitRequest() - { - static PerformanceConvMlirIgemm heur; - heur.SetMlirHeuristicInitRequest(); - return heur; - } - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(int, int, int, int, int, int, bool); - - PerformanceConvMlirIgemm(int a, int b, int c, int d, int e, int f) - : PerformanceConvMlirIgemm(a, b, c, d, e, f, false) - { - } - - PerformanceConvMlirIgemm() : PerformanceConvMlirIgemm(-1, -1, -1, -1, -1, -1, false) {} - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemm& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - -private: - void SetMlirHeuristicInitRequest(); -}; - -struct ConvMlirIgemmFwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct PerformanceConvMlirIgemmXdlops : PerfConfigBase -{ - int GemmMPerBlock; // 2^n[32..128] - int GemmNPerBlock; // 2^n[8..16] - int GemmKPerBlock; // 2^n[4..16] - int GemmMPerWave; - int GemmNPerWave; - int GemmKPACKSize; // 2^[1..4] - - // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - bool use_spare_set; - - /// \ref https://github.com/ROCm/MIOpen/issues/1154 - static PerformanceConvMlirIgemmXdlops& MlirHeuristicInitRequest() - { - static PerformanceConvMlirIgemmXdlops heur; - heur.SetMlirHeuristicInitRequest(); - return heur; - } - - MIOPEN_INTERNALS_EXPORT - PerformanceConvMlirIgemmXdlops(int, int, int, int, int, int, bool, bool, bool); - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(bool spare); - PerformanceConvMlirIgemmXdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceConvMlirIgemmXdlops(a, b, c, d, e, f, g, h, false) - { - } - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemmXdlops& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPACKSize, "GemmKPACKSize"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - -private: - void SetMlirHeuristicInitRequest(); -}; - -struct ConvMlirIgemmFwdXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct ConvHipImplicitGemmV4R4WrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4WrW&) const override; - -private: - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmV4R4WrW; -}; - -struct ConvMlirIgemmWrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct ConvMlirIgemmWrWXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct PerformanceImplicitGemmForwardV4R4Xdlops - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmForwardV4R4Xdlops(int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops(); - PerformanceImplicitGemmForwardV4R4Xdlops(bool) : PerformanceImplicitGemmForwardV4R4Xdlops() {} - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmForwardV4R5Xdlops - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmForwardV4R5Xdlops(int, int, int, int, int, int, bool, bool, int, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(bool spare); - - PerformanceImplicitGemmForwardV4R5Xdlops( - int a, int b, int c, int d, int e, int f, bool g, bool h, int i) - : PerformanceImplicitGemmForwardV4R5Xdlops(a, b, c, d, e, f, g, h, i, false) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R5Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - int GemmMFactor; - int GemmNFactor; - int GemmKFactor; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm( - int, int, int, int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(); - PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(bool) - : PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm() - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmMFactor, "GemmMFactor"); - f(self.GemmNFactor, "GemmNFactor"); - f(self.GemmKFactor, "GemmKFactor"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmBwdV1R1Xdlops : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmBwdV1R1Xdlops(int, int, int, int, int, int, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops(); - PerformanceImplicitGemmBwdV1R1Xdlops(bool) : PerformanceImplicitGemmBwdV1R1Xdlops() {} - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmBwdV1R1Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmForwardV4R4Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static std::tuple - CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmForwardV4R4Xdlops; -}; - -struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static std::tuple CalculateGemmSize( - const miopen::conv::ProblemDescription&, int GemmMFactor, int GemmNFactor, int GemmKFactor); - - friend struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm; -}; - -struct ConvHipImplicitGemmForwardV4R5Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmV4R1WrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmBwdDataV1R1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - -private: - static std::tuple CalculateGemmSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmBwdDataV1R1; -}; - -struct ConvMlirIgemmBwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct ConvMlirIgemmBwdXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct ConvHipImplicitGemmBwdDataV4R1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1&) const override; - -private: - static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, - int gemm_id); - - friend struct PerformanceImplicitGemmBwdDataV4R1; -}; - -struct ConvHipImplicitGemmBwdDataV4R1Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, - int gemm_id); - - friend struct PerformanceImplicitGemmBwdDataV4R1Xdlops; -}; - -struct ConvHipImplicitGemmBwdDataV1R1Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; - -private: - static std::tuple - CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmBwdV1R1Xdlops; -}; - -struct ConvAsmImplicitGemmV4R1DynamicFwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicFwd_1x1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicWrw final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicWrwXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicBwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicBwdXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -/// Holds common member functions for the Solvers which share the same -/// "legacy exhaustive search" machinery. -struct ConvOclDirectFwdLegacyExhaustiveSearch : ConvTunableSolver -{ - MIOPEN_INTERNALS_EXPORT LegacyPerformanceConfig GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT LegacyPerformanceConfig - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - template - LegacyPerformanceConfig SearchImpl(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const; -}; - -struct ConvOclDirectFwd : ConvOclDirectFwdLegacyExhaustiveSearch -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT static ConvSolution - BaseGetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&); - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; -}; - -struct ConvOclDirectFwd1x1 final : ConvOclDirectFwdLegacyExhaustiveSearch -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; - - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override - { - return true; - } -}; - -struct ConvBinWinograd3x3U final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvBinWinogradRxS final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceConfigConvBinWinogradRxS : PerfConfigBase -{ - int n_groups; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS(int n_groups_); - PerformanceConfigConvBinWinogradRxS() : PerformanceConfigConvBinWinogradRxS(-1) {} - PerformanceConfigConvBinWinogradRxS(bool) : PerformanceConfigConvBinWinogradRxS(1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.n_groups, "n_groups"); - } - int GetNGroups() const { return n_groups; } - - template - void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext& ctx, const miopen::conv::ProblemDescription&) const - { - return IsValid(ctx); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvBinWinogradRxS& other) const; -}; - -template -struct ConvBinWinoRxS final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - static const std::string& GetSolverDbId() - { - static const std::string dbId = std::string("ConvBinWinogradRxSf") - .append(std::to_string(Winodata)) - .append("x") - .append(std::to_string(Winofilter)); - return dbId; - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvBinWinogradRxS&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvBinWinogradRxS&) const override; - -private: - static size_t GetNGroups(const size_t group_conv, const size_t grid_group_size) - { - assert(group_conv != 0); - return grid_group_size / group_conv; - } -}; - -// Suppress misleading clang warnings -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvBinWinoRxS<2, 3>; -extern template struct ConvBinWinoRxS<3, 2>; - -#if defined(__clang__) -#pragma clang diagnostic pop -#endif - -struct ConvBinWinogradRxSf2x3g1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -template -struct ConvMPBidirectWinograd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvMPBidirectWinograd>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - static const fs::path names[3] = {"xform_bidirect_winograd_data.s", - "xform_bidirect_winograd_filter.s", - "xform_bidirect_winograd_out.s"}; - return names[id]; - } - - static std::string GetSolverKernelNames(int id) - { - static const std::string name_suffix = - '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + - std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); - static const std::string names[3] = { - "miopenGcnAsmMPBidirectWinogradXformData" + name_suffix, - "miopenGcnAsmMPBidirectWinogradXformFilter" + name_suffix, - "miopenGcnAsmMPBidirectWinogradXformOut" + name_suffix}; - return names[id]; - } - - static int GetSolverWinoXformHWSize() { return WinoDataH + WinoFilterH - 1; } -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvMPBidirectWinograd<2, 3>; -extern template struct ConvMPBidirectWinograd<3, 3>; -extern template struct ConvMPBidirectWinograd<4, 3>; -extern template struct ConvMPBidirectWinograd<5, 3>; -extern template struct ConvMPBidirectWinograd<6, 3>; - -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic pop -#endif - -template -struct ConvMPBidirectWinograd_xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvMPBidirectWinograd_xdlops>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override - { - return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsDynamic() && - ConvMPBidirectWinograd{} - .IsDynamic() && - IsThisSolverDynamic(); - } - - PerformanceImplicitGemmForwardV4R4Xdlops - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvHipImplicitGemmForwardV4R4Xdlops{}.GetDefaultPerformanceConfig(xdlops_ctx, - xdlops_problem); - } - - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const PerformanceImplicitGemmForwardV4R4Xdlops& config) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsValidPerformanceConfig( - xdlops_ctx, xdlops_problem, config); - } - - size_t GetWorkspaceSize(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvMPBidirectWinograd() - .GetWorkspaceSize(ctx, problem) + - ConvHipImplicitGemmForwardV4R4Xdlops{}.GetWorkspaceSize(xdlops_ctx, xdlops_problem); - } - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - -private: - ExecutionContext - GetTransformedConvContext(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& transformed_problem) const; - miopen::conv::ProblemDescription - GetTransformedProblem(const miopen::conv::ProblemDescription& problem) const; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - return ConvMPBidirectWinograd:: - GetSolverFileNames(id); - } - - static std::string GetSolverKernelNames(int id) - { - return ConvMPBidirectWinograd:: - GetSolverKernelNames(id); - } - - static int GetSolverWinoXformHWSize() - { - return ConvMPBidirectWinograd:: - GetSolverWinoXformHWSize(); - } - - bool IsThisSolverDynamic() const { return true; } -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvMPBidirectWinograd_xdlops<2, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<3, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<4, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<5, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<6, 3>; - -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic pop -#endif - -template -struct ConvWinograd3x3MultipassWrW final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvWinograd3x3MultipassWrW>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - static const fs::path names[3] = {"xform_data.s", "xform_filter.s", "xform_out.s"}; - return names[id]; - } - - static std::string GetSolverKernelNames(int id) - { - static const std::string name_suffix = - '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + - std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); - static const std::string names[3] = {"miopenGcnAsmWinogradXformData" + name_suffix, - "miopenGcnAsmWinogradXformFilter" + name_suffix, - "miopenGcnAsmWinogradXformOut" + name_suffix}; - - return names[id]; - } - - static int GetGroupCountMult() { return 4; } - - static int GetSolverWinoXformHWSize(const miopen::conv::ProblemDescription& problem, int id) - { - if(id == 0) - { - return WinoDataH + - (WinoFilterH - 1) * (WinoDataH == 7 ? 2 : problem.GetKernelStrideH()); - } - else - { - return WinoDataW + - (WinoFilterW - 1) * (WinoDataW == 7 ? 2 : problem.GetKernelStrideW()); - } - } - -private: - InvokerFactory PrepareInvokerFactory(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - std::size_t ws_sz) const; -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvWinograd3x3MultipassWrW<3, 2>; -extern template struct ConvWinograd3x3MultipassWrW<3, 3>; -extern template struct ConvWinograd3x3MultipassWrW<3, 4>; -extern template struct ConvWinograd3x3MultipassWrW<3, 5>; -extern template struct ConvWinograd3x3MultipassWrW<3, 6>; -extern template struct ConvWinograd3x3MultipassWrW<7, 2>; -extern template struct ConvWinograd3x3MultipassWrW<7, 3>; -extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 2>; -extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 3>; -extern template struct ConvWinograd3x3MultipassWrW<7, 2, 1, 1>; -extern template struct ConvWinograd3x3MultipassWrW<7, 3, 1, 1>; -extern template struct ConvWinograd3x3MultipassWrW<5, 3>; -extern template struct ConvWinograd3x3MultipassWrW<5, 4>; - -#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) -#pragma clang diagnostic pop -#endif - -struct PerformanceConfigAsmDirect3x3WrW : PerfConfigBase -{ - int limit_wave_cnt; // [0..9] - int reverse_inout; // [0..1], 1 is allowed for stride=1x1 only. - int chunk_size; // {16,8}, Smaller values increase register pressure. - int k_per_wave; // {1,2,4,8} && ((chunk_size * k_per_wave) <= 64). - // Higher values increase register pressure. - int pipe_lines_depth; // [1..16] && (pipe_lines_depth <= img_h). - // Higher values increase register pressure. - int n_per_group; // [1..8] && (n_per_group <= batch_size). - - PerformanceConfigAsmDirect3x3WrW(int lwc, int rio, int csz, int kpw, int pld, int npg); - PerformanceConfigAsmDirect3x3WrW() : PerformanceConfigAsmDirect3x3WrW(-1, -1, -1, -1, -1, -1) {} - PerformanceConfigAsmDirect3x3WrW(bool) : PerformanceConfigAsmDirect3x3WrW(0, 0, 8, 1, 1, 1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.limit_wave_cnt, "limit_wave_cnt"); - f(self.reverse_inout, "reverse_inout"); - f(self.chunk_size, "chunk_size"); - f(self.k_per_wave, "k_per_wave"); - f(self.pipe_lines_depth, "pipe_lines_depth"); - f(self.n_per_group, "n_per_group"); - } - - // clang-format off - int GetLimitWaveCnt() const { return limit_wave_cnt; } - int GetReverseInout() const { return reverse_inout; } - int GetChunkSize() const { return chunk_size; } - int GetKPerWave() const { return k_per_wave; } - int GetPipeLinesDepth() const { return pipe_lines_depth; } - int GetNPerGroup() const { return n_per_group; } - int GetCPerWave() const { assert(chunk_size); return 64 / chunk_size; } // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmDirect3x3WrW& other) const; -}; - -struct ConvAsmBwdWrW3x3 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW& config) const override; -}; - -template -struct ConvWinoFuryRxS final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId>(); - } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; -}; - -#ifndef CONV_WINO_FURY_RXS_CPP -extern template struct ConvWinoFuryRxS<2, 3>; -// extern template struct ConvWinoFuryRxS<3, 2>; -#endif - -struct PerformanceConfigConvAsmBwdWrW1x1 : PerfConfigBase -{ - - int chunk_size; // {1,2,4,8,16} - int c_per_gpr; // {1,2,4,8,16} - int c_mult; // {1,2,4,8,16} - int k_per_gpr; // {1,2,4,8,16} - int k_mult; // {1,2,4,8,16} - int n_per_gpr; // {1,2,4} - int n_part_cnt; // [1..8] - int read_size; // [1..4] - int short_store; // {0,1} - int data_prefetch; // [0..4] - bool use_spare_set; - - /// The following conditions must be met. - /// - /// Shader design-related constraints: - /// - (A) (chunk_size * c_per_gpr) == 16 - /// - (B) k_per_gpr <= c_per_gpr - /// - (C) (c_mult > 1 || k_mult > 1) - /// ? ((fwd_C % (c_per_gpr * c_mult) == 0) && (fwd_K % (k_per_gpr * k_mult) == 0)) - /// : (true) - /// - /// Resource-related constraints: - /// - (D) c_mult * k_mult * k_per_gpr + 9 + (c_mult + k_mult) * read_size * pipe_depth <= 256 - /// - /// Where: - /// - fwd_C := Num input channels for forward convolution (-c). - /// For backward, this is actually n_outputs. - /// - fwd_K := Num output channels for forward convolution (-k). - /// For backward, this is actually n_inputs. - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1(int chunk_size_, - int c_per_gpr_, - int c_mult_, - int k_per_gpr_, - int k_mult_, - int n_per_gpr_, - int n_part_cnt_, - int read_size_, - int short_store_, - int data_prefetch_, - bool); - PerformanceConfigConvAsmBwdWrW1x1() - : PerformanceConfigConvAsmBwdWrW1x1(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - PerformanceConfigConvAsmBwdWrW1x1(bool spare) - : PerformanceConfigConvAsmBwdWrW1x1(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.chunk_size, "chunk_size"); - f(self.c_per_gpr, "c_per_gpr"); - f(self.c_mult, "c_mult"); - f(self.k_per_gpr, "k_per_gpr"); - f(self.k_mult, "k_mult"); - f(self.n_per_gpr, "n_per_gpr"); - f(self.n_part_cnt, "n_part_cnt"); - f(self.read_size, "read_size"); - f(self.short_store, "short_store"); - f(self.data_prefetch, "data_prefetch"); - } - - // clang-format off - int GetChunkSize() const { return chunk_size; } - int GetCPerGpr() const { return c_per_gpr; } - int GetCMult() const { return c_mult; } - int GetKPerGpr() const { return k_per_gpr; } - int GetKMult() const { return k_mult; } - int GetNPerGpr() const { return n_per_gpr; } - int GetNPartCnt() const { return n_part_cnt; } - int GetHWPerGpr() const { assert(c_per_gpr); assert(n_per_gpr); assert(chunk_size); - return wave_size / (c_per_gpr * n_per_gpr * chunk_size); } // "hw" stands for "height-and-width". - int GetReadSize() const { return read_size; } - int GetShortStore() const {return short_store; } - int GetDataPrefetch() const { return data_prefetch; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsmBwdWrW1x1& other) const; -}; - -struct ConvAsmBwdWrW1x1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 MIOPEN_INTERNALS_EXPORT - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; -}; - -/// N_BATCH_LOOPS - {1,2,4,8,16} Num batches processed in single workitem. -/// Required workspace size depends on it. However there is a restriction in the internal -/// Solver API that this shouldn't be so. Therefore the family of Solvers created. -/// Each Solver in the family has constant value of this parameter. -template -struct PerformanceConfigConvOclBwdWrw2 - : PerfConfigBase> -{ - // Num waves involved a workgroup. - int n_waves = -1; // {1,2,4,8} - // Num values to read in a workitem (read_unit). - int read_size = -1; // [6..12] - // Num of output channels (top/bottom layer in forward/backward direction) - // that share the same input channel in single workgroup. - // Also represents number of output channels in single tile. - int n_out_channels_per_tile = -1; // {1,2,4,8} - // How many tiles of output channels are processed in a single workgroup? - // n_out_channels_in_lcl * n_out_channels_tiles = total number of - // output channels processed in single workgroup. - int n_out_channels_tiles = -1; // {1,2,4,8} - // Num of output rows processed in a single iteration of loop in a workitem - // (N_ALIGNED_OUT_SCAN_BLK). - int n_out_rows_in_lcl = -1; // [2..11] - - PerformanceConfigConvOclBwdWrw2(int nw, int rs, int nocpt, int noct, int noril) - : n_waves(nw), - read_size(rs), - n_out_channels_per_tile(nocpt), - n_out_channels_tiles(noct), - n_out_rows_in_lcl(noril) - { - } - PerformanceConfigConvOclBwdWrw2() {} - PerformanceConfigConvOclBwdWrw2(bool) : PerformanceConfigConvOclBwdWrw2(1, 6, 1, 1, 2) {} - // spare_set is not used in this solver. - - template - static void Visit(Self&& self, F f) - { - f(self.n_waves, "n_waves"); - f(self.read_size, "read_size"); - f(self.n_out_channels_per_tile, "n_out_channels_per_tile"); - f(self.n_out_channels_tiles, "n_out_channels_tiles"); - f(self.n_out_rows_in_lcl, "n_out_rows_in_lcl"); - } - - // clang-format off - int GetNumWaves() const { return n_waves; } - int GetReadSize() const { return read_size; } - int GetNumOutChannelsPerTile() const { return n_out_channels_per_tile; } - int GetNumOutChannelTiles() const { return n_out_channels_tiles; } - int GetNumOutRowsPerIterPerWork() const { return n_out_rows_in_lcl; } // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvOclBwdWrw2& other) const; -}; - -template -struct ConvOclBwdWrW2 : ConvTunableSolver> -{ - const std::string& SolverDbId() const override - { - return this->template GetSolverDbId>(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvOclBwdWrw2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvOclBwdWrw2&) const override; - -protected: - bool IsApplicableBase(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct PerformanceConfigConvOclBwdWrw2<1>; -extern template struct PerformanceConfigConvOclBwdWrw2<2>; -extern template struct PerformanceConfigConvOclBwdWrw2<4>; -extern template struct PerformanceConfigConvOclBwdWrw2<8>; -extern template struct PerformanceConfigConvOclBwdWrw2<16>; - -extern template struct ConvOclBwdWrW2<1>; -extern template struct ConvOclBwdWrW2<2>; -extern template struct ConvOclBwdWrW2<4>; -extern template struct ConvOclBwdWrW2<8>; -extern template struct ConvOclBwdWrW2<16>; - -#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) -#pragma clang diagnostic pop -#endif - -/// A separate solver from ConvOclBwdWrW2 to disable auto-tuning for certain configs. -/// Basically, this is *hack* for non-group 3x3 and 1x1 cases. -/// It is assumed that Solutions provided by the ConvOclBwdWrW2 solver -/// would never beat 3x3 and 1x1 assembly WrW kernels, even after tuning. -struct ConvOclBwdWrW2NonTunable final : ConvOclBwdWrW2<1> -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - InvokerFactory GetInvokerFactory(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const - { - return *GetSolution(ctx, problem).invoker_factory; - } - -private: - // This function dervied from ConvOclBwdWrW2 is declared private - // so that this solver is not marked searchable/tunable. - using ConvOclBwdWrW2<1>::GetDefaultPerformanceConfig; - using ConvOclBwdWrW2<1>::GetSolution; - using ConvOclBwdWrW2<1>::GetInvokerFactory; -}; - -struct ConvOclBwdWrW53 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclBwdWrW1x1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct fft final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceImplicitGemmWrwV4R4Xdlops : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmK; - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmWrwV4R4Xdlops(int, int, int, int, int, int, bool, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(bool spare); - PerformanceImplicitGemmWrwV4R4Xdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceImplicitGemmWrwV4R4Xdlops(a, b, c, d, e, f, g, h, false) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmWrwV4R4Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmWrwV4R4Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - int GemmMFactor; - int GemmNFactor; - int GemmKTotalFactor; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmK; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm( - int, int, int, int, int, int, int, int, int, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(); - PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(bool) - : PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm() - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmMFactor, "GemmMFactor"); - f(self.GemmNFactor, "GemmNFactor"); - f(self.GemmKTotalFactor, "GemmKTotalFactor"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - std::tuple CalculateGridSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct PerformanceConvCkIgemmFwdV6r1DlopsNchw - : PerfConfigBase -{ - int ck_tunable_list_id; - - PerformanceConvCkIgemmFwdV6r1DlopsNchw(int a) : ck_tunable_list_id(a) {} - - PerformanceConvCkIgemmFwdV6r1DlopsNchw() : PerformanceConvCkIgemmFwdV6r1DlopsNchw(-1) {} - - PerformanceConvCkIgemmFwdV6r1DlopsNchw(bool) : PerformanceConvCkIgemmFwdV6r1DlopsNchw(0) {} - - template - static void Visit(Self&& self, F f) - { - f(self.ck_tunable_list_id, "ck_tunable_list_id"); - } - - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - bool operator==(const PerformanceConvCkIgemmFwdV6r1DlopsNchw& config) const - { - return ck_tunable_list_id == config.ck_tunable_list_id; - } -}; - -struct ConvCkIgemmFwdV6r1DlopsNchw final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - bool IsDynamic() const override { return false; } - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; -}; - -struct ConvDirectNaiveConvFwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvDirectNaiveConvBwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvDirectNaiveConvWrw final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmFwdBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - -private: - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwd1x1_0_2; - friend struct GemmFwd1x1_0_1_int8; - friend struct GemmFwd1x1_0_1; - friend struct GemmFwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_2 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_1_int8 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_1 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmFwdRest final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmBwdBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - -private: - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmBwd1x1_stride2; - friend struct GemmBwd1x1_stride1; - friend struct GemmBwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmBwd1x1_stride2 final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmBwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmBwd1x1_stride1 final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription& problem) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription& problem) const override; - - friend struct GemmBwdRest; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmBwdRest final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmWrwBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - -private: - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmWrw1x1_stride1; - friend struct GemmWrwUniversal; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmWrw1x1_stride1 final : GemmWrwBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - friend struct GemmWrwUniversal; -}; - -struct MIOPEN_INTERNALS_EXPORT GemmWrwUniversal final : GemmWrwBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - size_t GetWorkspaceSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - bool IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - - ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase -{ - std::string direction; - std::string tensor_layout; - std::string precision; - int nxb; - int nxe; - - int gemm_m_per_block; - int gemm_n_per_block; - int gemm_k_per_block; - - int wave_tile_m; - int wave_tile_n; - int wave_tile_k; - int wave_step_m; - int wave_step_n; - int wave_repeat_m; - int wave_repeat_n; - - int multihead; - int vector_store; - int gemm_k_global_split; - int merge_e; - int tensor_a_pass_through; - - std::vector tensor_a_thread_lengths; - std::vector tensor_a_cluster_lengths; - std::vector tensor_b_thread_lengths; - std::vector tensor_b_cluster_lengths; - - bool use_spare_set; - int index; - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - PerformanceConfigAsmImplicitGemmGTC() - : PerformanceConfigAsmImplicitGemmGTC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTC(bool spare) - : PerformanceConfigAsmImplicitGemmGTC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.direction, "dir"); - f(self.tensor_layout, "lyt"); - f(self.precision, "pre"); - f(self.nxb, "nxb"); - f(self.nxe, "nxe"); - f(self.gemm_m_per_block, "mpb"); - f(self.gemm_n_per_block, "npb"); - f(self.gemm_k_per_block, "kpb"); - - f(self.wave_tile_m, "wtm"); - f(self.wave_tile_n, "wtn"); - f(self.wave_tile_k, "wtk"); - f(self.wave_step_m, "wsm"); - f(self.wave_step_n, "wsn"); - f(self.wave_repeat_m, "wrm"); - f(self.wave_repeat_n, "wrn"); - - f(self.multihead, "mh"); - f(self.vector_store, "vs"); - f(self.gemm_k_global_split, "gks"); - f(self.merge_e, "me"); - f(self.tensor_a_pass_through, "pta"); - - f(self.tensor_a_thread_lengths[0], "ta0"); - f(self.tensor_a_thread_lengths[1], "ta1"); - f(self.tensor_a_thread_lengths[2], "ta2"); - f(self.tensor_a_thread_lengths[3], "ta3"); - - f(self.tensor_a_cluster_lengths[0], "ca0"); - f(self.tensor_a_cluster_lengths[1], "ca1"); - f(self.tensor_a_cluster_lengths[2], "ca2"); - f(self.tensor_a_cluster_lengths[3], "ca3"); - - f(self.tensor_b_thread_lengths[0], "tb0"); - f(self.tensor_b_thread_lengths[1], "tb1"); - f(self.tensor_b_thread_lengths[2], "tb2"); - f(self.tensor_b_thread_lengths[3], "tb3"); - - f(self.tensor_b_cluster_lengths[0], "cb0"); - f(self.tensor_b_cluster_lengths[1], "cb1"); - f(self.tensor_b_cluster_lengths[2], "cb2"); - f(self.tensor_b_cluster_lengths[3], "cb3"); - f(self.index, "index"); - } - - // Chilrden must provide support for ComputedContainer. - void HeuristicInit(const ExecutionContext&) = delete; - bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; - bool IsValidValue() const = delete; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; - - MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmImplicitGemmGTC& other) const; - MIOPEN_INTERNALS_EXPORT void CopyParameters(const PerformanceConfigAsmImplicitGemmGTC& other); - MIOPEN_INTERNALS_EXPORT std::string ToString() const override; - MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT int BlockSize() const; -}; - -struct PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription& config); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT size_t ComputeKernelOccupancy() const; - -private: - void SetParamsForKSplit(const miopen::conv::ProblemDescription& problem, - const size_t& occupancy); -}; - -struct ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCvector - : PerfConfigBase -{ - std::string direction; - std::string tensor_layout; - std::string precision; - int nxb; - int nxe; - - int gemm_m_per_block; - int gemm_n_per_block; - int gemm_k_per_block; - - int lanegroup_tile_m; - int lanegroup_tile_n; - int lanegroup_wave_m; - int lanegroup_wave_n; - int lanegroup_repeat_m; - int lanegroup_repeat_n; - - int vector_c; - - std::vector tensor_a_thread_lengths; - std::vector tensor_a_cluster_lengths; - std::vector tensor_b_thread_lengths; - std::vector tensor_b_cluster_lengths; - - bool use_spare_set; - int index; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - - PerformanceConfigAsmImplicitGemmGTCvector() - : PerformanceConfigAsmImplicitGemmGTCvector("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCvector(bool spare) - : PerformanceConfigAsmImplicitGemmGTCvector("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.direction, "dir"); - f(self.tensor_layout, "lyt"); - f(self.precision, "pre"); - f(self.nxb, "nxb"); - f(self.nxe, "nxe"); - f(self.gemm_m_per_block, "mpb"); - f(self.gemm_n_per_block, "npb"); - f(self.gemm_k_per_block, "kpb"); - - f(self.lanegroup_tile_m, "lgtm"); - f(self.lanegroup_tile_n, "lgtn"); - f(self.lanegroup_wave_m, "lgpwm"); - f(self.lanegroup_wave_n, "lgpwn"); - f(self.lanegroup_repeat_m, "lgrm"); - f(self.lanegroup_repeat_n, "lgrn"); - - f(self.vector_c, "vec_c"); - - f(self.tensor_a_thread_lengths[0], "ta0"); - f(self.tensor_a_thread_lengths[1], "ta1"); - f(self.tensor_a_thread_lengths[2], "ta2"); - f(self.tensor_a_thread_lengths[3], "ta3"); - - f(self.tensor_a_cluster_lengths[0], "ca0"); - f(self.tensor_a_cluster_lengths[1], "ca1"); - f(self.tensor_a_cluster_lengths[2], "ca2"); - f(self.tensor_a_cluster_lengths[3], "ca3"); - - f(self.tensor_b_thread_lengths[0], "tb0"); - f(self.tensor_b_thread_lengths[1], "tb1"); - f(self.tensor_b_thread_lengths[2], "tb2"); - f(self.tensor_b_thread_lengths[3], "tb3"); - - f(self.tensor_b_cluster_lengths[0], "cb0"); - f(self.tensor_b_cluster_lengths[1], "cb1"); - f(self.tensor_b_cluster_lengths[2], "cb2"); - f(self.tensor_b_cluster_lengths[3], "cb3"); - f(self.index, "index"); - } - - // Chilrden must provide support for ComputedContainer. - void HeuristicInit(const ExecutionContext&) = delete; - bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; - bool IsValidValue() const = delete; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; - - MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigAsmImplicitGemmGTCvector& other) const; - MIOPEN_INTERNALS_EXPORT void - CopyParameters(const PerformanceConfigAsmImplicitGemmGTCvector& other); - MIOPEN_INTERNALS_EXPORT std::string ToString() const override; - MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT int BlockSize() const; -}; -struct PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC : PerformanceConfigAsmImplicitGemmGTCvector -{ - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTCvector(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - lgtm, - lgtn, - lgpwm, - lgpwn, - lgrm, - lgrn, - vec_c, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTCvector(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - lgtm, - lgtn, - lgpwm, - lgpwn, - lgrm, - lgrn, - vec_c, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC() - : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; -}; - -struct PerformanceConfigHipImplicitGemmFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmFwdXdlops(bool) - : PerformanceConfigHipImplicitGemmFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmFwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; - /// \anchor igemm_get_wti_magic_number - // Magic Number Alert: - // Naive convolutions have GetWti() that return very small value (0.01f). - // This allows MIOpen to use Naive Solvers if no other applicable Solvers - // have known WTIs. Right now this means that in case of find-db miss, - // the library will try to use Winograd or GEMM (whatever is faster according - // to their GetWti's), but if both are not applicable, the library will - // use Naive Solver - // Since we would like to us CK before naive, and use it instead (because - // we do expect that CK is faster than Naive), therefore we use a - // value bigger than 0.01f, e.g. 0.02f. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmBwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmBwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmBwdXdlops(bool) - : PerformanceConfigHipImplicitGemmBwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmBwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmGroupFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmGroupFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmGroupFwdXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupFwdXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(const std::string& type); - bool ModelApplyToken(int idx, std::string value, const std::string& arch); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemm3DGroupFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops() - : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupWrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops() - : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupBwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmGroupBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmGroupBwdXdlops() - : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmGroupBwdXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupBwdXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(); - bool ModelApplyToken(int idx, std::string value); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupWrwXdlops - : PerfConfigBaseCK -{ - int index; - int split_k; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmGroupWrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmGroupWrwXdlops() - : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmGroupWrwXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupWrwXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(const std::string& type); - bool ModelApplyToken(int idx, - std::string value, - const std::string& arch, - const miopen::conv::ProblemDescription& problem); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupWrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16FwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops() - : PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16BwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops() - : PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16WrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -} // namespace conv -} // namespace solver -} // namespace miopen diff --git a/src/include/miopen/cumulative_reduction.hpp b/src/include/miopen/cumulative_reduction.hpp new file mode 100644 index 0000000000..b6cadcb8ec --- /dev/null +++ b/src/include/miopen/cumulative_reduction.hpp @@ -0,0 +1,48 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT miopenStatus_t +CumulativeReductionForward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputDesc, + Data_t output, + const TensorDescriptor& indicesDesc, + Data_t indices, + int dim, + bool exclusive, + bool reverse, + miopenCumOp_t cumOp); + +} // namespace miopen diff --git a/src/include/miopen/cumulative_reduction/invoke_params.hpp b/src/include/miopen/cumulative_reduction/invoke_params.hpp new file mode 100644 index 0000000000..fac69df6f0 --- /dev/null +++ b/src/include/miopen/cumulative_reduction/invoke_params.hpp @@ -0,0 +1,57 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace cumulative_reduction { + +struct InvokeParams : public miopen::InvokeParams +{ + InvokeParams() = default; + + const TensorDescriptor* inputDesc = nullptr; + const TensorDescriptor* outputDesc = nullptr; + const TensorDescriptor* indicesDesc = nullptr; + + ConstData_t input = nullptr; + Data_t output = nullptr; + Data_t indices = nullptr; + + int dim = 0; + bool exclusive = false; + bool reverse = false; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +} // namespace cumulative_reduction + +} // namespace miopen diff --git a/src/include/miopen/cumulative_reduction/problem_description.hpp b/src/include/miopen/cumulative_reduction/problem_description.hpp new file mode 100644 index 0000000000..73fbd937f5 --- /dev/null +++ b/src/include/miopen/cumulative_reduction/problem_description.hpp @@ -0,0 +1,131 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + *all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +#include + +namespace miopen { + +struct NetworkConfig; + +namespace cumulative_reduction { + +bool checkSameLength(const TensorDescriptor& x, const TensorDescriptor& y); + +struct ForwardProblemDescription : ProblemDescriptionBase +{ + ForwardProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputDesc_, + const TensorDescriptor& indicesDesc_, + const int& dim_, + const miopenCumOp_t& cumOp_) + : inputDesc(inputDesc_), + outputDesc(outputDesc_), + indicesDesc(indicesDesc_), + dim(dim_), + cumOp(cumOp_) + { + if(IsValidDim()) + dim = (dim < 0 ? dim + inputDesc.GetNumDims() : dim); + IsValidIndicesType(); + IsSameLength(); + } + + const TensorDescriptor& GetInputDesc() const { return inputDesc; } + const TensorDescriptor& GetOutputDesc() const { return outputDesc; } + const TensorDescriptor& GetIndicesDesc() const { return indicesDesc; } + const int& GetDim() const { return dim; } + const miopenCumOp_t& GetCumOp() const { return cumOp; } + + bool IsValidDim() const + { + const int ndims = inputDesc.GetNumDims(); + if(dim < -ndims || ndims - 1 < dim) + { + MIOPEN_THROW(miopenStatusBadParm, + (std::stringstream() + << "Cumulative Reduction: Operating dim value must be in range [" + << -ndims << "," << ndims - 1 << "].") + .str()); + } + return true; + } + + bool IsValidIndicesType() const + { + if(indicesDesc.GetElementSize() > 0 && indicesDesc.GetType() != miopenInt32) + MIOPEN_THROW(miopenStatusBadParm, + "Cumulative Reduction: Indices tensor type must be int32."); + return true; + } + + bool IsSameLength() const + { + if(outputDesc.GetElementSize() > 0 && !checkSameLength(inputDesc, outputDesc)) + MIOPEN_THROW(miopenStatusBadParm, + "Cumulative Reduction: Input and Output tensor sizes do not match."); + if(indicesDesc.GetElementSize() > 0 && !checkSameLength(inputDesc, indicesDesc)) + MIOPEN_THROW(miopenStatusBadParm, + "Cumulative Reduction: Input and Indices tensor sizes do not match."); + return true; + } + + bool IsAllPacked() const + { + if(!inputDesc.IsPacked() || !outputDesc.IsPacked() || !indicesDesc.IsPacked()) + return false; + return true; + } + + bool IsAllDimStride1() const + { + if(inputDesc.GetStrides()[dim] != 1) + return false; + if(outputDesc.GetElementSize() > 0 && outputDesc.GetStrides()[dim] != 1) + return false; + if(indicesDesc.GetElementSize() > 0 && indicesDesc.GetStrides()[dim] != 1) + return false; + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor inputDesc; + TensorDescriptor outputDesc; + TensorDescriptor indicesDesc; + int dim; + miopenCumOp_t cumOp; + + NetworkConfig MakeForwardNetworkConfig() const; +}; + +} // namespace cumulative_reduction + +} // namespace miopen diff --git a/src/include/miopen/cumulative_reduction/solvers.hpp b/src/include/miopen/cumulative_reduction/solvers.hpp new file mode 100644 index 0000000000..1e78b8851d --- /dev/null +++ b/src/include/miopen/cumulative_reduction/solvers.hpp @@ -0,0 +1,59 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace cumulative_reduction { + +using ForwardSolverBase = + NonTunableSolverBase; + +struct ForwardContiguousLastDim final : ForwardSolverBase +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + bool IsApplicable( + const ExecutionContext& context, + const miopen::cumulative_reduction::ForwardProblemDescription& problem) const override; + ConvSolution GetSolution( + const ExecutionContext& context, + const miopen::cumulative_reduction::ForwardProblemDescription& problem) const override; +}; + +} // namespace cumulative_reduction + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/cumulative_reduction/utils.hpp b/src/include/miopen/cumulative_reduction/utils.hpp new file mode 100644 index 0000000000..9137019ff4 --- /dev/null +++ b/src/include/miopen/cumulative_reduction/utils.hpp @@ -0,0 +1,45 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include +#include + +namespace miopen { +namespace solver { +namespace cumulative_reduction { + +KernelInfo make_hip_kernel(std::vector localsize, + std::vector gridsize, + std::string kernel_file, + std::string kernel_name, + KernelBuildParameters build_params); + +} // namespace cumulative_reduction +} // namespace solver +} // namespace miopen diff --git a/src/include/miopen/fusion.hpp b/src/include/miopen/fusion.hpp index babf192b74..e69de29bb2 100644 --- a/src/include/miopen/fusion.hpp +++ b/src/include/miopen/fusion.hpp @@ -1,275 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2018 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef MIOPEN_FUSION_HPP_ -#define MIOPEN_FUSION_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace miopen { - -struct Handle; - -// Perhaps redundant -enum FusionKernelSourceType -{ - OpenclText, - AsmText, - Binary, /// \todo Unused, consider removing. -}; - -struct MIOPEN_INTERNALS_EXPORT FusionOpDescriptor : miopenFusionOpDescriptor -{ - virtual ~FusionOpDescriptor() = default; - FusionOpDescriptor(const FusionOpDescriptor&) = delete; - FusionOpDescriptor() = default; - FusionOpDescriptor& operator=(const FusionOpDescriptor&) = delete; - void SetIdx(int _id) { plan_idx = _id; }; - int GetIdx() const { return plan_idx; }; - virtual miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const = 0; - virtual miopenStatus_t GetNetworkConfig(std::ostringstream& network_config); - friend std::ostream& operator<<(std::ostream& stream, const FusionOpDescriptor& x); - virtual miopenFusionOp_t kind() const = 0; - void SetInputDesc(TensorDescriptor i_desc) { input_desc = i_desc; }; - TensorDescriptor input_desc; - - int plan_idx = 0; -}; - -struct MIOPEN_INTERNALS_EXPORT BiasFusionOpDescriptor : FusionOpDescriptor -{ - BiasFusionOpDescriptor(const TensorDescriptor& desc) : base_desc(desc) {} - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t - SetArgs(OperatorArgs& args, const void* alpha, const void* beta, ConstData_t bdata); - miopenFusionOp_t kind() const override { return miopenFusionOpBiasForward; }; - TensorDescriptor base_desc; -}; - -struct MIOPEN_INTERNALS_EXPORT TensorScaleAddOpDescriptor : public FusionOpDescriptor -{ - TensorScaleAddOpDescriptor(const TensorDescriptor& desc) : tensor_desc(desc) {} - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, float alpha, ConstData_t tensor_ptr); - miopenFusionOp_t kind() const override { return miopenFusionOpTensorScaleAdd; }; - TensorDescriptor tensor_desc; -}; - -struct MIOPEN_INTERNALS_EXPORT ActivFwdFusionOpDescriptor : FusionOpDescriptor -{ - ActivFwdFusionOpDescriptor(miopenActivationMode_t mode) : activMode(mode) {} - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, - const void* alpha, - const void* beta, - double activAlpha, - double activBeta, - double activGamma); - miopenFusionOp_t kind() const override { return miopenFusionOpActivForward; }; - miopenActivationMode_t activMode; -}; - -struct MIOPEN_INTERNALS_EXPORT ActivBwdFusionOpDescriptor : FusionOpDescriptor -{ - ActivBwdFusionOpDescriptor(miopenActivationMode_t mode) : activMode(mode) {} - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, - const void* alpha, - const void* beta, - ConstData_t y, - ConstData_t x, - double activAlpha, - double activBeta, - double activGamma); - miopenFusionOp_t kind() const override { return miopenFusionOpActivBackward; }; - miopenActivationMode_t activMode; -}; - -struct MIOPEN_INTERNALS_EXPORT BatchNormInferenceFusionOpDescriptor : FusionOpDescriptor -{ - BatchNormInferenceFusionOpDescriptor(miopenBatchNormMode_t bn_mode, - const TensorDescriptor& desc) - : mode(bn_mode), base_desc(desc) - { - } - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, - const void* alpha, - const void* beta, - ConstData_t bnScale, - ConstData_t bnBias, - ConstData_t estimatedMean, - ConstData_t estimatedVariance, - double epsilon) const; - miopenFusionOp_t kind() const override { return miopenFusionOpBatchNormInference; }; - std::vector GetLocalWGSz(Handle& handle, std::string algorithm_name); - std::vector GetGlobalWGSz(Handle& handle, std::string algorithm_name); - - miopenBatchNormMode_t mode; - TensorDescriptor base_desc; -}; - -struct MIOPEN_INTERNALS_EXPORT BatchNormFwdTrainFusionOpDescriptor : FusionOpDescriptor -{ - BatchNormFwdTrainFusionOpDescriptor(miopenBatchNormMode_t bn_mode, bool runningMeanVariance) - : mode(bn_mode), runningMeanVar(runningMeanVariance) - { - } - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, - const void* alpha, - const void* beta, - Data_t runningMean, - Data_t runningVariance, - Data_t savedMean, - Data_t savedInvVariance, - ConstData_t bnScale, - ConstData_t bnBias, - double expAvgFactor, - double epsilon) const; - miopenFusionOp_t kind() const override { return miopenFusionOpBatchNormFwdTrain; }; - std::vector GetLocalWGSz(); - std::vector GetGlobalWGSz(); - void calcBNParams(std::vector in_lens, - int& variant, - size_t& in_cstride, - size_t& in_nstride, - size_t& in_nchw, - unsigned int& ldsgcn, - unsigned int& ldsnogcn); - miopenBatchNormMode_t mode; - TensorDescriptor base_desc; - bool runningMeanVar; -}; - -struct MIOPEN_INTERNALS_EXPORT BatchNormBwdTrainFusionOpDescriptor : FusionOpDescriptor -{ - BatchNormBwdTrainFusionOpDescriptor(miopenBatchNormMode_t bn_mode) - : mode(bn_mode), useBatchStats(true) - { - } - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - miopenStatus_t SetArgs(OperatorArgs& args, - const void* alpha, - const void* beta, - ConstData_t x, - ConstData_t bnScale, - ConstData_t bnBias, - Data_t resBnScaleDiff, - Data_t resBnBiasDiff, - ConstData_t savedMean, - ConstData_t savedInvVariance) const; - miopenFusionOp_t kind() const override { return miopenFusionOpBatchNormBwdTrain; }; - std::vector GetLocalWGSz(); - std::vector GetGlobalWGSz(); - void calcBNParams(std::vector in_lens, - int& variant, - size_t& in_cstride, - size_t& in_nstride, - size_t& in_nchw, - unsigned int& ldsgcn, - unsigned int& ldsnogcn); - - miopenBatchNormMode_t mode; - bool useBatchStats; -}; - -struct MIOPEN_INTERNALS_EXPORT ConvForwardOpDescriptor : FusionOpDescriptor -{ - ConvForwardOpDescriptor(const ConvolutionDescriptor& conv_descriptor, - const TensorDescriptor& filter_descriptor) - : base_desc(conv_descriptor), - filter_desc(filter_descriptor), - kernel_info_valid(false), - conv_compiler_options(""){}; - miopenStatus_t GetOutputDesc(TensorDescriptor& output_desc) const override; - miopenStatus_t SetArgs(OperatorArgs& args, const void* alpha, const void* beta, ConstData_t w); - // miopenStatus_t SetArgs(OperatorArgs& args, float alpha, float beta, ConstData_t w); - miopenStatus_t GetNetworkConfig(std::ostringstream& network_config) override; - bool isASMApplicable(Handle& handle); - miopenFusionOp_t kind() const override { return miopenFusionOpConvForward; }; - - ConvolutionDescriptor base_desc; - TensorDescriptor filter_desc; - solver::KernelInfo kernel_info; - bool kernel_info_valid; - std::string conv_compiler_options; - -private: - conv::ProblemDescription GetConvProblem(); -}; - -MIOPEN_INTERNALS_EXPORT -miopenStatus_t ConvBiasActivFusion(Handle& handle, - const void* alpha1, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& wDesc, - ConstData_t w, - const ConvolutionDescriptor& conv_desc, - miopenConvFwdAlgorithm_t algo, - void* workspace, - size_t workspaceSizeInBytes, - const void* alpha2, - const TensorDescriptor& zDesc, - ConstData_t z, - const TensorDescriptor& biasDesc, - ConstData_t bias, - const ActivationDescriptor& activationDesc, - const TensorDescriptor& yDesc, - Data_t y); - -MIOPEN_INTERNALS_EXPORT -solver::ConvSolution MakeFusedSolution(const struct FusionContext& ctx, - solver::Id id, - const std::optional& perf_cfg_override, - const struct FusionDescription& problem, - const AnyInvokeParams& invoke_params); - -} // namespace miopen -MIOPEN_DEFINE_OBJECT(miopenFusionOpDescriptor, miopen::FusionOpDescriptor); -MIOPEN_DEFINE_OBJECT(miopenOperatorArgs, miopen::OperatorArgs); - -#endif // _MIOPEN_FUSION_HPP_ diff --git a/src/include/miopen/fusion/solvers.hpp b/src/include/miopen/fusion/solvers.hpp index 2a30bb8915..e69de29bb2 100644 --- a/src/include/miopen/fusion/solvers.hpp +++ b/src/include/miopen/fusion/solvers.hpp @@ -1,352 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -namespace miopen { -namespace solver { -namespace fusion { - -using FusionSolverBase = NonTunableSolverBase; - -template -using FusionTunableSolver = - TunableSolverMixin; -; - -struct PerformanceConfigConvBiasActivAsm1x1U : conv::PerformanceConfigConvAsm1x1U -{ - PerformanceConfigConvBiasActivAsm1x1U(const bool spare) : PerformanceConfigConvAsm1x1U(spare) {} - PerformanceConfigConvBiasActivAsm1x1U() - : PerformanceConfigConvAsm1x1U(-1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionContext& ctx, - const FusionDescription& problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& problem); - bool IsValid(const FusionContext&, const FusionDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionDescription& problem) const; -}; - -struct ConvBiasActivAsm1x1U : FusionTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, - const FusionDescription& problem, - const PerformanceConfigConvBiasActivAsm1x1U& /*config*/) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBiasActivAsm1x1U - GetDefaultPerformanceConfig(const FusionContext&, const FusionDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBiasActivAsm1x1U - Search(const FusionContext& context, - const FusionDescription& problem, - const AnyInvokeParams& invoke_params) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const FusionContext&, - const FusionDescription&, - const PerformanceConfigConvBiasActivAsm1x1U&) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; -}; - -using PerformanceConfigConvOclDirectFwdFused = LegacyPerformanceConfig; -struct ConvOclDirectFwdFused final : FusionTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, - const FusionDescription& problem, - const PerformanceConfigConvOclDirectFwdFused&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclDirectFwdFused - GetDefaultPerformanceConfig(const FusionContext&, const FusionDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclDirectFwdFused - Search(const FusionContext&, - const FusionDescription&, - const AnyInvokeParams& invoke_params) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const FusionContext&, - const FusionDescription&, - const PerformanceConfigConvOclDirectFwdFused&) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription& problem) const override; -}; - -struct PerformanceConfigConvCKIgemmFwdBiasActivFused - : PerfConfigBase -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigConvCKIgemmFwdBiasActivFused(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigConvCKIgemmFwdBiasActivFused() - : PerformanceConfigConvCKIgemmFwdBiasActivFused(0, "") - { - } - PerformanceConfigConvCKIgemmFwdBiasActivFused(bool) - : PerformanceConfigConvCKIgemmFwdBiasActivFused(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; - - template - static void Visit(Self&& s, F f) - { - f(s.kernel_id, "kernel_id"); - } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvCKIgemmFwdBiasActivFused& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvCKIgemmFwdBiasActivFused final - : FusionTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmFwdBiasActivFused - GetDefaultPerformanceConfig(const FusionContext& ctx, - const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const PerformanceConfigConvCKIgemmFwdBiasActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvCKIgemmFwdBiasActivFused - Search(const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const PerformanceConfigConvCKIgemmFwdBiasActivFused& config) const override; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerfConfigConvCKIgemmFwdBiasResAddActivFused - : PerfConfigBase -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerfConfigConvCKIgemmFwdBiasResAddActivFused(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerfConfigConvCKIgemmFwdBiasResAddActivFused() - : PerfConfigConvCKIgemmFwdBiasResAddActivFused(0, "") - { - } - PerfConfigConvCKIgemmFwdBiasResAddActivFused(bool) - : PerfConfigConvCKIgemmFwdBiasResAddActivFused(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const FusionDescription& fdesc_problem); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const FusionContext&, - const FusionDescription& fdesc_problem) const; - - template - static void Visit(Self&& s, F f) - { - f(s.kernel_id, "kernel_id"); - } - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerfConfigConvCKIgemmFwdBiasResAddActivFused& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvCKIgemmFwdBiasResAddActivFused final - : FusionTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerfConfigConvCKIgemmFwdBiasResAddActivFused - GetDefaultPerformanceConfig(const FusionContext& ctx, - const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const PerfConfigConvCKIgemmFwdBiasResAddActivFused& config) const override; - MIOPEN_INTERNALS_EXPORT PerfConfigConvCKIgemmFwdBiasResAddActivFused - Search(const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& ctx, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& ctx, - const FusionDescription& fdesc_problem, - const PerfConfigConvCKIgemmFwdBiasResAddActivFused& config) const override; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvBinWinogradRxSFused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const FusionContext& context, - const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution( - const FusionContext& context, const FusionDescription& fdesc_problem) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; -}; - -struct ConvBinWinogradRxSf2x3g1Fused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT float GetWti(const FusionContext&, - const FusionDescription&) const override; -}; - -template -struct ConvWinoFuryRxSFused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId>(); - } - - bool IsApplicable(const FusionContext&, const FusionDescription&) const override; - bool IsDynamic() const override { return true; } - float GetWti(const FusionContext&, const FusionDescription&) const override; - size_t GetWorkspaceSize(const FusionContext&, const FusionDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - - ConvSolution GetSolution(const FusionContext&, const FusionDescription&) const override; -}; - -#ifndef CONV_WINO_FURY_RXS_CPP -extern template struct ConvWinoFuryRxSFused<2, 3>; -// extern template struct ConvWinoFuryRxSFused<3, 2>; -#endif - -struct BnFwdInferActivationFused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; -}; - -struct BnFwdTrgActivationFused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; -}; - -struct BnBwdTrgActivationFused final : FusionSolverBase -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool IsApplicable(const FusionContext& context, - const FusionDescription& problem) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const FusionContext& context, const FusionDescription& problem) const override; -}; - -} // namespace fusion -} // namespace solver -} // namespace miopen diff --git a/src/include/miopen/generic_search.hpp b/src/include/miopen/generic_search.hpp index e960c04018..e69de29bb2 100644 --- a/src/include/miopen/generic_search.hpp +++ b/src/include/miopen/generic_search.hpp @@ -1,619 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef GUARD_MIOPEN_GENERIC_SEARCH_HPP_ -#define GUARD_MIOPEN_GENERIC_SEARCH_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace miopen { -namespace solver { - -namespace debug { -// This struct is not MT-safe, meaning one should use it before starting threads, thus avoiding -// constructing it inside a worker thread. -struct MIOPEN_INTERNALS_EXPORT TuningIterationScopedLimiter -{ - TuningIterationScopedLimiter(std::size_t new_limit); - ~TuningIterationScopedLimiter(); - -private: - std::optional old_limit; -}; -} // namespace debug - -/// This STL-like container together with corresponding iterator provide access -/// to a set of all available performance configs for the given problem config. -/// -/// Implementation does not hold values themselves as these would take too much memory. -/// The container holds problem config information instead. This info -/// is required for advancing the iterator to the next valid configuration. -/// -/// PerformanceConfig type requirements: -/// - (ctor)() -/// Constructs an instance with invalid value. -/// - (ctor)(bool) -/// Constructs an instance with minimal value. -/// - SetNextValue(const Problem& p) -/// Advances instance value to the next available value and returns true. -/// If max value reached, returns false. -/// - IsValid(const Context& c, const Problem& p) const -/// Checks if instance is valid for the given c. -/// For convolutions, Context represents a problem configuration. -/// - operator==(const PerformanceConfig&) -/// Ordinary semantics. -template -class ComputedContainer; - -template -class ComputedIterator -{ - PerformanceConfig v; - const Context* c; // For Next(). - const Problem* p; // For Next(). - - ComputedIterator& Next() - { - if(p != nullptr) - { - do - { - if(!v.SetNextValue(*p)) - { // Wraparound, end reached. Iterator is useless from now. - p = nullptr; - break; - } - } while(!v.IsValid(*c, *p)); - } - return *this; - } - - // Implements container's begin() - ComputedIterator(const Context& context, const Problem& problem, const bool spare) - : v(spare), c(&context), p(&problem) - { - if(!v.IsValid(*c, *p)) - Next(); - } - -public: - using iterator_category = std::input_iterator_tag; - using value_type = PerformanceConfig; - using difference_type = int; - using pointer = PerformanceConfig*; - using reference = PerformanceConfig&; - // STL-like iterator shall be default contructible. Also implements container's end() - ComputedIterator() : v(), c(nullptr), p(nullptr) {} - // STL-like iterator shall be copy contructible. The default copy ctor is ok. - - ComputedIterator& operator++() { return Next(); } - const PerformanceConfig& operator*() const { return v; } - bool operator!=(ComputedIterator const& other) const - { - if(p == other.p) - { - if(p == nullptr // Ends are always equal. - || v == other.v) - return false; - } - return true; - } - bool operator==(ComputedIterator const& other) const { return !(*this != other); } - - friend class ComputedContainer; -}; - -template -class ComputedContainer -{ - Context context; // Hold a copy make the object independent of the environment. - Problem problem; // - bool spare; // Use spare set of perf configs. Those are usually slower than main set. - // Splitting the theoretically available set of perf configs to "main" - // and "spare" sets allows for acceleration of the auto-tune process: - // * If the "main" set is not empty, then skipping the "spare" set - // avoids wasting time, because the latter is slower by definition. - // * Combining "spare" and "main" would lead to exponential growth of - // the resulting container, and thus to exponential slowdown. - // - // Nevertheless, a Solver is free to either use or not use this capability - // (i.e. it is ok for PerformanceConfig(bool) to ignore its parameter). - - /// \note We do not add 'const' to keep the object assignable - /// for the sake of flexibility. Nevertheless, all element accesses of - /// the "computed container" shall be const. - -public: - using const_iterator = ComputedIterator; - - ComputedContainer(const Context& context_, const Problem& problem_, const bool spare_ = false) - : context(context_), problem(problem_), spare(spare_) - { - } - const_iterator begin() const { return {context, problem, spare}; } - const_iterator end() const { return {}; } -}; - -template -class HeartBeat -{ - size_t n_within_beat; - size_t n_best; - float best_time; // within beat - float elapsed_cumulative; - Timer timer; - PerformanceConfig best_config; - - void Continue() - { - best_time = std::numeric_limits::max(); - n_within_beat = 0; - timer.start(); - } - -public: - HeartBeat() : n_within_beat(), n_best(), best_time(), elapsed_cumulative() {} - - void Start() - { - elapsed_cumulative = 0.0f; - best_config = PerformanceConfig(); - Continue(); - } - - void Monitor(const bool is_recent_failed, - const float recent_time, - const size_t n_recent, - const float total_best, - size_t n_failed, - size_t n_total, - const PerformanceConfig& recent_config) - { - ++n_within_beat; - if(!is_recent_failed && (recent_time < best_time)) - { - best_time = recent_time; - n_best = n_recent; - best_config = recent_config; - } - const float elapsed = timer.elapsed_ms(); - if(elapsed > 3000) - { - elapsed_cumulative += elapsed; - const float eta_sec = - n_recent != 0u ? (static_cast(n_total - n_recent) * - (elapsed_cumulative / static_cast(n_recent)) / 1000.0f) - : 0.0f; // paraniod - MIOPEN_LOG_W(n_recent << '/' << n_failed << '/' << n_total << ' ' << total_best - << ", best within recent " << n_within_beat << ": " << best_time - << " #" << n_best << ' ' << best_config << ", ETA:" << eta_sec - << " sec."); - Continue(); - } - } -}; - -/// Solver member function requirements: -/// * GetDefaultPerformanceConfig shall be implemented. -/// - Its return type shall be suitable for instantiation of the ComputedContainer. -/// * GetSolution shall be implemented. -/// * Solution should provide invoker -/// * RunAndMeasureSolution must NOT be implemented. Invoker will be used instead. -/// -/// clang-format-off -/// ----------------------------------------------- -/// Dataflow: -/// Forward: -/// wei[] (w) --> +--------+ -/// | kernel | --> top[] (y) -/// bot[] (x) --> +--------+ -/// -/// Backward data: -/// wei[] (w) --> +--------+ -/// | kernel | --> top[] (dx) -/// bot[] (dy) --> +--------+ -/// -/// Backward WrW: -/// top[] (dx) --> +--------+ -/// | kernel | --> wei[] (dw) -/// bot[] (dy) --> +--------+ -/// ------------------------------------------------ -/// clang-format-on - -template -using RunAndMeasure_t = - decltype(std::declval().RunAndMeasureSolution(std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval())); - -template -auto GetAllConfigs(const Solver s, const Context& context, const Problem& problem) - -> ComputedContainer -{ - using PerformanceConfig = decltype(s.GetDefaultPerformanceConfig(context, problem)); - - const ComputedContainer primary(context, problem); - const int primary_size = std::distance(primary.begin(), primary.end()); - const ComputedContainer spare(context, problem, true); - const int spare_size = std::distance(spare.begin(), spare.end()); - const bool useSpare = (primary_size == 0); - - ComputedContainer all_configs = useSpare ? spare : primary; - const int n_runs_total = useSpare ? spare_size : primary_size; - MIOPEN_LOG_W(s.SolverDbId() << ": Searching the best solution among " << n_runs_total - << (useSpare ? " (spare)" : "") << "..."); - - return all_configs; -} - -template -std::vector -GetAllSolutions(const Solver s, const Context& context_, const Problem& problem) -{ - auto context = context_; - context.is_for_generic_search = true; - - auto all_configs = GetAllConfigs(s, context, problem); - - std::vector solutions; - for(const auto& current_config : all_configs) - { - ConvSolution current_solution = s.GetSolution(context, problem, current_config); - solutions.push_back(current_solution); - } - return solutions; -} - -std::size_t GetTuningIterationsMax(); -std::chrono::milliseconds GetTuningTimeMax(); // returns the max allowed time in milliseconds -std::size_t GetTuningThreadsMax(); - -template -void CompileAgent(size_t thread_index, - size_t total_threads, - const Solver& s, - const Context& context, - const Problem& problem, - std::vector& data, - ThreadSafeQueue>& comp_queue) -{ - const auto start_time = - std::chrono::time_point_cast(std::chrono::steady_clock::now()); - const auto data_size = data.size(); - const auto time_budget = GetTuningTimeMax(); - const auto& profile_h = context.GetStream(); - // start the counter - for(auto idx = thread_index; idx < data_size; idx += total_threads) - { - // Check if we are out of time - const auto current_time = std::chrono::time_point_cast( - std::chrono::steady_clock::now()); - if(current_time - start_time > time_budget) - { - MIOPEN_LOG_I2("Thread: " << thread_index << " Done, exhausted time budget"); - auto tmp = std::make_tuple({}, {}, true); - comp_queue.push(std::move(tmp)); - break; - } - auto& current_config = data.at(idx); - ConvSolution current_solution = s.GetSolution(context, problem, current_config); - for(const auto& kernel : current_solution.construction_params) - { - if(profile_h.HasProgram(kernel.kernel_file, kernel.comp_options)) - continue; - std::ignore = profile_h.LoadProgram(kernel.kernel_file, kernel.comp_options, ""); - } - auto tup = std::make_tuple( - std::move(current_config), std::move(current_solution), false); - comp_queue.push(std::move(tup)); - } - MIOPEN_LOG_I2("Thread: " << thread_index << " Done, completed tuning"); -} - -template -auto GenericSearch(const Solver s, - const Context& context_, - const Problem& problem, - const AnyInvokeParams& invoke_ctx_) - -> decltype(s.GetDefaultPerformanceConfig(context_, problem)) -{ - static_assert( - !(HasMember{} || - HasMember{}), - "RunAndMeasure is obsolete. Solvers should implement auto-tune evaluation in invoker"); - - auto context = context_; - context.is_for_generic_search = true; - - using PerformanceConfig = decltype(s.GetDefaultPerformanceConfig(context, problem)); - PerformanceConfig best_config; - const auto default_solution = - s.GetSolution(context, problem, s.GetDefaultPerformanceConfig(context, problem)); - const auto invoke_ctx = [invoke_ctx_]() { - auto copy = invoke_ctx_; - copy.SetInvokeType(InvokeType::AutoTune); - return copy; - }(); - - auto& profile_h = context.GetStream(); - const AutoEnableProfiling enableProfiling{profile_h}; - - auto tmp_all_configs = GetAllConfigs(s, context, problem); - // For random access - std::vector all_configs; - std::copy(tmp_all_configs.begin(), tmp_all_configs.end(), std::back_inserter(all_configs)); - // shuffle the configs - std::random_device rd{}; - auto rng = std::default_random_engine{rd()}; - std::shuffle(all_configs.begin(), all_configs.end(), rng); - std::size_t n_runs_total = std::min(all_configs.size(), GetTuningIterationsMax()); - all_configs.resize(n_runs_total); - std::size_t patience = env::value(MIOPEN_TUNING_PATIENCE); - - if(all_configs.empty()) - { - const auto default_config = s.GetDefaultPerformanceConfig(context, problem); - - if(default_config.IsValid(context, problem)) - { - all_configs.emplace_back(default_config); - n_runs_total += 1; - } - else - { - const auto id = s.SolverDbId(); - MIOPEN_THROW("Generic search has failed. Solver " + id + - " cannot produce any valid configuration."); - } - } - - bool is_passed = false; // left false only if all iterations failed. - float best_time = std::numeric_limits::max(); - size_t n_failed = 0; - size_t n_best = 0; - HeartBeat heartbeat; - heartbeat.Start(); - - const auto total_threads = GetTuningThreadsMax(); - - ThreadSafeQueue> solution_queue; - std::vector compile_agents; - compile_agents.reserve(total_threads); - for(auto idx = 0; idx < total_threads; ++idx) - { - compile_agents.emplace_back(CompileAgent, - idx, - total_threads, - std::cref(s), - std::cref(context), - std::cref(problem), - std::ref(all_configs), - std::ref(solution_queue)); - } - - if(!env::enabled(MIOPEN_DEBUG_COMPILE_ONLY)) - { - size_t n_current = 0; - size_t last_imprv = 0; - auto threads_remaining = total_threads; - while(true) - { - if(n_current >= n_runs_total) - { - MIOPEN_LOG_I2("Ending Search by total runs: " << n_runs_total); - break; - } - if(last_imprv >= patience) - { - MIOPEN_LOG_I2("Ending Search by patience: " << patience); - break; - } - - last_imprv++; - MIOPEN_LOG_I2("Waiting for item in queue"); - const auto kinder = solution_queue.pop(); - auto current_config = std::get<0>(kinder); - auto current_solution = std::get<1>(kinder); - - if(std::get<2>(kinder)) - { - threads_remaining--; - if(threads_remaining == 0) - { - break; - } - else - { - continue; - } - } - - float elapsed_time = 0.0f; - int ret = 0; - MIOPEN_LOG_I2('#' << n_current << '/' << n_failed << '/' << n_runs_total << ' ' - << current_config); - - Invoker invoker; - - try - { - if(default_solution.workspace_sz != current_solution.workspace_sz) - { - ret = -2; - MIOPEN_LOG_E('#' << n_current << " (" << n_runs_total << ") " - << "Workspace size should not depend on PerformanceConfig: " - << default_solution.workspace_sz - << " != " << current_solution.workspace_sz); - } - - invoker = profile_h.PrepareInvoker(*current_solution.invoker_factory, - current_solution.construction_params); - invoker(profile_h, invoke_ctx); - elapsed_time = profile_h.GetKernelTime(); - } - catch(const std::exception& e) - { - MIOPEN_LOG_E("Error: Exception encountered : " << e.what()); - ret = 1; - } - catch(...) - { - MIOPEN_LOG_E("Error: Unknown exception thrown."); - ret = 1; - } - - MIOPEN_LOG_T("##" - << "(n_current, n_failed, n_runs_total): " << n_current << '/' << n_failed - << '/' << n_runs_total << " elapsed_time: " << elapsed_time - << ", best_time: " << best_time << ", " << current_config); - - if(ret == 0) - { - // Smooth the jitter of measurements: - // If the 1st probe is NOT too bad (measured time <= 1.10 * best known time), - // then re-run it 9 times more and compute average time, - // and decide using average of all 10 attempts vs. the best. - constexpr int N_RUNS = 10; - if(elapsed_time / best_time < 1.10f) - { - MIOPEN_LOG_I2("Finding average for: " << elapsed_time << " / " << best_time - << " = " << (elapsed_time / best_time)); - - try - { - for(int i = 1; i < N_RUNS; ++i) - { - invoker(profile_h, invoke_ctx); - elapsed_time += profile_h.GetKernelTime(); - } - } - catch(...) - { - ret = 1; - } - - if(ret == 0) - { - is_passed = true; - elapsed_time /= N_RUNS; - if(elapsed_time < best_time) - { - MIOPEN_LOG_I('#' << n_current << '/' << n_failed << '/' << n_runs_total - << ' ' << elapsed_time << " < " << best_time << ' ' - << current_config); - best_config = current_config; - best_time = elapsed_time; - n_best = n_current; - last_imprv = 0; - } - else - { - MIOPEN_LOG_I2("Average is not better: " << elapsed_time - << " >= " << best_time); - } - } - } - } - - // Banchmarked kernels will not be used anymore. - // Now we can delete Program objects that belong to OCL/HIP - // runtime and free the associated resources (memory, file handles...) - for(const auto& kernelInfo : current_solution.construction_params) - profile_h.ClearProgram(kernelInfo.kernel_file, kernelInfo.comp_options); - - if(ret != 0) - { - MIOPEN_LOG_E('#' << n_current << " (" << n_runs_total << ") " - << " Failed rc=" << ret); - ++n_failed; - } - heartbeat.Monitor(ret != 0, - elapsed_time, - n_current, - best_time, - n_failed, - n_runs_total, - current_config); - ++n_current; - } - } - else - { - MIOPEN_THROW(miopenStatusGpuOperationsSkipped, - "Running kernels on GPU is disabled. Search skipped"); - } - - for(auto& agent : compile_agents) - agent.join(); - - MIOPEN_LOG_W("Done: " << n_runs_total << '/' << n_failed << '/' << n_runs_total << ", best #" - << n_best << ' ' << best_time << ' ' << best_config); - - if(!is_passed) - MIOPEN_THROW("Search failed"); - // Run once with the default config and show score. - - const auto& invoker = profile_h.PrepareInvoker(*default_solution.invoker_factory, - default_solution.construction_params); - invoker(profile_h, invoke_ctx); - const auto default_time = profile_h.GetKernelTime(); - const auto score = (best_time > 0.0f) ? default_time / best_time : 0.0f; - MIOPEN_LOG_W("...Score: " << score << " (default time " << default_time << ')'); - - return best_config; -} - -} // namespace solver -} // namespace miopen - -#endif // GUARD_MIOPEN_GENERIC_SEARCH_HPP_ diff --git a/src/include/miopen/generic_search_controls.hpp b/src/include/miopen/generic_search_controls.hpp index 65cc0ff3dc..e69de29bb2 100644 --- a/src/include/miopen/generic_search_controls.hpp +++ b/src/include/miopen/generic_search_controls.hpp @@ -1,48 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once -#include -#include -#include -#include - -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DEBUG_TUNING_ITERATIONS_MAX, - std::numeric_limits::max()) -MIOPEN_DECLARE_ENV_VAR_UINT64( - MIOPEN_TUNING_TIME_MS_MAX, - std::chrono::duration_cast(std::chrono::hours{2}).count()) -MIOPEN_DECLARE_ENV_VAR_UINT64( - MIOPEN_TUNING_PATIENCE, - std::numeric_limits::max()) // End tuning if no improvement in X iterations - -#if MIOPEN_USE_COMGR -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_COMPILE_PARALLEL_LEVEL, 1) // COMGR is not parallelizable -#else -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_COMPILE_PARALLEL_LEVEL, - std::thread::hardware_concurrency() / 2) -#endif -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_COMPILE_ONLY) diff --git a/src/include/miopen/glu.hpp b/src/include/miopen/glu.hpp index 448929ef94..e69de29bb2 100644 --- a/src/include/miopen/glu.hpp +++ b/src/include/miopen/glu.hpp @@ -1,55 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include - -namespace miopen { - -struct Handle; -struct TensorDescriptor; - -namespace glu { - -MIOPEN_INTERNALS_EXPORT miopenStatus_t GLUForward(Handle& handle, - const TensorDescriptor& inputDesc, - ConstData_t input, - const TensorDescriptor& outputDesc, - Data_t output, - uint32_t dim); - -MIOPEN_INTERNALS_EXPORT miopenStatus_t GLUBackward(Handle& handle, - const TensorDescriptor& inputDesc, - ConstData_t input, - const TensorDescriptor& outputGradDesc, - ConstData_t outputGrad, - const TensorDescriptor& inputGradDesc, - Data_t inputGrad, - uint32_t dim); - -} // namespace glu - -} // namespace miopen diff --git a/src/include/miopen/glu/invoke_params.hpp b/src/include/miopen/glu/invoke_params.hpp index 0a8bc6fb62..e69de29bb2 100644 --- a/src/include/miopen/glu/invoke_params.hpp +++ b/src/include/miopen/glu/invoke_params.hpp @@ -1,71 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include - -namespace miopen { -namespace glu { - -struct FwdInvokeParams : public miopen::InvokeParams -{ - FwdInvokeParams() = default; - - const TensorDescriptor* inputDesc = nullptr; - const TensorDescriptor* outputDesc = nullptr; - - ConstData_t input = nullptr; - Data_t output = nullptr; - uint32_t dim = 0; - - std::size_t GetWorkspaceSize() const { return 0; } - Data_t GetWorkspace() const { return nullptr; } -}; - -struct BwdInvokeParams : public miopen::InvokeParams -{ - BwdInvokeParams() = default; - - const TensorDescriptor* inputDesc = nullptr; - const TensorDescriptor* inputGradDesc = nullptr; - const TensorDescriptor* outputGradDesc = nullptr; - - ConstData_t input = nullptr; - ConstData_t outputGrad = nullptr; - Data_t inputGrad = nullptr; - uint32_t dim = 0; - - std::size_t GetWorkspaceSize() const { return 0; } - Data_t GetWorkspace() const { return nullptr; } -}; - -} // namespace glu - -} // namespace miopen diff --git a/src/include/miopen/glu/problem_description.hpp b/src/include/miopen/glu/problem_description.hpp index d5a6523d66..e69de29bb2 100644 --- a/src/include/miopen/glu/problem_description.hpp +++ b/src/include/miopen/glu/problem_description.hpp @@ -1,83 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include - -namespace miopen { - -struct NetworkConfig; - -namespace glu { - -enum class Direction -{ - Forward, - Backward, -}; - -struct ProblemDescription : ProblemDescriptionBase -{ - // Forward constructor - ProblemDescription(const TensorDescriptor& inputDesc_, - const TensorDescriptor& outputDesc_, - uint32_t dim_); - - // Backward constructor - ProblemDescription(const TensorDescriptor& inputDesc_, - const TensorDescriptor& outputGradDesc_, - const TensorDescriptor& inputGradDesc_, - uint32_t dim_); - - Direction GetDirection() const { return direction; } - const TensorDescriptor& GetInputDesc() const { return inputDesc; } - const TensorDescriptor& GetInputGradDesc() const { return inputGradDesc; } - const TensorDescriptor& GetOutputDesc() const { return outputDesc; } - const TensorDescriptor& GetOutputGradDesc() const { return outputGradDesc; } - uint32_t GetDim() const { return dim; } - - bool IsSameType() const; - - bool IsAllContiguous() const; - - NetworkConfig MakeNetworkConfig() const override; - -private: - Direction direction; - TensorDescriptor inputDesc; - TensorDescriptor outputDesc; - TensorDescriptor outputGradDesc; - TensorDescriptor inputGradDesc; - - uint32_t dim; -}; - -} // namespace glu - -} // namespace miopen diff --git a/src/include/miopen/glu/solvers.hpp b/src/include/miopen/glu/solvers.hpp index 4531d94008..e69de29bb2 100644 --- a/src/include/miopen/glu/solvers.hpp +++ b/src/include/miopen/glu/solvers.hpp @@ -1,63 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include - -namespace miopen { - -namespace solver { - -namespace glu { - -using GLUSolver = NonTunableSolverBase; - -struct GLUForward final : GLUSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext& context, - const miopen::glu::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::glu::ProblemDescription& problem) const override; -}; - -struct GLUBackward final : GLUSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext& context, - const miopen::glu::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::glu::ProblemDescription& problem) const override; -}; - -} // namespace glu - -} // namespace solver - -} // namespace miopen diff --git a/src/include/miopen/graphapi/tensor.hpp b/src/include/miopen/graphapi/tensor.hpp index 0284734f2d..e69de29bb2 100644 --- a/src/include/miopen/graphapi/tensor.hpp +++ b/src/include/miopen/graphapi/tensor.hpp @@ -1,159 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include - -#include -#include - -namespace miopen { - -namespace graphapi { - -class Tensor : public TensorDescriptor -{ -private: - int64_t mId = 0; - bool mVirtual = false; - - // Deprecated - using TensorDescriptor::GetLayout_t; - -public: - Tensor() noexcept = default; - Tensor(const Tensor&) = default; - Tensor(Tensor&&) noexcept = default; - Tensor& operator=(const Tensor&) = default; - Tensor& operator=(Tensor&&) noexcept = default; - Tensor(const TensorDescriptor& other, int64_t id, bool isVirtual) - : TensorDescriptor(other), mId(id), mVirtual(isVirtual) - { - } - Tensor(TensorDescriptor&& other, int64_t id, bool isVirtual) - : TensorDescriptor(std::move(other)), mId(id), mVirtual(isVirtual) - { - } - Tensor(miopenDataType_t dataType, - const std::vector& dimensions, - const std::vector& strides, - int64_t id, - bool isVirtual) - : TensorDescriptor(dataType, dimensions, strides), mId(id), mVirtual(isVirtual) - { - } - Tensor(miopenDataType_t dataType, - std::vector&& dimensions, - std::vector&& strides, - int64_t id, - bool isVirtual) noexcept - : TensorDescriptor(dataType, std::move(dimensions), std::move(strides)), - mId(id), - mVirtual(isVirtual) - { - } - - int64_t getId() const noexcept { return mId; } - bool isVirtual() const noexcept { return mVirtual; } -}; - -class MIOPEN_INTERNALS_EXPORT TensorBuilder -{ -private: - std::vector mDimensions; - std::vector mStrides; - int64_t mId = 0; - miopenDataType_t mDataType = miopenFloat; - bool mVirtual = false; - bool mUniqueIdSet = false; - bool mDataTypeSet = false; - bool mDimensionsSet = false; - bool mStridesSet = false; - -public: - TensorBuilder& setDataType(miopenDataType_t dataType) &; - TensorBuilder& setDim(const std::vector& dimensions) &; - TensorBuilder& setDim(std::vector&& dimensions) &; - TensorBuilder& setStride(const std::vector& strides) &; - TensorBuilder& setStride(std::vector&& strides) &; - TensorBuilder& setId(int64_t id) &; - TensorBuilder& setVirtual(bool isVirtual) &; - - TensorBuilder&& setDataType(miopenDataType_t dataType) && - { - return std::move(setDataType(dataType)); - } - TensorBuilder&& setDim(const std::vector& dimensions) && - { - return std::move(setDim(dimensions)); - } - TensorBuilder&& setDim(std::vector&& dimensions) && - { - return std::move(setDim(std::move(dimensions))); - } - TensorBuilder&& setStride(const std::vector& strides) && - { - return std::move(setStride(strides)); - } - TensorBuilder&& setStride(std::vector&& strides) && - { - return std::move(setStride(std::move(strides))); - } - TensorBuilder&& setId(int64_t id) && { return std::move(setId(id)); } - TensorBuilder&& setVirtual(bool isVirtual) && { return std::move(setVirtual(isVirtual)); } - - Tensor build() const&; - Tensor build() &&; -}; - -class MIOPEN_INTERNALS_EXPORT BackendTensorDescriptor : public BackendDescriptor -{ -private: - TensorBuilder mBuilder; - Tensor mDescriptor; - -public: - BackendTensorDescriptor() = default; - virtual ~BackendTensorDescriptor() override; - virtual void setAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t elementCount, - void* arrayOfElements) override; - virtual void finalize() override; - virtual void getAttribute(miopenBackendAttributeName_t attributeName, - miopenBackendAttributeType_t attributeType, - int64_t requestedElementCount, - int64_t* elementCount, - void* arrayOfElements) override; - - const Tensor* getTensor() const { return &mDescriptor; } - Tensor* getTensor() { return &mDescriptor; } -}; - -} // namespace graphapi - -} // namespace miopen diff --git a/src/include/miopen/graphapi/util.hpp b/src/include/miopen/graphapi/util.hpp index 303cefaeca..e69de29bb2 100644 --- a/src/include/miopen/graphapi/util.hpp +++ b/src/include/miopen/graphapi/util.hpp @@ -1,271 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include - -#include -#include -#include -#include - -namespace miopen { -namespace graphapi { - -inline std::string tensorIdAsStr(int64_t tens_id) -{ - - char* b = reinterpret_cast(&tens_id); - - return {b, sizeof(tens_id)}; -} - -template -Tensor makeTensor(std::string_view name, miopenDataType_t dt, const Vec& dims, const Vec& strides) -{ - int64_t id = 0; - MIOPEN_THROW_IF(name.size() > sizeof(id), "tensor name exceeds 8 chars"); - std::copy_n(name.begin(), std::min(sizeof(id), name.size()), reinterpret_cast(&id)); - - return TensorBuilder{} - .setDataType(dt) - .setDim(dims) - .setStride(strides) - .setId(id) - .setVirtual(isVirtual) - .build(); -} - -template -Tensor makeTensor(std::string_view name, miopenDataType_t dt, const Vec& dims) -{ - TensorDescriptor desc{dt, dims}; - return makeTensor(name, dt, desc.GetLengths(), desc.GetStrides()); -} - -/// An RAII style class that captures a pointer to an object on heap and frees it -/// upon destruction. It's different from std::unique_ptr in that it allows -/// capturing multiple types of pointers -struct HeapPtrDeleter -{ - using Fn = std::function; - Fn mFn = {}; - - template - explicit HeapPtrDeleter(T* ptr) - : mFn([ptr]() { delete ptr; }) // NOLINT (cppcoreguidelines-owning-memory) - { - } - - HeapPtrDeleter(const HeapPtrDeleter&) = delete; - HeapPtrDeleter& operator=(const HeapPtrDeleter&) = delete; - - friend void swap(HeapPtrDeleter& left, HeapPtrDeleter& right) noexcept - { - std::swap(left.mFn, right.mFn); - } - - HeapPtrDeleter(HeapPtrDeleter&& that) noexcept : mFn(std::move(that.mFn)) { that.mFn = {}; } - - HeapPtrDeleter& operator=(HeapPtrDeleter&& that) noexcept - { - if(this != &that) - { - HeapPtrDeleter tmp{std::move(that)}; - swap(*this, tmp); - } - return *this; - } - - ~HeapPtrDeleter() - { - // default initialized std::function cannot be invoked - if(mFn) - mFn(); - } -}; - -/// an automatically deleting allocator that frees the allocated objects upon -/// destruction -struct AutoDeleteAllocator -{ - std::vector mPtrsToFree; - - AutoDeleteAllocator() = default; - AutoDeleteAllocator(const AutoDeleteAllocator&) = delete; - AutoDeleteAllocator& operator=(const AutoDeleteAllocator&) = delete; - - AutoDeleteAllocator(AutoDeleteAllocator&&) = default; - AutoDeleteAllocator& operator=(AutoDeleteAllocator&&) = default; - ~AutoDeleteAllocator() = default; - - template - T* allocate(T&& val) - { - T* ret = new T(std::forward(val)); // NOLINT (cppcoreguidelines-owning-memory) - mPtrsToFree.emplace_back(ret); - return ret; - } -}; - -struct PatternGraphGenerator -{ - - struct DummyNode : public OpNode - { - std::string mName; - std::vector mInTensors; - std::vector mOutTensors; - - DummyNode(const std::string& name, - const std::vector& ins, - const std::vector& outs) - : mName(name), mInTensors(ins), mOutTensors(outs) - { - } - - const std::string& signName() const final { return mName; } - - std::vector getInTensors() const final { return mInTensors; } - - std::vector getOutTensors() const final { return mOutTensors; } - }; - - struct DummyNodeGenSpec - { - std::string mName; - std::vector mInTensors; - std::vector mOutTensors; - }; - - inline Tensor* makeDummyTensor(std::string_view name) - { - - return mAlloc.allocate(makeTensor(name, miopenFloat, std::vector({1}))); - } - -private: - AutoDeleteAllocator mAlloc{}; - OpGraph mGraph{}; - - PatternGraphGenerator(const std::vector& node_specs) - { - - std::unordered_map tensor_map; - OpGraphBuilder builder; - - for(const auto& ns : node_specs) - { - std::vector in_tensors; - - for(const auto& ti : ns.mInTensors) - { - auto [it, flag] = tensor_map.try_emplace(ti, makeDummyTensor(ti)); - in_tensors.emplace_back(it->second); - } - - std::vector out_tensors; - for(const auto& to : ns.mOutTensors) - { - auto [it, flag] = tensor_map.try_emplace(to, makeDummyTensor(to)); - out_tensors.emplace_back(it->second); - } - - builder.addNode(mAlloc.allocate(DummyNode{ns.mName, in_tensors, out_tensors})); - } - - mGraph = std::move(builder).build(); - } - -public: - PatternGraphGenerator() = default; - PatternGraphGenerator(const PatternGraphGenerator&) = delete; - PatternGraphGenerator& operator=(const PatternGraphGenerator&) = delete; - PatternGraphGenerator(PatternGraphGenerator&&) = default; - PatternGraphGenerator& operator=(PatternGraphGenerator&&) = default; - ~PatternGraphGenerator() = default; - - static std::unique_ptr - Make(const std::vector& node_specs) - { - return std::unique_ptr(new PatternGraphGenerator(node_specs)); - } - - const auto& graph() const { return mGraph; } -}; - -/// \todo move this function out so that other find 2.0 code can use it -/// --amberhassaan May, 2024 -inline std::string_view tensorEnumIdToStr(miopenTensorArgumentId_t id) -{ - -#define ENUM_CASE(k) \ - case k: return #k; - - switch(id) - { - ENUM_CASE(miopenTensorMhaK) - ENUM_CASE(miopenTensorMhaQ) - ENUM_CASE(miopenTensorMhaV) - ENUM_CASE(miopenTensorMhaDescaleK) - ENUM_CASE(miopenTensorMhaDescaleQ) - ENUM_CASE(miopenTensorMhaDescaleV) - ENUM_CASE(miopenTensorMhaDescaleS) - ENUM_CASE(miopenTensorMhaScaleS) - ENUM_CASE(miopenTensorMhaScaleO) - ENUM_CASE(miopenTensorMhaDropoutProbability) - ENUM_CASE(miopenTensorMhaDropoutSeed) - ENUM_CASE(miopenTensorMhaDropoutOffset) - ENUM_CASE(miopenTensorMhaO) - ENUM_CASE(miopenTensorMhaAmaxO) - ENUM_CASE(miopenTensorMhaAmaxS) - ENUM_CASE(miopenTensorMhaM) - ENUM_CASE(miopenTensorMhaZInv) - ENUM_CASE(miopenTensorMhaDO) - ENUM_CASE(miopenTensorMhaDescaleO) - ENUM_CASE(miopenTensorMhaDescaleDO) - ENUM_CASE(miopenTensorMhaDescaleDS) - ENUM_CASE(miopenTensorMhaScaleDS) - ENUM_CASE(miopenTensorMhaScaleDQ) - ENUM_CASE(miopenTensorMhaScaleDK) - ENUM_CASE(miopenTensorMhaScaleDV) - ENUM_CASE(miopenTensorMhaDQ) - ENUM_CASE(miopenTensorMhaDK) - ENUM_CASE(miopenTensorMhaDV) - ENUM_CASE(miopenTensorMhaAmaxDQ) - ENUM_CASE(miopenTensorMhaAmaxDK) - ENUM_CASE(miopenTensorMhaAmaxDV) - ENUM_CASE(miopenTensorMhaAmaxDS) - ENUM_CASE(miopenTensorMhaBias) - ENUM_CASE(miopenTensorMhaMask) - default: MIOPEN_THROW(miopenStatusInternalError, "unknown tensor enum id"); - } -#undef ENUM_CASE -} - -} // end namespace graphapi -} // end namespace miopen diff --git a/src/include/miopen/kthvalue.hpp b/src/include/miopen/kthvalue.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/include/miopen/kthvalue/invoke_params.hpp b/src/include/miopen/kthvalue/invoke_params.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/include/miopen/kthvalue/problem_description.hpp b/src/include/miopen/kthvalue/problem_description.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/include/miopen/kthvalue/solvers.hpp b/src/include/miopen/kthvalue/solvers.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/include/miopen/mha/mha.hpp b/src/include/miopen/mha/mha.hpp index aa75ad4f8e..e69de29bb2 100644 --- a/src/include/miopen/mha/mha.hpp +++ b/src/include/miopen/mha/mha.hpp @@ -1,190 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include - -namespace miopen { -namespace mha { - -struct MhaInputDescsForward -{ - // input tensors - TensorDescriptor kDesc; - TensorDescriptor qDesc; - TensorDescriptor vDesc; - - // input scaling tensors - TensorDescriptor descaleKDesc; - TensorDescriptor descaleQDesc; - TensorDescriptor descaleVDesc; - TensorDescriptor descaleSDesc; - TensorDescriptor scaleSDesc; - TensorDescriptor scaleODesc; - - // input scalars - float scale; - - // input dropout tensors - TensorDescriptor dropoutProbabilityDesc; - TensorDescriptor dropoutSeedDesc; - TensorDescriptor dropoutOffsetDesc; - - TensorDescriptor biasDesc; - - // output tensors - TensorDescriptor oDesc; - TensorDescriptor amaxODesc; - TensorDescriptor amaxSDesc; - - // output tensors for training only - TensorDescriptor mDesc; - TensorDescriptor zInvDesc; -}; - -struct MhaInputDescsBackward -{ - // input tensors - TensorDescriptor kDesc; - TensorDescriptor qDesc; - TensorDescriptor vDesc; - - TensorDescriptor oDesc; - TensorDescriptor doDesc; - - // input tensors from fwd pass - TensorDescriptor mDesc; - TensorDescriptor zInvDesc; - - // input scaling tensors - TensorDescriptor descaleKDesc; - TensorDescriptor descaleQDesc; - TensorDescriptor descaleVDesc; - TensorDescriptor descaleSDesc; - TensorDescriptor descaleODesc; - TensorDescriptor descaleDODesc; - TensorDescriptor descaleDSDesc; - - TensorDescriptor scaleSDesc; - TensorDescriptor scaleDSDesc; - TensorDescriptor scaleDQDesc; - TensorDescriptor scaleDKDesc; - TensorDescriptor scaleDVDesc; - - // input scalars - float scale; - - TensorDescriptor dropoutProbabilityDesc; - TensorDescriptor dropoutSeedDesc; - TensorDescriptor dropoutOffsetDesc; - - // output tensors - TensorDescriptor dqDesc; - TensorDescriptor dkDesc; - TensorDescriptor dvDesc; - TensorDescriptor amaxDQDesc; - TensorDescriptor amaxDKDesc; - TensorDescriptor amaxDVDesc; - TensorDescriptor amaxDSDesc; -}; - -struct MhaDataForward -{ - // input tensors - ConstData_t kData; - ConstData_t qData; - ConstData_t vData; - - // input scaling tensors - ConstData_t descaleKData; - ConstData_t descaleQData; - ConstData_t descaleVData; - ConstData_t descaleSData; - ConstData_t scaleSData; - ConstData_t scaleOData; - - ConstData_t dropoutProbabilityData; - ConstData_t dropoutSeedData; - ConstData_t dropoutOffsetData; - - ConstData_t biasData; - miopenMhaMask_t mask; - - // output tensors - Data_t oData; - Data_t amaxOData; - Data_t amaxSData; - - // output tensors for training only - Data_t mData; - Data_t zInvData; -}; - -struct MhaDataBackward -{ - // input tensors - ConstData_t kData; - ConstData_t qData; - ConstData_t vData; - - ConstData_t oData; - ConstData_t doData; - - // input tensors from fwd pass - ConstData_t mData; - ConstData_t zInvData; - - // input scaling tensors - ConstData_t descaleKData; - ConstData_t descaleQData; - ConstData_t descaleVData; - ConstData_t descaleSData; - ConstData_t descaleOData; - ConstData_t descaleDOData; - ConstData_t descaleDSData; - ConstData_t scaleSData; - ConstData_t scaleDSData; - ConstData_t scaleDQData; - ConstData_t scaleDKData; - ConstData_t scaleDVData; - - ConstData_t dropoutProbabilityData; - ConstData_t dropoutSeedData; - ConstData_t dropoutOffsetData; - - // output tensors - Data_t dqData; - Data_t dkData; - Data_t dvData; - Data_t amaxDQData; - Data_t amaxDKData; - Data_t amaxDVData; - Data_t amaxDSData; -}; - -} // namespace mha -} // namespace miopen diff --git a/src/include/miopen/nogpu/handle_impl.hpp b/src/include/miopen/nogpu/handle_impl.hpp index 3a1ebdf295..e69de29bb2 100644 --- a/src/include/miopen/nogpu/handle_impl.hpp +++ b/src/include/miopen/nogpu/handle_impl.hpp @@ -1,69 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_NOGPU_HANDLE_IMPL_HPP_ -#define GUARD_MIOPEN_NOGPU_HANDLE_IMPL_HPP_ -namespace miopen { - -struct HandleImpl -{ - using StreamPtr = std::shared_ptr::type>; - - HandleImpl() : ctx() {} - - void elapsed_time(hipEvent_t start, hipEvent_t stop) - { - if(enable_profiling) - hipEventElapsedTime(&this->profiling_result, start, stop); - } - - std::function elapsed_time_handler() - { - return std::bind( - &HandleImpl::elapsed_time, this, std::placeholders::_1, std::placeholders::_2); - } - - bool enable_profiling = false; - StreamPtr stream = nullptr; - rocblas_handle_ptr rhandle_; -#if MIOPEN_USE_HIPBLASLT - hipblasLt_handle_ptr hip_blasLt_handle; -#endif - float profiling_result = 0.0; - int device = -1; - std::string device_name; - std::size_t num_cu = 0; - std::size_t local_mem_size = 0; - std::size_t global_mem_size = 0; - std::size_t img3d_max_width = 0; - std::size_t warp_size = 64; - std::size_t max_mem_alloc_size = 0; - Allocator allocator{}; - KernelCache cache; - std::int64_t ctx; - TargetProperties target_properties; -}; -} // namespace miopen -#endif // GUARD_MIOPEN_NOGPU_HANDLE_IMPL_HPP_ diff --git a/src/include/miopen/reduce/problem_description.hpp b/src/include/miopen/reduce/problem_description.hpp index 7e327ba565..e69de29bb2 100644 --- a/src/include/miopen/reduce/problem_description.hpp +++ b/src/include/miopen/reduce/problem_description.hpp @@ -1,310 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#pragma once - -#include -#include -#include -#include -#include - -namespace miopen { - -struct NetworkConfig; - -namespace reduce { - -struct ProblemDescriptionExtreme : ProblemDescriptionBase -{ - ProblemDescriptionExtreme(const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - const TensorDescriptor& indiceDesc_, - int32_t dim_, - miopenReduceExtremeOp_t reduceExtremeOp_) - : xDesc(xDesc_), - yDesc(yDesc_), - indiceDesc(indiceDesc_), - dim(dim_), - reduceExtremeOp(reduceExtremeOp_) - { - } - - ProblemDescriptionExtreme(const TensorDescriptor& xDesc_, - const TensorDescriptor& indiceDesc_, - int32_t dim_, - miopenReduceExtremeOp_t reduceExtremeOp_) - : xDesc(xDesc_), indiceDesc(indiceDesc_), dim(dim_), reduceExtremeOp(reduceExtremeOp_) - { - } - - const TensorDescriptor& GetXDesc() const { return xDesc; } - const TensorDescriptor& GetYDesc() const { return yDesc; } - const TensorDescriptor& GetIndiceDesc() const { return indiceDesc; } - int32_t GetDim() const { return dim; } - - bool IsValidLength() const - { - if(xDesc.GetLengths().size() == 1) - return true; - - int32_t posy = 0; - for(int32_t i = 0; i < xDesc.GetLengths().size(); ++i) - { - if(i == dim) - continue; - - if(xDesc.GetLengths()[i] != yDesc.GetLengths()[posy]) - { - MIOPEN_THROW(miopenStatusBadParm, "Reduce: Tensor dimension lengths do not match."); - } - - ++posy; - } - return true; - } - - bool IsValidLengthIndice() const - { - if(xDesc.GetLengths().size() == 1) - return true; - - int32_t posy = 0; - for(int32_t i = 0; i < xDesc.GetLengths().size(); ++i) - { - if(i == dim) - continue; - - if(xDesc.GetLengths()[i] != indiceDesc.GetLengths()[posy]) - { - MIOPEN_THROW(miopenStatusBadParm, "Reduce: Tensor dimension lengths do not match."); - } - - ++posy; - } - return true; - } - - bool IsValidDim() const - { - if((dim < 0) || (dim > xDesc.GetLengths().size())) - { - MIOPEN_THROW( - miopenStatusBadParm, - "Reduce: is greater than 0 and less than or equal tensor dimension length."); - } - return true; - } - - bool IsValidInputNumel() const - { - auto xdims = xDesc.GetLengths(); - auto input_numel = - std::accumulate(xdims.begin(), xdims.end(), 1ULL, std::multiplies()); - if(input_numel > INT32_MAX) - MIOPEN_THROW(miopenStatusBadParm, "Reduce: input numel is bigger than INT_MAX."); - - return true; - } - - bool IsSameType() const - { - if(xDesc.GetType() != yDesc.GetType()) - { - return false; - } - return true; - } - - bool IsAllContiguous() const - { - if(!(xDesc.IsContiguous() && yDesc.IsContiguous())) - { - return false; - } - - return true; - } - - bool IsAllContiguousWithIndice() const - { - if(!(xDesc.IsContiguous() && yDesc.IsContiguous() && indiceDesc.IsContiguous())) - { - return false; - } - - return true; - } - - bool IsAllContiguousIndice() const - { - if(!(xDesc.IsContiguous() && indiceDesc.IsContiguous())) - { - return false; - } - - return true; - } - - bool IsNotLastDim() const - { - if(dim == xDesc.GetLengths().size() - 1) - return false; - return true; - } - - bool IsLargeReduceSize() const - { - if(xDesc.GetLengths()[dim] > 64) - return false; - return true; - } - - NetworkConfig MakeNetworkConfig() const override; - -private: - TensorDescriptor xDesc; - TensorDescriptor yDesc; - TensorDescriptor indiceDesc; - - int32_t dim; - - miopenReduceExtremeOp_t reduceExtremeOp; - - NetworkConfig MakeForwardNetworkConfig() const; -}; - -struct ProblemDescriptionCalculation : ProblemDescriptionBase -{ - ProblemDescriptionCalculation(miopenReduceCalculationNanPropagation_t nanPropagation_, - const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - int32_t dim_, - miopenReduceCalculationOp_t reduceCalculationOp_) - : nanPropagation(nanPropagation_), - xDesc(xDesc_), - yDesc(yDesc_), - dim(dim_), - reduceCalculationOp(reduceCalculationOp_) - { - } - - miopenReduceCalculationNanPropagation_t GetNanPropagation_() const { return nanPropagation; } - const TensorDescriptor& GetXDesc() const { return xDesc; } - const TensorDescriptor& GetYDesc() const { return yDesc; } - int32_t GetDim() const { return dim; } - - bool IsValidLength() const - { - if(xDesc.GetLengths().size() == 1) - return true; - - int32_t posy = 0; - for(int32_t i = 0; i < xDesc.GetLengths().size(); ++i) - { - if(i == dim) - continue; - - if(xDesc.GetLengths()[i] != yDesc.GetLengths()[posy]) - { - MIOPEN_THROW(miopenStatusBadParm, "Reduce: Tensor dimension lengths do not match."); - } - - ++posy; - } - return true; - } - - bool IsValidDim() const - { - if((dim < 0) || (dim > xDesc.GetLengths().size())) - { - MIOPEN_THROW( - miopenStatusBadParm, - "Reduce: is greater than 0 and less than or equal tensor dimension length."); - } - return true; - } - - bool IsValidInputNumel() const - { - auto xdims = xDesc.GetLengths(); - auto input_numel = - std::accumulate(xdims.begin(), xdims.end(), 1ULL, std::multiplies()); - if(input_numel > INT32_MAX) - MIOPEN_THROW(miopenStatusBadParm, "Reduce: input numel is bigger than INT_MAX."); - - return true; - } - - bool IsSameType() const - { - if(xDesc.GetType() != yDesc.GetType()) - { - return false; - } - return true; - } - - bool IsAllContiguous() const - { - if(!(xDesc.IsContiguous() && yDesc.IsContiguous())) - { - return false; - } - - return true; - } - - bool IsNotLastDim() const - { - if(dim == xDesc.GetLengths().size() - 1) - return false; - return true; - } - - bool IsLargeReduceSize() const - { - if(xDesc.GetLengths()[dim] > 64) - return false; - return true; - } - - NetworkConfig MakeNetworkConfig() const override; - -private: - miopenReduceCalculationNanPropagation_t nanPropagation; - TensorDescriptor xDesc; - TensorDescriptor yDesc; - - int32_t dim; - miopenReduceCalculationOp_t reduceCalculationOp; - - NetworkConfig MakeForwardNetworkConfig() const; -}; - -} // namespace reduce - -} // namespace miopen diff --git a/src/include/miopen/rnn.hpp b/src/include/miopen/rnn.hpp index fcdee74519..e69de29bb2 100644 --- a/src/include/miopen/rnn.hpp +++ b/src/include/miopen/rnn.hpp @@ -1,597 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef GUARD_MIOPEN_RNN_HPP_ -#define GUARD_MIOPEN_RNN_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace miopen { - -struct Handle; -struct TensorDescriptor; - -template -struct c_array_view -{ - T* data; - size_t n; - - using value_type = - typename std::remove_cv::type>::type; - - size_t size() const { return n; } - - const value_type& operator[](size_t i) const { return deref(data[i]); } - - value_type& operator[](size_t i) { return deref(data[i]); } -}; - -void profileRNNkernels(const Handle& handle, unsigned char select, float& ctime); - -struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor -{ - - RNNDescriptor(); - RNNDescriptor(int hsz, - int layers, - miopenRNNMode_t rmode, - miopenRNNInputMode_t inMode, - miopenRNNDirectionMode_t bidir, - miopenRNNBiasMode_t bmode, - miopenRNNAlgo_t amode, - miopenDataType_t dType); - - RNNDescriptor(int hsz, - int layers, - miopenRNNMode_t rmode, - miopenRNNInputMode_t inMode, - miopenRNNDirectionMode_t bidir, - miopenRNNBiasMode_t bmode, - miopenRNNAlgo_t amode, - miopenDataType_t dType, - miopenDropoutDescriptor_t dropDesc); - - size_t hsize; // DLOWELL: is this uniform over all layers? - size_t nLayers; // This may be twice the number of actually wDesc layers since the layout for - // wDesc is 2-D? - - size_t nHiddenTensorsPerLayer; // TODO dlowell: set via constructor, or "set" functions - size_t workspaceScale; - - miopenRNNMode_t rnnMode; - miopenRNNDirectionMode_t dirMode; - miopenRNNAlgo_t algoMode; - miopenRNNInputMode_t inputMode; - miopenRNNBiasMode_t biasMode; - miopenDataType_t dataType; - miopenRNNPaddingMode_t paddingMode = miopenRNNIONotPadded; - - std::size_t typeSize; - miopenDropoutDescriptor_t dropoutDesc{}; - - size_t biasOffsetCalculation(const TensorDescriptor& xDesc, int layer, int biasID) const; - - size_t paramsOffsetCalculation(const TensorDescriptor& xDesc, int layer, int paramID) const; - - std::vector - pTensorLengthsCalculation(const TensorDescriptor& xDesc, int layer, int paramID) const; - - static SeqTensorDescriptor makeSeqTensorDescriptor(miopenDataType_t t, - miopenRNNBaseLayout_t layout, - int maxSeqLength, - int batchSize, - int vectorSize, - const int* lensPerSeq, - const void* padding_marker_ptr); - - static SeqTensorDescriptor makeSeqTensorDescriptor( - c_array_view descs, - size_t seq_len, - miopenRNNBaseLayout_t layout = miopenRNNBaseLayout_t::miopenRNNDataSeqMajorNotPadded); - - static void SeqTensorToTensorDescArray(const SeqTensorDescriptor& desc, - std::vector& td, - std::vector& ptd); - - static miopenRNNBaseLayout_t getBaseLayoutFromDataTensor(const SeqTensorDescriptor& desc); - static std::tuple, bool> - convertRNNBaseLayout(miopenRNNBaseLayout_t layout); - - size_t GetMainSolWorkspaceSize(size_t batchLenSum, - miopenRNNFWDMode_t fwdMode, - miopenRNNBaseLayout_t ioLayout) const; - - size_t GetWorkspaceSize(Handle& handle, - int seqLength, - c_array_view xDesc) const; - size_t GetWorkspaceSize(Handle& handle, - const SeqTensorDescriptor& xDesc, - miopenRNNFWDMode_t fwdMode) const; - - size_t GetReserveSize(size_t batchLenSum) const; - size_t GetReserveSize(Handle& handle, - int seqLength, - c_array_view xDesc) const; - - size_t GetMaxWorkspaceSize(Handle& handle, - const SeqTensorDescriptor& xDesc, - miopenRNNFWDMode_t fwdMode) const; - size_t GetMaxReserveSize(Handle& handle, const SeqTensorDescriptor& xDesc) const; - - size_t - GetParamsSize(Handle& handle, const TensorDescriptor& xDesc, miopenDataType_t dtype) const; - size_t GetParamsSize(size_t inputVector) const; - - void GetParamsDescriptor(Handle& handle, - const TensorDescriptor& xDesc, - TensorDescriptor& wDesc, - miopenDataType_t dtype) const; - - std::size_t - GetLayerParamSize(Handle& handle, int layer, const TensorDescriptor& xDesc, int paramID) const; - - std::size_t GetLayerBiasSize(Handle& handle, int layer, int biasID) const; - - void GetLayerParam(const Handle& handle, - int layer, - const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - ConstData_t w, - int paramID, - TensorDescriptor& paramDesc, - Data_t param) const; - - void GetLayerBias(const Handle& handle, - int layer, - const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - ConstData_t w, - int biasID, - TensorDescriptor& biasDesc, - Data_t bias) const; - - void SetLayerParam(const Handle& handle, - int layer, - const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - Data_t w, - int paramID, - const TensorDescriptor& paramDesc, - ConstData_t param) const; - - void SetLayerBias(const Handle& handle, - int layer, - const TensorDescriptor& xDesc, - const TensorDescriptor& wDesc, - Data_t w, - int biasID, - const TensorDescriptor& biasDesc, - ConstData_t bias) const; - - void SetPaddingmode(miopenRNNPaddingMode_t padding); - - void GetLayerParamOffset(int layer, - const TensorDescriptor& xDesc, - int paramID, - TensorDescriptor& paramDesc, - size_t* paramOffset) const; - - void GetLayerBiasOffset(int layer, - const TensorDescriptor& xDesc, - int biasID, - TensorDescriptor& biasDesc, - size_t* biasOffset) const; - - size_t GetRNNInputSuperTensorSize(Handle& handle, - int seqLength, - c_array_view xDesc) const; - - size_t GetRNNHiddenSuperTensorSize(Handle& handle, - c_array_view xDesc) const; - - void RNNForward(Handle& handle, - miopenRNNFWDMode_t fwdMode, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - Data_t hy, - const TensorDescriptor& cDesc, - ConstData_t cx, - Data_t cy, - const SeqTensorDescriptor& yDesc, - Data_t y, - ConstData_t w, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNForwardTraining(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNForwardInference(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize) const; - - void RNNBackwardData(Handle& handle, - const SeqTensorDescriptor& yDesc, - ConstData_t y, - ConstData_t dy, - const TensorDescriptor& hDesc, - ConstData_t hx, - ConstData_t dhy, - Data_t dhx, - const TensorDescriptor& cDesc, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - const SeqTensorDescriptor& xDesc, - Data_t dx, - ConstData_t w, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNBackwardData(Handle& handle, - int seqLen, - c_array_view yDesc, - ConstData_t y, - c_array_view dyDesc, - ConstData_t dy, - const TensorDescriptor& dhyDesc, - ConstData_t dhy, - const TensorDescriptor& dcyDesc, - ConstData_t dcy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - c_array_view dxDesc, - Data_t dx, - const TensorDescriptor& dhxDesc, - Data_t dhx, - const TensorDescriptor& dcxDesc, - Data_t dcx, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNBackwardWeights(Handle& handle, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - const SeqTensorDescriptor& yDesc, - ConstData_t y, - Data_t dw, - size_t weightSpaceSize, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNBackwardWeights(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - c_array_view dyDesc, - ConstData_t dy, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; - - inline bool isNotRNNskip() const { return inputMode != miopenRNNskip; } - inline bool isRNNskip() const { return inputMode == miopenRNNskip; } - -private: - size_t RNNTransformerWorkspaceSize(const SeqTensorDescriptor& xDesc, - miopenRNNFWDMode_t fwdMode) const; - - // TODO rename - void ModularBackward(Handle& handle, - const SeqTensorDescriptor& yDesc, - ConstData_t dy, - const TensorDescriptor& hDesc, - ConstData_t hx, - ConstData_t dhy, - Data_t dhx, - const TensorDescriptor& cDesc, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - const SeqTensorDescriptor& xDesc, - Data_t dx, - ConstData_t w, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void ModularBackwardWeights(Handle& handle, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - const SeqTensorDescriptor& yDesc, - Data_t w, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t /*reserveSpaceSize*/) const; - - void RNNTransformerForward(Handle& handle, - miopenRNNFWDMode_t fwdMode, - ConstData_t w, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - Data_t hy, - const TensorDescriptor& cDesc, - ConstData_t cx, - Data_t cy, - const SeqTensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNTransformerBackwardData(Handle& handle, - const SeqTensorDescriptor& yDesc, - ConstData_t dy, - const TensorDescriptor& hDesc, - ConstData_t hx, - ConstData_t dhy, - Data_t dhx, - const TensorDescriptor& cDesc, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - const SeqTensorDescriptor& xDesc, - Data_t dx, - ConstData_t w, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNTransformerBackwardWeights(Handle& handle, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - const SeqTensorDescriptor& yDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNVanillaForward(Handle& handle, - miopenRNNFWDMode_t fwdMode, - ConstData_t w, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - Data_t hy, - const TensorDescriptor& cDesc, - ConstData_t cx, - Data_t cy, - const SeqTensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNVanillaBackwardData(Handle& handle, - const SeqTensorDescriptor& yDesc, - ConstData_t dy, - const TensorDescriptor& hDesc, - ConstData_t hx, - ConstData_t dhy, - Data_t dhx, - const TensorDescriptor& cDesc, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - const SeqTensorDescriptor& xDesc, - Data_t dx, - ConstData_t w, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNVanillaBackwardWeights(Handle& handle, - const SeqTensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hDesc, - ConstData_t hx, - const SeqTensorDescriptor& yDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNForwardTrainingPackedTensors(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNForwardMS(Handle& handle, - std::vector& seq_array, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& yDesc, - Data_t y, - Data_t hy, - Data_t cy, - Data_t extra_space, - size_t extra_space_size, - miopenRNNFWDMode_t fwd_mode) const; - - void RNNForwardInferencePacked(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize) const; - - void RNNBackwardDataPackedTensors(Handle& handle, - int seqLen, - c_array_view dyDesc, - ConstData_t dy, - ConstData_t dhy, - ConstData_t dcy, - ConstData_t w, - ConstData_t hx, - ConstData_t cx, - c_array_view dxDesc, - Data_t dx, - const TensorDescriptor& dhxDesc, - Data_t dhx, - const TensorDescriptor& dcxDesc, - Data_t dcx, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const; - - void RNNBackwardWeightsPackedTensors(Handle& handle, - int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - c_array_view dyDesc, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const; -}; - -MIOPEN_INTERNALS_EXPORT std::ostream& operator<<(std::ostream& stream, const RNNDescriptor& r); - -} // namespace miopen -MIOPEN_DEFINE_OBJECT(miopenRNNDescriptor, miopen::RNNDescriptor); - -#endif // GUARD_MIOPEN_RNN_HPP_ diff --git a/src/include/miopen/rnn/base_ops.hpp b/src/include/miopen/rnn/base_ops.hpp index e2efaa2ec2..e69de29bb2 100644 --- a/src/include/miopen/rnn/base_ops.hpp +++ b/src/include/miopen/rnn/base_ops.hpp @@ -1,255 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include - -namespace miopen { - -namespace rnn_base { - -template -std::array Concat(const std::array& a, const std::array& b) -{ - std::array result; - std::copy(a.cbegin(), a.cend(), result.begin()); - std::copy(b.cbegin(), b.cend(), result.begin() + N); - return result; -} - -inline miopen::GemmDescriptor GemmDescriptor64BitWraper(bool isColMajor_, - bool transA_, - bool transB_, - size_t m_, - size_t n_, - size_t k_, - size_t lda_, - size_t ldb_, - size_t ldc_, - size_t batch_count_, - long long int strideA_, - long long int strideB_, - long long int strideC_, - float alpha_, - float beta_, - miopenDataType_t dataType_, - bool deterministic_) -{ - return GemmDescriptor{isColMajor_, - transA_, - transB_, - static_cast(m_), - static_cast(n_), - static_cast(k_), - static_cast(lda_), - static_cast(ldb_), - static_cast(ldc_), - static_cast(batch_count_), - strideA_, - strideB_, - strideC_, - alpha_, - beta_, - dataType_, - deterministic_}; -} - -class RnnBaseFunctions -{ - RnnBaseFunctions() = default; - -public: - static miopenStatus_t BWD_GEMM_Hidden_Prop(const Handle& handle, - ConstData_t comb_gates_src_ptr, - size_t comb_gates_src_offset, - ConstData_t filter_src_ptr, - size_t filter_offset, - Data_t ht_dst_ptr, - size_t ht_dst_offset, - size_t gemm_batch_size, - size_t ht_dest_vec_size, - size_t comb_gates_size, - size_t tmp_gate_row_stride, - size_t filter_row_stride, - size_t ht_dest_row_stride, - miopenDataType_t data_type, - bool add_assign = true) - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - // no gemm work - if(gemm_batch_size == 0) - return miopenStatusSuccess; - - const miopen::GemmDescriptor gemm_desc = - GemmDescriptor64BitWraper(false, - false, - false, - gemm_batch_size, - ht_dest_vec_size, - comb_gates_size, - tmp_gate_row_stride, - filter_row_stride, - ht_dest_row_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - add_assign ? 1 : 0, // beta - data_type, - false); - - return CallGemm(handle, - gemm_desc, - comb_gates_src_ptr, - comb_gates_src_offset, - filter_src_ptr, - filter_offset, - ht_dst_ptr, - ht_dst_offset, - GemmBackend_t::rocblas); -#else - return miopenStatusNotImplemented; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - - static miopenStatus_t BWD_GEMM_Hidden_Prop(const Handle& handle, - ConstData_t comb_gates_src_ptr, - size_t comb_gates_src_offset, - const miopen::TensorDescriptor& tmp_gates_src_dsc, - - ConstData_t filter_src_ptr, - size_t filter_src_offset, - const miopen::TensorDescriptor& filter_src_dsc, - - Data_t ht_dst_ptr, - size_t ht_dst_offset, - const miopen::TensorDescriptor& ht_dest_dsc, - bool add_assign = true) - { - assert(filter_src_dsc.GetNumDims() == 2 && tmp_gates_src_dsc.GetNumDims() == 2 && - ht_dest_dsc.GetNumDims() == 2); - - const size_t batch_size = tmp_gates_src_dsc.GetLengths()[0]; - const size_t comb_gates_size = tmp_gates_src_dsc.GetLengths()[1]; - const size_t ht_vec_size = ht_dest_dsc.GetLengths()[1]; - - assert(filter_src_dsc.GetLengths()[0] == comb_gates_size); - assert(filter_src_dsc.GetLengths()[1] == ht_vec_size); - assert(ht_dest_dsc.GetLengths()[0] == batch_size); - - const size_t tmp_gates_ld_stride = tmp_gates_src_dsc.GetStrides()[0]; // {batch, comb_gates} - const size_t filter_ld_stride = filter_src_dsc.GetStrides()[0]; // {comb_gates, ht_vec} - const size_t ht_dest_ld_stride = ht_dest_dsc.GetStrides()[0]; // {batch, ht_vec} - - return BWD_GEMM_Hidden_Prop(handle, - comb_gates_src_ptr, - comb_gates_src_offset, - filter_src_ptr, - filter_src_offset, - ht_dst_ptr, - ht_dst_offset, - batch_size, - ht_vec_size, - comb_gates_size, - tmp_gates_ld_stride, - filter_ld_stride, - ht_dest_ld_stride, - ht_dest_dsc.GetType(), - add_assign); - } - - static miopenStatus_t BWWei_GEMM(const Handle& handle, - ConstData_t comb_gates_ptr, - size_t comb_gates_offset, - const miopen::TensorDescriptor& tmp_gates_dsc, - ConstData_t ht_ptr, - size_t ht_offset, - const miopen::TensorDescriptor& ht_dsc, - Data_t filter_ptr, - size_t filter_offset, - const miopen::TensorDescriptor& filter_dsc, - bool add_assign = true) - { - - assert(filter_dsc.GetNumDims() == 2 && tmp_gates_dsc.GetNumDims() == 2 && - ht_dsc.GetNumDims() == 2); - - const size_t batch_size = tmp_gates_dsc.GetLengths()[0]; - const size_t comb_gates_size = tmp_gates_dsc.GetLengths()[1]; - const size_t ht_vec_size = ht_dsc.GetLengths()[1]; - - assert(filter_dsc.GetLengths()[0] == comb_gates_size); - assert(filter_dsc.GetLengths()[1] == ht_vec_size); - assert(ht_dsc.GetLengths()[0] == batch_size); - - const size_t tmp_gates_ld_stride = tmp_gates_dsc.GetStrides()[0]; // {batch, comb_gates} - const size_t ht_dest_ld_stride = ht_dsc.GetStrides()[0]; // {batch, ht_vec} - const size_t filter_ld_stride = filter_dsc.GetStrides()[0]; // {comb_gates, ht_vec} - - // no gemm work - if(batch_size == 0) - return miopenStatusSuccess; - - [[maybe_unused]] const miopen::GemmDescriptor gemm_desc = - GemmDescriptor64BitWraper(false, - true, - false, - comb_gates_size, - ht_vec_size, - batch_size, - tmp_gates_ld_stride, - ht_dest_ld_stride, - filter_ld_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - add_assign ? 1 : 0, // beta - ht_dsc.GetType(), - false); -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return CallGemm(handle, - gemm_desc, - comb_gates_ptr, - comb_gates_offset, - ht_ptr, - ht_offset, - filter_ptr, - filter_offset, - GemmBackend_t::rocblas); -#else - - return miopenStatusNotImplemented; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } -}; - -} // namespace rnn_base -} // namespace miopen diff --git a/src/include/miopen/rnn/multi_stream_utils.hpp b/src/include/miopen/rnn/multi_stream_utils.hpp index 5866c10b34..e69de29bb2 100644 --- a/src/include/miopen/rnn/multi_stream_utils.hpp +++ b/src/include/miopen/rnn/multi_stream_utils.hpp @@ -1,135 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_MS_WA_FIX) - -namespace miopen { - -class MultiStreamController -{ -public: - static constexpr int rootStreamId = 0; - - MultiStreamController(const Handle& handle, int extra_stream_cnt) - : streamPoolIdsMapping{init_stream_pool_ids(handle, extra_stream_cnt)}, - streamPoolCache{init_stream_pool(handle, streamPoolIdsMapping)}, - activeHandle{handle} - { - } - - hipStream_t getStream(int stream_id) const { return streamPoolCache[stream_id]; } - - void ChangeActiveStream(int stream_id) const - { - activeHandle.SetStreamFromPool(streamPoolIdsMapping[stream_id]); - } - - hipError_t RecordEvent(const hipEvent_t event, int stream_id) const - { - return hipEventRecord(event, getStream(stream_id)); - } - - hipError_t SetWaitEvent(const hipEvent_t event, int stream_id) const - { - return hipStreamWaitEvent(getStream(stream_id), event, 0); - } - - void RootWaitToAllStreams() const - { - for(size_t i = 0, stream_cnt = size(); i < stream_cnt; i++) - { - if(i != rootStreamId) - { - const miopen::HipEventPtr sync_event = make_hip_fast_event(); - RecordEvent(sync_event.get(), i); - SetWaitEvent(sync_event.get(), rootStreamId); - } - } - } - - void AllStreamsWaitRoot() const - { - const miopen::HipEventPtr main_event = make_hip_fast_event(); - - RecordEvent(main_event.get(), rootStreamId); - - for(size_t i = 0, stream_cnt = size(); i < stream_cnt; i++) - { - if(i != rootStreamId) - SetWaitEvent(main_event.get(), i); - } - }; - - size_t size() const { return streamPoolIdsMapping.size(); } - - ~MultiStreamController() { ChangeActiveStream(rootStreamId); } - -private: - static std::vector init_stream_pool_ids(const Handle& handle, int extra_stream_cnt) - { - std::vector ids; - ids.reserve(extra_stream_cnt + 1); - - bool ms_wa_fix_active = extra_stream_cnt > 2 && !env::disabled(MIOPEN_MS_WA_FIX); - handle.SetStreamFromPool(0); - int wa_steams = ms_wa_fix_active ? (handle.GetStream() == nullptr ? 3 : 2) : 0; - - handle.ReserveExtraStreamsInPool(extra_stream_cnt + wa_steams); - - for(int i = 0; i <= extra_stream_cnt + wa_steams; i++) - if(!(ms_wa_fix_active && (i > 0 && i <= wa_steams))) - ids.push_back(i); - - return ids; - } - - static std::vector init_stream_pool(const Handle& handle, - const std::vector& pool_ids) - { - std::vector pool; - pool.reserve(pool_ids.size()); - - for(auto id : pool_ids) - { - handle.SetStreamFromPool(id); - pool.push_back(handle.GetStream()); - } - handle.SetStreamFromPool(0); - - return pool; - } - - const std::vector streamPoolIdsMapping; - const std::vector streamPoolCache; - const Handle& activeHandle; -}; - -} // namespace miopen diff --git a/src/include/miopen/rnn/solvers.hpp b/src/include/miopen/rnn/solvers.hpp index b75350db69..e69de29bb2 100644 --- a/src/include/miopen/rnn/solvers.hpp +++ b/src/include/miopen/rnn/solvers.hpp @@ -1,850 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include "miopen/rnn/tmp_buffer_utils.hpp" - -namespace miopen { - -namespace rnn_base { - -class RNNBackwardDataModularAlgo -{ -public: - static RNNBackwardDataModularAlgo create(const RNNDescriptor& rnnDesc, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - { - auto [max_layers_hid, max_batch_hid, hidden_vec_sz] = miopen::tien<3>(hDesc.GetLengths()); - auto [max_batch_in, max_seq, input_vec_sz] = miopen::tien<3>(xDesc.GetLengths()); - - assert(max_batch_in <= max_batch_hid); - - auto layers_cnt = static_cast(rnnDesc.nLayers); - const bool is_seq_bidir = rnnDesc.dirMode == miopenRNNbidirection; - - assert(static_cast(layers_cnt) * (is_seq_bidir ? 2 : 1) <= max_layers_hid); - - auto gates_cnt = static_cast(rnnDesc.nHiddenTensorsPerLayer); - - // class update req - assert(!is_seq_bidir); - const size_t seq_directions = is_seq_bidir ? 2 : 1; - - // TODO all size_t - GeneralLstmRedBuffer rb_layout = GeneralLstmRedBuffer::build( - layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); - - GeneralLstmTempBuffer workspace_info = GeneralLstmTempBuffer::build( - layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); - - WeightsBufferDescriptor weights_layout = - WeightsBufferDescriptor::create(static_cast(input_vec_sz), - static_cast(hidden_vec_sz), - layers_cnt, - rnnDesc.biasMode, - rnnDesc.inputMode, - gates_cnt, - is_seq_bidir); - - BatchController batch_controller = BatchController::Create(xDesc); - - HiddenBuffersDescriptor hidden_hxcx_info{hDesc}; - - IOBufferDescriptor x_info{IOBufferDescriptor::build(xDesc)}; - IOBufferDescriptor y_info{IOBufferDescriptor::build(yDesc)}; - - return {std::move(rb_layout), - workspace_info, - weights_layout, - hidden_hxcx_info, - x_info, - y_info, - rnnDesc, - batch_controller}; - } - - void PrepareWriteBuffers(const Handle& handle, Data_t dhx, Data_t dcx, Data_t workSpace) const; - - void PropDhy(const Handle& handle, - ConstData_t dhy, - Data_t workSpace, - unsigned int layer, - const SequenceIterator& currentSeq, - SequenceDirection direction) const; - - void PropHiddenDht(const Handle& handle, - ConstData_t w, - Data_t workSpace, - int layer, - const SequenceIterator& currentSeq, - SequenceDirection direction) const; - - void UpdateHStatePerTimeSeq(const Handle& handle, - ConstData_t dcy, - ConstData_t cx, - Data_t, - Data_t workSpace, - Data_t reserveSpace, - int layer, - const SequenceIterator& seq, - SequenceDirection direction) const; - - void PropDhxDcx(const Handle& handle, - ConstData_t w, - Data_t dhx, - Data_t dcx, - Data_t workSpace, - Data_t reserveSpace, - size_t layer, - const SequenceIterator& currentSeq, - SequenceDirection direction) const; - - void PropDy(const Handle& handle, ConstData_t dy, Data_t workSpace) const; - - void PropHiddenDy(const Handle& handle, - ConstData_t w, - Data_t workSpace, - Data_t reserveSpace, - size_t layer, - SequenceDirection direction) const; - - void PropHiddenDy(const Handle& handle, - ConstData_t w, - Data_t workSpace, - Data_t reserveSpace, - size_t layer, - SequenceDirection direction, - const SequenceIterator& firstSeq, - const SequenceIterator& lastSeq) const; - - void PropHiddenDy(const Handle& handle, - ConstData_t w, - Data_t workSpace, - Data_t reserveSpace, - size_t layer, - SequenceDirection direction, - size_t gemm_batch_size, - size_t gemm_batch_offset) const; - - void PropDx(const Handle& handle, - ConstData_t w, - ConstData_t workSpace, - Data_t dx, - SequenceDirection direction, - const SequenceIterator& firstSeq, - const SequenceIterator& lastSeq) const; - - void PropDx(const Handle& handle, - ConstData_t w, - ConstData_t workSpace, - Data_t dx, - SequenceDirection direction) const; - - void PropDx(const Handle& handle, - ConstData_t w, - ConstData_t workSpace, - Data_t dx, - SequenceDirection direction, - size_t gemm_batch_offset, - size_t gemm_batch_size) const; - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - -private: - RNNBackwardDataModularAlgo(GeneralLstmRedBuffer rb_layout, - GeneralLstmTempBuffer workspace_info, - WeightsBufferDescriptor weights_layout, - HiddenBuffersDescriptor hidden_hxcx_info, - IOBufferDescriptor x_info, - IOBufferDescriptor y_info, - const RNNDescriptor& rnn_desc, - BatchController batch_controller) - : reservLayout(std::move(rb_layout)), - workspaceInfo(std::move(workspace_info)), - weightsLayout(std::move(weights_layout)), - hiddenHxCxInfo(std::move(hidden_hxcx_info)), - xInfo(std::move(x_info)), - yInfo(std::move(y_info)), - rnnDesc(rnn_desc), - batchController(std::move(batch_controller)) - { - } - - template - inline miopen::TensorDescriptor BuildLstmTmpBlockDesc2D(const BufType& buf_info, - const size_t batch_size) const - { - const std::array& tmp_block_stride = buf_info.getGateBlockStride(); - const std::array& tmp_block_size = buf_info.getGateBlockSize(); - - // batch, gateBlock_elements - return miopen::TensorDescriptor{rnnDesc.dataType, - {batch_size, tmp_block_size[3]}, - {tmp_block_stride[1], tmp_block_stride[3]}}; - } - - inline miopen::TensorDescriptor BuildLstmFilterXDesc2D(int layer_id) const - { - assert(rnnDesc.inputMode == 0 || layer_id != 0); - // TODO replace by stride - auto x_vec = layer_id != 0 ? weightsLayout.xInVec : weightsLayout.inVec; - - // gateBlock_elements, ht_vec - return miopen::TensorDescriptor{ - rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, x_vec}, {x_vec, 1}}; - } - - inline miopen::TensorDescriptor BuildLstmFilterHidDesc2D() const - { - // TODO replace by stride - auto h_vec = weightsLayout.hVec; - - // gateBlock_elements, ht_vec - return miopen::TensorDescriptor{ - rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, h_vec}, {h_vec, 1}}; - } - - inline miopen::TensorDescriptor BuildWsHtDesc2D(size_t batch_size) const - { - auto& ht_stride = workspaceInfo.getHiddenStateStride(); - auto& ht_size = workspaceInfo.hStateSizes; - - // batch, gateBlock_elements - return miopen::TensorDescriptor{ - rnnDesc.dataType, {batch_size, ht_size[3]}, {ht_stride[1], ht_stride[3]}}; - } - - // 2 dims batch, vec - inline miopen::TensorDescriptor BuildHxCxDesc2D(size_t batch_size) const - { - const std::vector hx_size{batch_size, hiddenHxCxInfo.getHiddenSize()}; - const std::vector hx_stride{hiddenHxCxInfo.getStrides()[1], - hiddenHxCxInfo.getStrides()[2]}; - - return miopen::TensorDescriptor{rnnDesc.dataType, hx_size, hx_stride}; - } - - // 3 dims layer, batch, vec - inline miopen::TensorDescriptor BuildHxCxDesc3D(size_t layer_size, size_t batch_size) const - { - const std::vector hx_accum_size{ - layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; - - return miopen::TensorDescriptor{ - rnnDesc.dataType, hx_accum_size, hiddenHxCxInfo.getStrides()}; - } - - // 3 dims layer, batch, vec - inline miopen::TensorDescriptor BuildTempDhtDesc3D(size_t layer_size, size_t batch_size) const - { - const std::vector dy_dhy_accum_size{ - layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; - - const auto ws_dy_stride = [](const auto& ws_4dim_strides) -> std::vector { - // convert 4dim stride to 3 dim without direction - // TODO change hiddenBufferDesc - return std::vector{ws_4dim_strides[0], ws_4dim_strides[1], ws_4dim_strides[3]}; - }(workspaceInfo.getHiddenStateStride()); - - return miopen::TensorDescriptor{rnnDesc.dataType, dy_dhy_accum_size, ws_dy_stride}; - } - - inline size_t getVirtualLayer(const size_t layer_id, SequenceDirection direction) const - { - return layer_id * (isBidirectSeq ? 2 : 1) + - (direction == SequenceDirection::Forward ? 0 : 1); - } - - const GeneralLstmRedBuffer reservLayout; - // const WorkspaceBufferDescriptor workspaceInfo; - const GeneralLstmTempBuffer workspaceInfo; - - const WeightsBufferDescriptor weightsLayout; - const HiddenBuffersDescriptor hiddenHxCxInfo; - const IOBufferDescriptor xInfo; - const IOBufferDescriptor yInfo; - - const RNNDescriptor& rnnDesc; - - const ActivationDescriptor tanhDesc = {miopenActivationTANH, 1, 1, 1}; - const ActivationDescriptor sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; - const ActivationDescriptor reluDesc = {miopenActivationRELU, 1, 0, 1}; - - const BatchController batchController; - - const bool isBidirectSeq = false; -}; - -class RNNModularSingleStreamBWD -{ -public: - RNNModularSingleStreamBWD(const RNNDescriptor& rnn, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - : rnnAlgoModules(RNNBackwardDataModularAlgo::create(rnn, xDesc, yDesc, hDesc)), - rnnDesc(rnn), - max_seq_len(xDesc.GetMaxSequenceLength()) - { - } - - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - - // TODO - static size_t GetWsSize() { return 0; }; - - void ComputeBWD(Handle& handle, - ConstData_t dy, - ConstData_t dhy, - Data_t dhx, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - Data_t dx, - ConstData_t w, - Data_t workSpace, - Data_t reserveSpace) const; - - const rnn_base::RNNBackwardDataModularAlgo rnnAlgoModules; - const RNNDescriptor& rnnDesc; - const size_t max_seq_len; -}; - -class RNNModularMultiStreamBWD -{ -public: - RNNModularMultiStreamBWD(const RNNDescriptor& rnn, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - : rnnAlgoModules(RNNBackwardDataModularAlgo::create(rnn, xDesc, yDesc, hDesc)), - rnnDesc(rnn), - max_seq_len(xDesc.GetMaxSequenceLength()) - { - } - - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - - // TODO - static size_t GetWsSize() { return 0; }; - - struct runtimeArgsBwd - { - const Handle* handle; - ConstData_t dy; - ConstData_t dhy; - Data_t dhx; - ConstData_t cx; - ConstData_t dcy; - Data_t dcx; - Data_t dx; - ConstData_t w; - Data_t workSpace; - Data_t reserveSpace; - }; - - void ComputeBWD(Handle& handle, - ConstData_t dy, - ConstData_t dhy, - Data_t dhx, - ConstData_t cx, - ConstData_t dcy, - Data_t dcx, - Data_t dx, - ConstData_t w, - Data_t workSpace, - Data_t reserveSpace) const; - - bool ChunkDispatch(const runtimeArgsBwd& args, - size_t chunk_size, - size_t chunk_time_offset, - size_t chunk_layer_offset) const; - -private: - void PrologueDispatch(const runtimeArgsBwd& args) const; - - const rnn_base::RNNBackwardDataModularAlgo rnnAlgoModules; - const RNNDescriptor& rnnDesc; - const size_t max_seq_len; -}; - -class RNNBackwardWeightsModularAlgo -{ -public: - static RNNBackwardWeightsModularAlgo create(const RNNDescriptor& rnnDesc, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - { - auto [max_layers_hid, max_batch_hid, hidden_vec_sz] = miopen::tien<3>(hDesc.GetLengths()); - auto [max_batch_in, max_seq, input_vec_sz] = miopen::tien<3>(xDesc.GetLengths()); - - assert(max_batch_in <= max_batch_hid); - - size_t layers_cnt = rnnDesc.nLayers; - const bool is_seq_bidir = rnnDesc.dirMode == miopenRNNbidirection; - - assert(layers_cnt * (is_seq_bidir ? 2 : 1) <= max_layers_hid); - - auto gates_cnt = static_cast(rnnDesc.nHiddenTensorsPerLayer); - - // class update req - assert(!is_seq_bidir); - const size_t seq_directions = is_seq_bidir ? 2 : 1; - - GeneralLstmRedBuffer rb_layout = GeneralLstmRedBuffer::build( - layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); - - GeneralLstmTempBuffer workspace_info = GeneralLstmTempBuffer::build( - layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); - - WeightsBufferDescriptor weights_layout = WeightsBufferDescriptor::create(input_vec_sz, - hidden_vec_sz, - layers_cnt, - rnnDesc.biasMode, - rnnDesc.inputMode, - gates_cnt, - is_seq_bidir); - - BatchController batch_controller = BatchController::Create(xDesc); - - HiddenBuffersDescriptor hidden_hxcx_info{hDesc}; - - IOBufferDescriptor x_info{IOBufferDescriptor::build(xDesc)}; - IOBufferDescriptor y_info{IOBufferDescriptor::build(yDesc)}; - - return {std::move(rb_layout), - workspace_info, - weights_layout, - hidden_hxcx_info, - x_info, - y_info, - rnnDesc, - batch_controller}; - } - - void PrepareWriteBuffers(const Handle& handle, Data_t w) const; - - void PhisXInputWeights(const Handle& handle, Data_t dw, Data_t workSpace, ConstData_t x) const; - - void HiddenXInputWeights(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t reserveSpace, - size_t layer) const; - - void BiasUpdate(const Handle& handle, - Data_t dw, - Data_t workSpace, - size_t layer, - size_t workSpaceSize) const; - - void HiddenHStateWeights(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t reserveSpace, - const SequenceIterator& seq, - size_t layer, - SequenceDirection direction) const - { - const size_t gemm_batch_size = [&]() -> size_t { - if(seq.isFirst()) - return 0; - - if(direction == SequenceDirection::Reverse) - return batchController.getBatchSize(seq.getPhisVal()); - else - return batchController.getBatchSize(seq.getPrev().getPhisVal()); - }(); - - if(gemm_batch_size != 0) - return HiddenHStateWeights_Unchecked( - handle, dw, workSpace, reserveSpace, seq, layer, direction, gemm_batch_size); - } - - void HiddenHStateWeights(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t reserveSpace, - size_t layer, - size_t max_seq_len, - const SequenceDirection direction) const - { - size_t start_seq_id = 0; - const size_t last_seq = max_seq_len - 1; - for(auto i = start_seq_id + 1; i <= last_seq; i++) - { - - if(batchController.getBatchSize(i) != batchController.getBatchSize(start_seq_id) || - i == last_seq) - { - const size_t gemm_batch_size = (batchController.getBatchSum(i - 1) - - batchController.getBatchSum(start_seq_id)) + - batchController.getBatchSize(i); - - if(gemm_batch_size != 0) - { - const auto first_logical_val = direction == SequenceDirection::Forward - ? start_seq_id - : (max_seq_len - 1) - start_seq_id - 1; - const auto seq = - SequenceIterator(first_logical_val, direction, max_seq_len, false); - - HiddenHStateWeights_Unchecked(handle, - dw, - workSpace, - reserveSpace, - seq, - layer, - direction, - gemm_batch_size); - } - start_seq_id = i; - } - } - } - - void PhisHStateWeights(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t hx, - size_t layer, - size_t max_seq_len, - SequenceDirection direction) const - { - if(hx == nullptr) - return; - - for(auto i = max_seq_len; i > 0; i--) - { - const auto seq = SequenceIterator(i - 1, direction, max_seq_len, false); - - PhisHStateWeights(handle, dw, workSpace, hx, seq, layer, direction); - } - } - - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - -private: - RNNBackwardWeightsModularAlgo(GeneralLstmRedBuffer rb_layout, - GeneralLstmTempBuffer workspace_info, - WeightsBufferDescriptor weights_layout, - HiddenBuffersDescriptor hidden_hxcx_info, - IOBufferDescriptor x_info, - IOBufferDescriptor y_info, - const RNNDescriptor& rnn_desc, - BatchController batch_controller) - : reservLayout(std::move(rb_layout)), - workspaceInfo(std::move(workspace_info)), - weightsLayout(std::move(weights_layout)), - hiddenHxCxInfo(std::move(hidden_hxcx_info)), - xInfo(std::move(x_info)), - yInfo(std::move(y_info)), - rnnDesc(rnn_desc), - batchController(std::move(batch_controller)) - { - } - - void HiddenHStateWeights_Unchecked(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t reserveSpace, - const SequenceIterator& seq, - size_t layer, - SequenceDirection direction, - size_t gemm_batch_size) const; - - void PhisHStateWeights(const Handle& handle, - Data_t dw, - ConstData_t workSpace, - ConstData_t hx, - const SequenceIterator& seq, - size_t layer, - SequenceDirection direction) const; - - template - inline miopen::TensorDescriptor BuildLstmTmpBlockDesc2D(const BufType& buf_info, - const size_t batch_size) const - { - const std::array& tmp_block_stride = buf_info.getGateBlockStride(); - const std::array& tmp_block_size = buf_info.getGateBlockSize(); - - // batch, gateBlock_elements - return miopen::TensorDescriptor{rnnDesc.dataType, - {batch_size, tmp_block_size[3]}, - {tmp_block_stride[1], tmp_block_stride[3]}}; - } - - inline miopen::TensorDescriptor BuildLstmFilterXDesc2D(int layer_id) const - { - assert(rnnDesc.inputMode == 0 || layer_id != 0); - // TODO replace by stride - auto x_vec = layer_id != 0 ? weightsLayout.xInVec : weightsLayout.inVec; - - // gateBlock_elements, ht_vec - return miopen::TensorDescriptor{ - rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, x_vec}, {x_vec, 1}}; - } - - inline miopen::TensorDescriptor BuildLstmFilterHidDesc2D() const - { - // TODO replace by stride - auto h_vec = weightsLayout.hVec; - - // gateBlock_elements, ht_vec - return miopen::TensorDescriptor{ - rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, h_vec}, {h_vec, 1}}; - } - - template - inline miopen::TensorDescriptor BuildTmpHtDesc2D(const BufType& buf_info, - size_t batch_size) const - { - auto& ht_stride = buf_info.getHiddenStateStride(); - auto& ht_size = buf_info.hStateSizes; - - // batch, gateBlock_elements - return miopen::TensorDescriptor{ - rnnDesc.dataType, {batch_size, ht_size[3]}, {ht_stride[1], ht_stride[3]}}; - } - - // 2 dims batch, vec - inline miopen::TensorDescriptor BuildHxCxDesc2D(size_t batch_size) const - { - const std::vector hx_size{batch_size, hiddenHxCxInfo.getHiddenSize()}; - const std::vector hx_stride{hiddenHxCxInfo.getStrides()[1], - hiddenHxCxInfo.getStrides()[2]}; - - return miopen::TensorDescriptor{rnnDesc.dataType, hx_size, hx_stride}; - } - - // 3 dims layer, batch, vec - inline miopen::TensorDescriptor BuildHxCxDesc3D(size_t layer_size, size_t batch_size) const - { - const std::vector hx_accum_size{ - layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; - - return miopen::TensorDescriptor{ - rnnDesc.dataType, hx_accum_size, hiddenHxCxInfo.getStrides()}; - } - - // 3 dims layer, batch, vec - inline miopen::TensorDescriptor BuildTempDhtDesc3D(size_t layer_size, size_t batch_size) const - { - const std::vector dy_dhy_accum_size{ - layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; - - const auto ws_dy_stride = [](const auto& ws_4dim_strides) -> std::vector { - // convert 4dim stride to 3 dim without direction - // TODO change hiddenBufferDesc - return std::vector{ws_4dim_strides[0], ws_4dim_strides[1], ws_4dim_strides[3]}; - }(workspaceInfo.getHiddenStateStride()); - - return miopen::TensorDescriptor{rnnDesc.dataType, dy_dhy_accum_size, ws_dy_stride}; - } - - // 3 dims layer, batch, vec - inline miopen::TensorDescriptor BuildWeiBiasDesc2D() const - { - const std::vector bias_size = [](const auto& wei_4dim_size) -> std::vector { - // wei_4dim_size{layer, dir, gate, vec} - return {1, wei_4dim_size[1] * wei_4dim_size[2] * wei_4dim_size[3]}; - }(weightsLayout.getBiasSize()); - - const auto bias_stride = [](const auto& wei_4dim_strides) -> std::vector { - // convert 4dim stride to 2 dim without direction - return std::vector{wei_4dim_strides[0], wei_4dim_strides[3]}; - }(weightsLayout.getBiasStride()); - - return miopen::TensorDescriptor{rnnDesc.dataType, bias_size, bias_stride}; - } - - inline size_t getVirtualLayer(const size_t layer_id, SequenceDirection direction) const - { - return layer_id * (isBidirectSeq ? 2 : 1) + - (direction == SequenceDirection::Forward ? 0 : 1); - } - - inline size_t getHxBatchSizeReadAtTime(const SequenceIterator& seq, - SequenceDirection direction) const - { - if(seq.isLast()) - return batchController.getBatchSize(seq.getPhisVal()); - - if(direction == SequenceDirection::Reverse) - { - return batchController.getBatchSize(seq.getPhisVal()) - - batchController.getBatchSize(seq.getPrev().getPhisVal()); - } - return 0; - } - - const GeneralLstmRedBuffer reservLayout; - // const WorkspaceBufferDescriptor workspaceInfo; - const GeneralLstmTempBuffer workspaceInfo; - - const WeightsBufferDescriptor weightsLayout; - const HiddenBuffersDescriptor hiddenHxCxInfo; - - const IOBufferDescriptor xInfo; - const IOBufferDescriptor yInfo; - - const RNNDescriptor& rnnDesc; - - const ActivationDescriptor tanhDesc = {miopenActivationTANH, 1, 1, 1}; - const ActivationDescriptor sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; - const ActivationDescriptor reluDesc = {miopenActivationRELU, 1, 0, 1}; - - const BatchController batchController; - - const bool isBidirectSeq = false; -}; - -class RNNModularSingleStreamBWWeights -{ -public: - RNNModularSingleStreamBWWeights(const RNNDescriptor& rnn, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - : rnnAlgoModules(RNNBackwardWeightsModularAlgo::create(rnn, xDesc, yDesc, hDesc)), - rnnDesc(rnn), - max_seq_len(xDesc.GetMaxSequenceLength()) - { - } - - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - - // TODO - static size_t GetWsSize() { return 0; }; - - void Compute(const Handle& handle, - ConstData_t x, - ConstData_t hx, - Data_t dw, - Data_t workSpace, - size_t /*workSpaceSize*/, - ConstData_t reserveSpace, - size_t /*reserveSpaceSize*/) const; - - const rnn_base::RNNBackwardWeightsModularAlgo rnnAlgoModules; - const RNNDescriptor& rnnDesc; - const size_t max_seq_len; -}; - -class RNNModularMultiStreamBWWeights -{ -public: - RNNModularMultiStreamBWWeights(const RNNDescriptor& rnn, - const SeqTensorDescriptor& xDesc, - const SeqTensorDescriptor& yDesc, - const TensorDescriptor& hDesc) - : rnnAlgoModules(RNNBackwardWeightsModularAlgo::create(rnn, xDesc, yDesc, hDesc)), - rnnDesc(rnn), - max_seq_len(xDesc.GetMaxSequenceLength()) - { - } - - static bool IsApplicable() - { -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - return true; -#else - return false; -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - } - - // TODO - static size_t GetWsSize() { return 0; }; - - struct runtimeArgsBww - { - const Handle* handle; - ConstData_t x; - ConstData_t hx; - Data_t dw; - Data_t workSpace; - ConstData_t reserveSpace; - }; - - void Compute(const Handle& handle, - ConstData_t x, - ConstData_t hx, - Data_t dw, - Data_t workSpace, - size_t /*workSpaceSize*/, - ConstData_t reserveSpace, - size_t /*reserveSpaceSize*/) const; - -private: - void PrologueDispatch(const runtimeArgsBww& args) const; - - const rnn_base::RNNBackwardWeightsModularAlgo rnnAlgoModules; - const RNNDescriptor& rnnDesc; - const size_t max_seq_len; -}; - -} // namespace rnn_base -} // namespace miopen diff --git a/src/include/miopen/rnn/tmp_buffer_utils.hpp b/src/include/miopen/rnn/tmp_buffer_utils.hpp index 39806aa73d..e69de29bb2 100644 --- a/src/include/miopen/rnn/tmp_buffer_utils.hpp +++ b/src/include/miopen/rnn/tmp_buffer_utils.hpp @@ -1,930 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include -#include - -namespace miopen { - -// https://github.com/ROCm/MIOpen/issues/3048 -namespace WA_RHEL { - -template -OutputIt exclusive_scan_wa(InputIt first, InputIt last, OutputIt d_first, T init, BinaryOp op) -{ - if(first == last) - return d_first; - - auto acc = init; - - for(; first != last; ++first) - { - *d_first++ = acc; - acc = op(std::move(acc), *first); // std::move since C++11 - } - - return d_first; -} - -} // namespace WA_RHEL - -namespace rnn_base { - -enum class SequenceDirection -{ - Forward = 0, - Reverse = 1 -}; - -enum class LstmGateAndState -{ - I = 0, - F = 1, - O = 2, - G = 3, - St = 4, - Ht = 5 -}; - -/** - * \section Extensions for layouts. - */ - -// GRU -// TODO - -// LSTM -template -class GeneralLstmWsExt -{ -public: - size_t getGasOffset(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction, - LstmGateAndState gas) const - { - return static_cast(this)->getGasOffsetImpl( - layer_id, vector_id, direction, gas); - } - - const std::array& getGateAndStateStride(LstmGateAndState gas) const - { - return static_cast(this)->getGateAndStateStrideImpl(gas); - } -}; - -template -class LstmWsGateBlockExt -{ -public: - size_t getGateBlockOffset(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - return static_cast(this)->getGasOffset( - layer_id, vector_id, direction, LstmGateAndState::I); - } - - const std::array& getGateBlockStride() const - { - return static_cast(this)->getGateAndStateStride(LstmGateAndState::I); - } - - const std::array& getGateBlockSize() const - { - return static_cast(this)->getGateBlockSizeImpl(); - } -}; - -template -class LstmActiveCellExt -{ -public: - size_t getActiveCellOffset(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - return static_cast(this)->getActiveCellOffsetImpl( - layer_id, vector_id, direction); - } - - const std::array& getActiveCellStride() const - { - return static_cast(this)->getActiveCellStrideImpl(); - } -}; - -// pure RNN -template -class BaseRnnWsBufferPacked -{ -public: - BaseRnnWsBufferPacked() = default; - size_t getHiddenStateOffset(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - return static_cast(this)->getHiddenStateOffsetImpl( - layer_id, vector_id, direction); - } - - // layer, minor dim(seq or sample), directions, element - const std::array& getHiddenStateStride() const - { - return static_cast(this)->getHiddenStateStrideImpl(); - } - - size_t getBufferSize() const { return static_cast(this)->getBufferSizeImpl(); } -}; - -/** - * \section Standard layouts for temporary buffers - */ -////// - -class GeneralRNNTempBuffer : public BaseRnnWsBufferPacked -{ -protected: - GeneralRNNTempBuffer(const std::array& hstate_strides, - const std::array& hstate_sizes, - size_t total_element_cnt) - : hStateStrides{hstate_strides}, - hStateSizes{hstate_sizes}, - totalElementCnt{total_element_cnt} - { - } - -public: - static GeneralRNNTempBuffer - build(size_t layers_cnt, size_t vectors_per_layer, size_t directions, size_t hidden_vec_sz) - { - const std::array h_state_sizes{ - layers_cnt, vectors_per_layer, directions, hidden_vec_sz}; - std::array h_state_strides = {0, 0, 0, 1}; - std::partial_sum(h_state_sizes.crbegin(), - std::prev(h_state_sizes.crend()), - std::next(h_state_strides.rbegin()), - std::multiplies{}); - - auto total_element_cnt = h_state_strides[0] * h_state_sizes[0]; - return GeneralRNNTempBuffer{h_state_strides, h_state_sizes, total_element_cnt}; - } - - size_t getHiddenStateOffsetImpl(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - const std::array pos{layer_id, vector_id, static_cast(direction)}; - - return std::inner_product( - pos.cbegin(), pos.cend(), hStateStrides.cbegin(), static_cast(0)); - } - - size_t getBufferSizeImpl() const { return totalElementCnt; } - - const std::array& getHiddenStateStrideImpl() const { return hStateStrides; } - - // layer, comb dim(seq, sample), direction, vector element - const std::array hStateStrides; - // layers, comb dims(seq, sample), directions, elements - const std::array hStateSizes; - const size_t totalElementCnt; -}; - -/* - *struct Workspace_LSTM_BWD{ //packed - * struct layer{ - * struct all_sequences_all_samples{ - * struct mat_mul_block{ - * struct gate_vector{ - * float[hidden_size]; HidUpdate-> write - * } I, F, O, G; - * } gemm_block[directions]; - * - * struct dcx_vec{ - * float[hidden_size]; HidUpdate-> write - * } dy[directions]; - * - * struct dy_vec{ - * float[hidden_size]; HidUpdate <-read - * } dy[directions]; - * - * } combi_dim[seq_len * batch_size=totalBatch]; [seq][batch] - * } layers[nLayer]; - *} - */ - -class GeneralLstmTempBuffer : public GeneralRNNTempBuffer, - public GeneralLstmWsExt, - public LstmWsGateBlockExt -{ -protected: - GeneralLstmTempBuffer(const std::array& h_state_strides, - const std::array& h_state_sizes, - const std::array& lstm_gate_sizes, - const std::array& lstm_gate_strides, - const std::array& lstm_gates_block_sizes, - size_t total_element_cnt) - : GeneralRNNTempBuffer{h_state_strides, h_state_sizes, total_element_cnt}, - gateSizes{lstm_gate_sizes}, - gateStride{lstm_gate_strides}, - gateBlockSizes{lstm_gates_block_sizes} - { - } - -public: - static GeneralLstmTempBuffer - build(size_t layers_cnt, size_t comp_dim_per_layer, size_t directions, size_t hidden_vec_sz) - { - - constexpr size_t MS_len = 2; - constexpr size_t LS_size = 2; - // Most significant sizes - //{layer, comp_dim_per_layer} - std::array block_MS_size = {layers_cnt, comp_dim_per_layer}; - - //{layer, comp_dim_per_layer, directions,vec} - const auto h_state_sizes = - Concat(block_MS_size, std::array{directions, hidden_vec_sz}); - - const auto lstm_gate_sizes = - Concat(block_MS_size, std::array{directions, hidden_vec_sz}); - - const auto lstm_gates_block_sizes = - Concat(block_MS_size, std::array{directions, 4 * hidden_vec_sz}); - - // Less significant stride - // LSStride{directions,vec} - std::array h_state_LS_strides{}; - std::array lstm_gate_LS_strides{}; - - WA_RHEL::exclusive_scan_wa(h_state_sizes.crbegin(), - std::next(h_state_sizes.crbegin(), h_state_LS_strides.size()), - h_state_LS_strides.rbegin(), - 1LL, - std::multiplies{}); - - WA_RHEL::exclusive_scan_wa( - lstm_gates_block_sizes.crbegin(), - std::next(lstm_gates_block_sizes.crbegin(), lstm_gate_LS_strides.size()), - lstm_gate_LS_strides.rbegin(), - 1LL, - std::multiplies{}); - - const auto gas_block_stride = [&lstm_gates_block_sizes, - &h_state_sizes, - &lstm_gate_LS_strides, - &h_state_LS_strides](const auto last_dim) { - auto gate_size_in_block = lstm_gates_block_sizes[last_dim] * lstm_gate_LS_strides[0]; - auto h_state_size_in_block = h_state_sizes[last_dim] * h_state_LS_strides[0]; - auto c_state_size_in_block = h_state_size_in_block; - return gate_size_in_block + c_state_size_in_block + h_state_size_in_block; - }(MS_len); - - // Most significant stride - // MSStride{layer, comp_dim_per_layer} - std::array block_MS_strides{}; - - WA_RHEL::exclusive_scan_wa(block_MS_size.crbegin(), - std::next(block_MS_size.crbegin(), block_MS_strides.size()), - block_MS_strides.rbegin(), - gas_block_stride, - std::multiplies{}); - - const std::array h_state_strides = Concat(block_MS_strides, h_state_LS_strides); - const std::array lstm_gate_strides = - Concat(block_MS_strides, lstm_gate_LS_strides); - - auto total_element_cnt = h_state_strides[0] * h_state_sizes[0]; - - return {h_state_strides, - h_state_sizes, - lstm_gate_sizes, - lstm_gate_strides, - lstm_gates_block_sizes, - total_element_cnt}; - } - - size_t getHiddenStateOffset(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - return getGasOffset(layer_id, vector_id, direction, LstmGateAndState::Ht); - } - - size_t getGasOffsetImpl(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction, - LstmGateAndState gas) const - { - auto start_ident = getGasIndent(gas); - - if(gas == LstmGateAndState::Ht || gas == LstmGateAndState::St) - return start_ident + - GeneralRNNTempBuffer::getHiddenStateOffset(layer_id, vector_id, direction); - - const std::array pos{layer_id, vector_id, static_cast(direction)}; - - return start_ident + - std::inner_product( - pos.cbegin(), pos.cend(), hStateStrides.cbegin(), static_cast(0)); - } - - // layer, minor dim(seq or sample), directions, element - const std::array& getGateAndStateStrideImpl(LstmGateAndState gas) const - { - if(gas == LstmGateAndState::Ht || gas == LstmGateAndState::St) - return getHiddenStateStride(); - return gateStride; - } - - // layer, minor dim(seq or sample), directions, element - const std::array& getGateBlockSizeImpl() const { return gateBlockSizes; } - - const std::array gateSizes; - const std::array gateStride; - const std::array gateBlockSizes; - -private: - size_t getGasIndent(LstmGateAndState gas) const - { - switch(gas) - { - case LstmGateAndState::I: - case LstmGateAndState::G: - case LstmGateAndState::F: - case LstmGateAndState::O: return static_cast(gas) * gateStride[3] * gateSizes[3]; - case LstmGateAndState::St: return gateStride[2] * gateSizes[2]; // direction DIM - case LstmGateAndState::Ht: - return (gateStride[2] + getHiddenStateStride()[2]) * gateSizes[2]; - } - return 0; - } -}; - -/* - *struct ReserveSpace_LSTM{ //packed - * struct layer{ - * struct all_sequences_all_samples{ - * - * struct mat_mul_block{ - * struct gate_vector{ - * float[hidden_size]; HidUpdate-> write - * } I, F, O, G; - * } gemm_block[directions]; - * - * struct dcx_vec{ - * float[hidden_size]; HidUpdate-> write - * } dy[directions]; - * - * struct dy_vec{ - * float[hidden_size]; HidUpdate <-read - * } dy[directions]; - * - * } combi_dim[seq_len * batch_size=totalBatch]; [seq][batch] - * } layers[nLayer]; - * - * struct extra_activation_vec{ - * float[hidden_size]; - * } active_cell[nLayer*bidirect*totalBatch]; - *} - */ - -class GeneralLstmRedBuffer : public GeneralLstmTempBuffer, - public LstmActiveCellExt -{ -protected: - GeneralLstmRedBuffer(const GeneralLstmTempBuffer& base, - const std::array& active_cells_sizes, - const std::array& active_cells_strides, - size_t active_cells_ident, - size_t active_cell_elements) - : GeneralLstmTempBuffer{base}, - activeCellSize{active_cells_sizes}, - activeCellStride{active_cells_strides}, - activeCellsIdent{active_cells_ident}, - activeCellElements{active_cell_elements} - { - } - -public: - static GeneralLstmRedBuffer - build(size_t layers_cnt, size_t comp_dim_per_layer, size_t directions, size_t hidden_vec_sz) - { - - auto base = - GeneralLstmTempBuffer::build(layers_cnt, comp_dim_per_layer, directions, hidden_vec_sz); - - auto active_cells_ident = base.gateStride[0] * base.gateSizes[0]; - - std::array active_cells_sizes{ - layers_cnt, comp_dim_per_layer, directions, hidden_vec_sz}; - - std::array active_cells_strides = {}; - - WA_RHEL::exclusive_scan_wa(active_cells_sizes.crbegin(), - active_cells_sizes.crend(), - active_cells_strides.rbegin(), - 1LL, - std::multiplies{}); - - auto active_cell_elements = active_cells_strides[0] * active_cells_sizes[0]; - - return {base, - active_cells_sizes, - active_cells_strides, - active_cells_ident, - active_cell_elements}; - } - - size_t getBufferSizeImpl() const { return activeCellsIdent + activeCellElements; } - - size_t getActiveCellOffsetImpl(const size_t layer_id, - const size_t vector_id, - const SequenceDirection direction) const - { - const std::array pos{layer_id, vector_id, static_cast(direction)}; - - return activeCellsIdent + - std::inner_product( - pos.cbegin(), pos.cend(), activeCellStride.cbegin(), static_cast(0)); - } - - const std::array& getActiveCellStrideImpl() const { return activeCellStride; } - - const std::array activeCellSize; - const std::array activeCellStride; - const size_t activeCellsIdent; - const size_t activeCellElements; -}; - -/** - * \section Сlasses for easier navigation. - */ - -class BatchController -{ -public: - template - static BatchController Create(VectorT&& batchs_at_time) - { - std::vector batch{std::forward(batchs_at_time)}; - std::vector b_prefix_sums(batch.size()); - b_prefix_sums[0] = 0; - - std::partial_sum(batch.cbegin(), std::prev(batch.cend()), std::next(b_prefix_sums.begin())); - - return BatchController{std::move(batch), std::move(b_prefix_sums)}; - } - - static BatchController Create(const SeqTensorDescriptor& desc) - { - return Create(desc.GetBatchesPerSequence()); - } - - size_t getTotalBatchSum() const - { - return *batchAtTime.crbegin() + *batchPrefSumAtTime.crbegin(); - } - - template - size_t getBatchSize(TimeIndexT time_id) const - { - return batchAtTime[time_id]; - } - - template - size_t getBatchSum(TimeIndexT time_id) const - { - return batchPrefSumAtTime[time_id]; - } - -private: - template >::value, bool> = true> - explicit BatchController(T&& batch_at_time, T&& batch_prefix_sums) - : batchAtTime(batch_at_time), batchPrefSumAtTime{std::forward(batch_prefix_sums)} - { - } - - const std::vector batchAtTime; - const std::vector batchPrefSumAtTime; -}; - -class SequenceIterator -{ -public: - SequenceIterator(size_t logical_value, SequenceDirection dir, size_t seq_size, bool is_fwd_pass) - : value(logical_value), - startVal(is_fwd_pass ? 0 : seq_size - 1), - endVal(is_fwd_pass ? seq_size - 1 : 0), - maxVal(seq_size - 1), - isReverseSeq(dir == SequenceDirection::Reverse), - isFwdPass(is_fwd_pass) - { - assert(logical_value < seq_size); - } - - size_t getLogVal() const { return value; } - size_t getPhisVal() const { return !isReverseSeq ? value : maxVal - value; } - - SequenceIterator getNext() const - { - assert(!isLast()); - return PartClone(value + (isFwdPass ? 1 : -1)); - } - SequenceIterator getPrev() const - { - assert(!isFirst()); - return PartClone(value + (isFwdPass ? -1 : 1)); - } - - bool isFirst() const { return value == startVal; } - - bool isLast() const { return value == endVal; } - -private: - SequenceIterator PartClone(const size_t newValue) const - { - return {newValue, startVal, endVal, maxVal, isReverseSeq, isFwdPass}; - } - - SequenceIterator(size_t value_, - size_t start_val, - size_t end_val, - size_t max_val, - bool is_reverse_seq, - bool is_fwd_pass) - : value(value_), - startVal(start_val), - endVal(end_val), - maxVal(max_val), - isReverseSeq(is_reverse_seq), - isFwdPass(is_fwd_pass) - { - } - - const size_t value; - const size_t startVal; - const size_t endVal; - const size_t maxVal; - const bool isReverseSeq; - const bool isFwdPass; -}; - -/** - * \section User defined buffers. - */ - -/* - *struct Weights{ - * struct filter{ - * - * struct gateInputMatrix{ - * float[hidden_size][input_size] - * } gateFilters[gate_cnt]; - * - * struct gateHiddenMatrix{ - * float[hidden_size][hidden_size] - * } gateFiltersHiddden[gate_cnt]; - * } filters[n_layers][bidirect]; - * - * struct bias{ - * struct { - * float[hidden_size] - * }gateBias[gate_cnt] - * }biases[n_layers][bias_cnt][bidirect] - *} - */ -struct WeightsBufferDescriptor -{ -private: - static size_t hidden_xinput_size(size_t hidden_sz, bool bidirect_mode) - { - if(!bidirect_mode) - return hidden_sz; - MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); - } - - static auto filter_size(size_t input_vector_sz, size_t hidden_vec_sz, size_t gates) - { - return (input_vector_sz + hidden_vec_sz) * hidden_vec_sz * gates; - } - - static size_t bias_start_offset(size_t phis_layer_filter_sz, - size_t hidden_layer_filter_sz, - size_t layers_cnt, - bool bidirect_mode) - { - if(!bidirect_mode) - { - return phis_layer_filter_sz + hidden_layer_filter_sz * (layers_cnt - 1); - } - - MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); - } - - WeightsBufferDescriptor(size_t input_vector_sz, - size_t hidden_vec_sz, - size_t hidden_xinput_sz, - size_t layers_cnt, - size_t gates, - size_t bias_mode, - - size_t lin_filter_sz, - size_t hidden_filter_sz, - const std::array& bias_strides, - const std::array& bias_lens, - size_t bias_off) - : inVec(input_vector_sz), - hVec(hidden_vec_sz), - xInVec(hidden_xinput_sz), - layers(layers_cnt), - gatesCnt(gates), - biasCnt(bias_mode), - biasStrides{bias_strides}, - biasLens{bias_lens}, - biasStartOff(bias_off), - linLayerFilterSize(lin_filter_sz), - hiddenLayerFilterSize(hidden_filter_sz) - { - } - -public: - static WeightsBufferDescriptor create(size_t input_vector_sz, - size_t hidden_vec_sz, - size_t layers_cnt, - size_t bias_mode, - size_t input_mode, - size_t gates, - bool bidirect_mode) - { - const size_t directions_num = bidirect_mode ? 2 : 1; - const size_t x_in_vec = hidden_xinput_size(hidden_vec_sz, bidirect_mode); - - size_t input_vector_filter_sz = input_mode == 0 ? input_vector_sz : 0; - - const size_t linLayerFilterSize = filter_size(input_vector_filter_sz, hidden_vec_sz, gates); - const size_t hiddenLayerFilterSize = filter_size(x_in_vec, hidden_vec_sz, gates); - - const size_t bias_start_off = - bias_start_offset(linLayerFilterSize, hiddenLayerFilterSize, layers_cnt, bidirect_mode); - - //[layer][dir][param_id][vector] - const std::array bias_lens{layers_cnt, directions_num, gates, hidden_vec_sz}; - const std::array bias_strides = [](const auto& lens, size_t bias_cnt) { - std::array strides = {0, 0, 0, 1}; - std::partial_sum(lens.crbegin(), - std::prev(lens.crend()), - std::next(strides.rbegin()), - std::multiplies{}); - strides[0] *= bias_cnt; - return strides; - }(bias_lens, bias_mode * 2); - - return {input_vector_filter_sz, - hidden_vec_sz, - x_in_vec, - layers_cnt, - gates, - bias_mode, - linLayerFilterSize, - hiddenLayerFilterSize, - bias_strides, - bias_lens, - bias_start_off}; - } - - const size_t inVec, hVec; - const size_t xInVec; // for bidirect TODO - - const size_t layers; - const size_t gatesCnt; - const size_t - biasCnt; // 0 - no bisa; 1 - one bias; 2 - separate bias for x_vec and for hidden_vec -private: - //[layer][dir][param_id][vector] - const std::array biasStrides; - const std::array biasLens; - - const size_t biasStartOff; - - const size_t linLayerFilterSize; - const size_t hiddenLayerFilterSize; - -public: - size_t getParamRelativeOff([[maybe_unused]] size_t layer_id, int /*dir_id*/, int param_id) const - { - assert(layer_id > 0); - return param_id * hVec * xInVec; - } - - size_t getPhisParamRelativeOff(int /*dir_id*/, int param_id) const - { - return hVec * ((static_cast(param_id) >= gatesCnt) - ? inVec * gatesCnt + xInVec * (param_id - gatesCnt) - : inVec * param_id); - } - - size_t getMatrixOff(size_t layer_id, int dir_id, int param_id) const - { - size_t offset = 0; - if(layer_id > 0) - { - offset += linLayerFilterSize; - } - else - { - return getPhisParamRelativeOff(dir_id, param_id); - } - if(layer_id > 1) - { - offset += hiddenLayerFilterSize * (layer_id - 1); - } - return offset + getParamRelativeOff(layer_id, dir_id, param_id); - } - - size_t getMatrixXinOff(size_t layer_id, int dir_id) const - { - return getMatrixOff(layer_id, dir_id, 0); - } - - size_t getMatrixHidOff(size_t layer_id, int dir_id) const - { - return getMatrixOff(layer_id, dir_id, gatesCnt); - } - - size_t getBiasOff(size_t layer_id, int dir_id, int param_id) const - { - return biasStartOff + biasStrides[0] * layer_id + biasStrides[1] * dir_id + - biasStrides[2] * param_id; - } - - size_t getBiasXinOff(size_t layer_id, int dir_id, int param_id) const - { - assert(param_id < gatesCnt); - return getBiasOff(layer_id, dir_id, param_id); - } - - size_t getBiasHidOff(size_t layer_id, int dir_id, int param_id) const - { - assert(param_id < gatesCnt); - return getBiasOff(layer_id, dir_id, param_id + gatesCnt); - } - - //[layers][dirs][params][vec_size] - const std::array& getBiasSize() const { return biasLens; } - - //[layers][dirs][params][vec_size] - const std::array& getBiasStride() const { return biasStrides; } -}; - -class HiddenBuffersDescriptor -{ -public: - explicit HiddenBuffersDescriptor(const TensorDescriptor& hx_desc) - : lens(hx_desc.GetLengths().begin(), hx_desc.GetLengths().begin() + 3), - strides(hx_desc.GetStrides().begin(), hx_desc.GetStrides().begin() + 3) - { - } - - inline size_t getOffset(const size_t virtual_layer_id) const - { - return getOffset(virtual_layer_id, 0); - } - - inline size_t getOffset(const size_t virtual_layer_id, const size_t batch_id) const - { - return strides[0] * virtual_layer_id + strides[1] * batch_id; - } - - inline const std::vector& getStrides() const { return strides; } - - inline const std::vector& GetLengths() const { return lens; } - - inline size_t getVirtualLayersCnt() const { return lens[0]; } - inline size_t getMiniBatchSize() const { return lens[1]; } - inline size_t getHiddenSize() const { return lens[2]; } - - static std::vector MakeStrides(const std::vector& lengths) - { - return {lengths[1] * lengths[2], lengths[2], 1}; - } - -private: - // local caching - - const std::vector lens; - const std::vector strides; -}; - -class IOBufferDescriptor -{ - IOBufferDescriptor() = default; - IOBufferDescriptor(std::vector&& buffer_lens, - std::vector&& buffer_strides, - std::vector&& packed_lens, - std::vector&& packed_strides, - std::vector&& seq_lens_per_sample) - : lens{std::move(buffer_lens)}, - strides{std::move(buffer_strides)}, - packedLens{std::move(packed_lens)}, - packedStrides{std::move(packed_strides)}, - seqLensPerSample{std::move(seq_lens_per_sample)} - { - } - -public: - static IOBufferDescriptor build(const SeqTensorDescriptor& xyDesc) - { - //{batch, seq_cnt, vector} - auto lens = xyDesc.GetLengths(); - auto strides = xyDesc.GetPaddedStrides(); - - //{ combine(batch, seq_cnt), vector} - std::vector packed_lens{xyDesc.GetTotalSequenceLen(), lens[2]}; - std::vector packed_strides(2); - - WA_RHEL::exclusive_scan_wa(packed_lens.crbegin(), - std::next(packed_lens.crbegin(), packed_strides.size()), - packed_strides.rbegin(), - 1LL, - std::multiplies{}); - - std::vector seq_lens_per_sample = xyDesc.GetSequenceLengthsVector(); - return {(std::move(lens)), - (std::move(strides)), - (std::move(packed_lens)), - (std::move(packed_strides)), - (std::move(seq_lens_per_sample))}; - } - - inline size_t getPackedOffset(const size_t batch_id) const - { - return packedStrides[0] * batch_id; - } - - inline std::vector getFullSeqMajorStrides() const { return packedStrides; } - inline std::vector getFullSeqMajorSize() const { return packedLens; } - - inline size_t getMiniBatchSize() const { return lens[0]; } - inline size_t getMaxSeqSize() const { return lens[1]; } - inline size_t getHiddenSize() const { return lens[2]; } - - inline size_t getSeqSize(size_t sample_id) const { return seqLensPerSample[sample_id]; } - inline size_t getTotalSeqCnt() const { return packedLens[0]; } - - static std::vector MakeStrides(const std::vector& lengths) - { - return {lengths[1] * lengths[2], lengths[2], 1}; - } - - // private: - // local caching - - const std::vector lens; - const std::vector strides; - - const std::vector packedLens; - const std::vector packedStrides; - - const std::vector seqLensPerSample; -}; - -} // namespace rnn_base -} // namespace miopen diff --git a/src/include/miopen/solver.hpp b/src/include/miopen/solver.hpp index 69f47f6ed6..e69de29bb2 100644 --- a/src/include/miopen/solver.hpp +++ b/src/include/miopen/solver.hpp @@ -1,277 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef GUARD_MIOPEN_SOLVER_HPP_ -#define GUARD_MIOPEN_SOLVER_HPP_ - -#include - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -namespace miopen { - -struct AnyInvokeParams; - -namespace solver { - -/// Base class for problem solvers. -/// -/// Solvers are to be instantiated as const objects and shall not have any variable -/// internal state. Any non-const state information, if required, to be stored in the -/// solver-specific context objects. -/// -/// There could be multiple solvers of the same algorithm for a problem config. -struct SolverBase -{ - virtual ~SolverBase() = default; - - /// This will retrieve the id of the solver to write to the database. By - /// default it uses the class name. If the class is renamed, this function can - /// overriden to keep the name to avoid DB corruption. - virtual const std::string& SolverDbId() const = 0; - - /// In some instances (particularly fusions) the fused solver might like to - /// fallback to the non-fused variant for performance parameters, this information - /// is returned via AltSolverDbId - virtual const std::string& AltSolverDbId() const - { - static const std::string null_id = ""; - return null_id; - } - - /// Returns true if solution can work on given SW/HW platform (runtime/device) - /// and provides correct result for the problem config. - /// - /// Every SolverBase which IsApplicable() for some problem config must be able to - /// GetDefaultPerformanceConfig() so that GetSolution() would return valid - /// solution for a problem (i.e. convolution). In other words, if a Solution - /// says "I'm suitable" for a problem, it agrees to solve that problem correctly. - virtual bool IsApplicable(const ExecutionContext& ctx, const boost::any& problem) const = 0; - - /// [Informative as of Sep 2020] The minimum requirement for Dynamic Solvers: - /// Batch size and input picture size (N, W, H) must NOT be compiled into the - /// kernel(s) that consist a Solution. These must go into the kernel as a - /// run-time parameters. - virtual bool IsDynamic() const { return false; } - - static constexpr float wti_approximate_worst = -2; - - /// [Informative as of Sep 2020] Returns an approximated value of the expected - /// WTI or wti_approximate_worst when this value can't be computed. Tips: - /// * Value 1.0 corresponds to the 100% utilization of HW capabilities as - /// if Direct computational algorithm is used. - /// * [Notice] WTI may exceed 1.0 for highly optimized algorithms like Winograd. - /// * @see https://github.com/ROCm/MIOpen/issues/410 - virtual float GetWti(const ExecutionContext& ctx, const boost::any& problem) const = 0; - - /// Returns the workspace size required by the solver for a given ExecutionContext - virtual size_t GetWorkspaceSize(const ExecutionContext& ctx, - const boost::any& problem) const = 0; - - /// Must return true if a Solver has its own implementation of GetWorkspaceSize(). - virtual bool MayNeedWorkspace() const { return false; } - -protected: - template - static const std::string& GetSolverDbId() - { - static const auto result = ComputeSolverDbId(get_type_name()); - return result; - } - SolverBase() = default; - SolverBase(const SolverBase&) = default; - -private: - static std::string ComputeSolverDbId(const std::string& type_name) - { - auto idx = type_name.find_last_of(':'); - auto name = type_name.substr(idx + 1); - std::replace(name.begin(), name.end(), ',', '-'); - name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); - - return name; - } -}; - -template -struct SolverMixin : SolverBase -{ - static_assert(std::is_base_of{}, - "Context must be derived of ExecutionContext"); - - virtual bool IsApplicable(const Context&, const Problem&) const = 0; - virtual float GetWti(const Context&, const Problem&) const { return wti_approximate_worst; }; - virtual size_t GetWorkspaceSize(const Context&, const Problem&) const { return 0; }; - - bool IsApplicable(const ExecutionContext& ctx, const boost::any& problem) const final - { - return IsApplicable(dynamic_cast(ctx), - boost::any_cast(problem)); - } - - float GetWti(const ExecutionContext& ctx, const boost::any& problem) const final - { - return GetWti(dynamic_cast(ctx), boost::any_cast(problem)); - } - - size_t GetWorkspaceSize(const ExecutionContext& ctx, const boost::any& problem) const final - { - return GetWorkspaceSize(dynamic_cast(ctx), - boost::any_cast(problem)); - } -}; - -/// Base class for non tunable solvers -template -struct NonTunableSolverBase : SolverMixin -{ - /// Takes problem config, optimization parameters and other info - /// and computes information required to build and run the kernel(s). - virtual ConvSolution GetSolution(const Context&, const Problem&) const = 0; - - virtual InvokerFactory GetInvokerFactory(const Context& ctx, const Problem& problem) const - { - return *GetSolution(ctx, problem).invoker_factory; - } -}; - -struct TunableSolverTrait -{ -}; - -/// Base class for tunable solvers -template -struct TunableSolverBase : SolverMixin, TunableSolverTrait -{ - /// Initializes performance config to the default values. - /// The function may involve some heuristic to guess the best solution - /// configuration. It is assumed that the function takes constant time - /// to finish and does not run kernels to measure performance etc. - /// The function shall always return valid config. - /// - /// The int parameter is needed only to not change the name of the - /// function in the derived class. Function declarations that differ - /// only by its return type cannot be overloaded. - virtual boost::any - GetDefaultPerformanceConfig(const Context& ctx, const Problem& problem, int) const = 0; - - /// Should return false if performance config is wrong for a problem. - /// Main use is validation of values read from the perf db. - virtual bool IsValidPerformanceConfig(const Context& ctx, - const Problem& problem, - const PerfConfig& config) const = 0; - - /// Search - /// - /// The int parameter is needed only to not change the name of the - /// function in the derived class. Function declarations that differ - /// only by its return type cannot be overloaded. - virtual boost::any Search(const Context& ctx, - const Problem& problem, - const AnyInvokeParams& invoke_ctx, - int) const = 0; - - /// Tunable solvers provide a GetSolution that takes a Context and PerformanceConfig - virtual ConvSolution - GetSolution(const Context& ctx, const Problem& problem, const PerfConfig& config) const = 0; - - virtual InvokerFactory - GetInvokerFactory(const Context& ctx, const Problem& problem, const PerfConfig& config) const - { - return *GetSolution(ctx, problem, config).invoker_factory; - } -}; - -template -struct TunableSolverMixin : TunableSolverBase -{ - static_assert(std::is_base_of{}, - "PerformanceConfig must be derived of PerfConfig"); - - virtual PerformanceConfig GetDefaultPerformanceConfig(const Context&, const Problem&) const = 0; - virtual bool - IsValidPerformanceConfig(const Context&, const Problem&, const PerformanceConfig&) const = 0; - virtual PerformanceConfig - Search(const Context&, const Problem&, const AnyInvokeParams&) const = 0; - virtual ConvSolution - GetSolution(const Context&, const Problem&, const PerformanceConfig&) const = 0; - - boost::any - GetDefaultPerformanceConfig(const Context& ctx, const Problem& problem, int) const final - { - return GetDefaultPerformanceConfig(ctx, problem); - } - - bool IsValidPerformanceConfig(const Context& ctx, - const Problem& problem, - const PerfConfig& config) const final - { - return IsValidPerformanceConfig( - ctx, problem, dynamic_cast(config)); - } - - boost::any Search(const Context& ctx, - const Problem& problem, - const AnyInvokeParams& invoke_ctx, - int) const final - { - return Search(ctx, problem, invoke_ctx); - } - - ConvSolution - GetSolution(const Context& ctx, const Problem& problem, const PerfConfig& config) const final - { - return GetSolution(ctx, problem, dynamic_cast(config)); - } -}; - -template -struct IsTunable : std::is_base_of -{ - static_assert(!std::is_same_v, - "Raw trait shouldn't be passed, explicit type is needed"); -}; - -// Use struct as a syntactic sugar to make the intent as clear as possible. -struct ThisSolverIsDeprecatedStatic -{ - MIOPEN_INTERNALS_EXPORT static bool IsDisabled(const ExecutionContext& ctx); -}; - -} // namespace solver -} // namespace miopen - -#endif // GUARD_MIOPEN_SOLVER_HPP_ diff --git a/src/include/miopen/solver/implicitgemm_ck_util.hpp b/src/include/miopen/solver/implicitgemm_ck_util.hpp index 5e2f093d3e..e69de29bb2 100644 --- a/src/include/miopen/solver/implicitgemm_ck_util.hpp +++ b/src/include/miopen/solver/implicitgemm_ck_util.hpp @@ -1,1017 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL -#include -#include -#endif // MIOPEN_USE_COMPOSABLEKERNEL - -namespace miopen { - -namespace conv { -struct ProblemDescription; -} // namespace conv - -namespace solver { -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL -namespace conv { -template -using DeviceOpGWrw = ck::tensor_operation::device::DeviceGroupedConvBwdWeight< - 2, - ck::tensor_layout::convolution::NHWGC, - ck::tensor_layout::convolution::GKYXC, - ck::tensor_layout::convolution::NHWGK, - DataType, - DataType, - DataType, - ck::tensor_operation::element_wise::PassThrough, - ck::tensor_operation::element_wise::PassThrough, - ck::tensor_operation::element_wise::PassThrough>; -template -using DeviceOpGWrwPtrs = - ck::tensor_operation::device::instance::DeviceOperationInstanceFactory>; -} // namespace conv -#endif - -inline bool IsLinear(int L, int H, const int v) -{ - assert(L <= H); - return L <= v && v <= H; -} - -inline bool NextLinear(int L, int H, int& v) -{ - assert((IsLinear(L, H, v))); - if(H == v) - { - v = L; - return true; - } - ++v; - return false; -} - -struct ConvSolution; - -struct CKBWDWeightBufferDescriptor -{ - size_t ck_size; - size_t ck_offset; - - CKBWDWeightBufferDescriptor(size_t _ck_size, size_t _ck_offset) - : ck_size(_ck_size), ck_offset(_ck_offset) - { - } -}; - -template -typename ConvPtrsType::iterator FindConvPtrByID(ConvPtrsType& conv_ptrs, - const std::string& kernel_id) -{ - return std::find_if(conv_ptrs.begin(), conv_ptrs.end(), [&kernel_id](const auto& ptr) { - return ptr->GetTypeString() == kernel_id; - }); -} - -template -std::vector FillValidKernelsIDs(const ProblemDescriptionType& problem) -{ - const auto args = CKArgsType{problem}; - const auto conv_ptrs = DeviceOpType::GetInstances(); - assert(!conv_ptrs.empty()); - - std::vector valid_kernels; - valid_kernels.reserve(conv_ptrs.size()); - for(size_t idx = 0; idx < conv_ptrs.size(); ++idx) - { - if(args.IsSupportedBy(conv_ptrs[idx])) - valid_kernels.emplace_back(std::move(conv_ptrs[idx]->GetTypeString())); - } - assert(!valid_kernels.empty()); - return valid_kernels; -} - -template -bool IsCKArgsSupported(const ProblemDescriptionType& problem, const std::string& kernel_id) -{ -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - if(!kernel_id.empty()) - { - auto conv_ptrs = DeviceOpType::GetInstances(); - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) - { - auto pos = kernel_id.find_last_of('+'); - int split_k = std::stoi(kernel_id.substr(pos + 1)); - auto ptr_iter = FindConvPtrByID(conv_ptrs, kernel_id.substr(0, pos)); - return (ptr_iter != conv_ptrs.end()) && - CKArgsType{problem}.IsSupportedBySplitK(*ptr_iter, split_k); - } - else - { - auto ptr_iter = FindConvPtrByID(conv_ptrs, kernel_id); - return (ptr_iter != conv_ptrs.end()) && CKArgsType{problem}.IsSupportedBy(*ptr_iter); - } - } -#endif - return false; -} - -template -bool IsCKApplicable(const ProblemDescriptionType& problem) -{ - const auto args = CKArgsType{problem}; - - const auto ptrs = DeviceOpType::GetInstances(); - return std::any_of( - ptrs.begin(), ptrs.end(), [&args](auto& ptr) { return args.IsSupportedBy(ptr); }); -} - -#define WORKAROUND_CK_ISSUE_1184 1 -#if WORKAROUND_CK_ISSUE_1184 -using WorkAroundHipEventProfiler = HipEventProfiler; -#endif - -inline bool isDataTypeHalfAndChannelsEven(const miopen::conv::ProblemDescription& problem) -{ - return (problem.GetOutDataType() == miopenHalf) && - ((problem.GetInChannels() & 1) != 0 || - (problem.GetOutChannels() & 1) != 0 /* Test if odd*/); -} - -inline bool ShouldAllocateWorkSpaceBufferForWRW(const miopen::conv::ProblemDescription& problem) -{ - return (problem.GetAlphaBetaCase() != DEFAULT || isDataTypeHalfAndChannelsEven(problem)); -} - -template -ConvSolution InitAnyInvokerFactory(const ProblemDescriptionType& problem, - const std::string& kernel_id) -{ - auto conv_ptrs = DeviceOpType::GetInstances(); - auto ptr_iter = FindConvPtrByID(conv_ptrs, kernel_id); - - if(ptr_iter == conv_ptrs.end()) - return {miopenStatusInvalidValue}; - - ConvSolution result; - result.invoker_factory = - [ck_args = CKArgsType{problem}, - sh_conv_ptr = std::shared_ptr{std::move(*ptr_iter)}](const std::vector&) mutable { - return [ck_args = std::move(ck_args), sh_conv_ptr = std::move(sh_conv_ptr)]( - const Handle& handle, const AnyInvokeParams& primitive_parameters) { - const auto& data_ctx = primitive_parameters.CastTo(); - auto argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, data_ctx); - auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); - - const auto enable_profiling = handle.IsProfilingEnabled(); - float elapsed_time = - invoker_ptr->Run(argument_ptr.get(), {handle.GetStream(), enable_profiling}); - if(enable_profiling) - { - handle.ResetKernelTime(); - handle.AccumKernelTime(elapsed_time); - } - }; - }; - return result; -} - -namespace internal { - -enum class ConvOperandTag : int -{ - Input = 0, - Weights, - Output -}; - -enum class TranposeKind : int -{ - NHWC_TO_NCHW = 0, - NCHW_TO_NHWC -}; - -template -struct TransposeOperand -{ - static_assert(ND == 2 || ND == 3, "Num Dimensions must be 2 or 3"); - constexpr static int NDIM = ND; - constexpr static ConvOperandTag CONV_OP_TAG = CONV_OP; - constexpr static TranposeKind TRANSPOSE_KIND = TPOSE_KIND; - - using SolverType = - std::conditional_t, - // NCHW_TO_NHWC - std::conditional_t>; - - template - SolverType MakeTransposeSolver(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const CKArgsType& ck_args) const - { - - if constexpr(CONV_OP_TAG == ConvOperandTag::Input) - { - if constexpr(ND == 3) - { - - return SolverType{ctx, - problem.GetInDataType(), - static_cast(ck_args.N), - static_cast(ck_args.C1), - static_cast(ck_args.Di), - static_cast(ck_args.Hi), - static_cast(ck_args.Wi)}; - } - else - { - return SolverType{ctx, - problem.GetInDataType(), - static_cast(ck_args.N), - static_cast(ck_args.C1), - static_cast(ck_args.Hi), - static_cast(ck_args.Wi)}; - } - } - else if constexpr(CONV_OP_TAG == ConvOperandTag::Weights) - { - if constexpr(ND == 3) - { - return SolverType{ctx, - problem.GetWeightsDataType(), - static_cast(ck_args.K1), - static_cast(ck_args.C), - static_cast(ck_args.Z), - static_cast(ck_args.Y), - static_cast(ck_args.X)}; - } - else - { - return SolverType{ctx, - problem.GetWeightsDataType(), - static_cast(ck_args.K1), - static_cast(ck_args.C), - static_cast(ck_args.Y), - static_cast(ck_args.X)}; - } - } - else - { - static_assert(CONV_OP_TAG == ConvOperandTag::Output); - if constexpr(ND == 3) - { - return SolverType{ctx, - problem.GetOutDataType(), - static_cast(ck_args.N), - static_cast(ck_args.K1), - static_cast(ck_args.Do), - static_cast(ck_args.Ho), - static_cast(ck_args.Wo)}; - } - else - { - return SolverType{ctx, - problem.GetOutDataType(), - static_cast(ck_args.N), - static_cast(ck_args.K1), - static_cast(ck_args.Ho), - static_cast(ck_args.Wo)}; - } - } - } -}; - -// Shorthand aliases for CK assuming CK always expects and generates NHWC/NDHWC layouts -template -using CKTransposeInputOp = TransposeOperand; - -template -using CKTransposeOutputOp = TransposeOperand; - -class TransposeInstance -{ - size_t tensor_sz = 0; - std::vector kern_args{}; - size_t kern_idx = std::numeric_limits::max(); - size_t buf_offset = 0; - shared buf_handle{}; - -public: - template - TransposeInstance(const TransSolnType& trans_sol, - size_t k_idx, - const MultiBufferWorkspaceTraits& wt, - size_t wspace_index) - : tensor_sz(trans_sol.GetOutputTensorSize()), - kern_args(trans_sol.GetKernelArg()), - kern_idx(k_idx), - buf_offset(wt.GetOffset(wspace_index)) - { - } - - void AssignBuffer(const Handle& handle, Data_t workSpace) - { - buf_handle = handle.CreateSubBuffer(workSpace, buf_offset, tensor_sz); - assert(buf_handle.get()); - } - - Data_t GetBufferPtr() const { return buf_handle.get(); } - - void ConvertFrom(const Handle& handle, const std::vector& kernels, ConstData_t in_ptr) - { - Run(handle, kernels, buf_handle.get(), in_ptr); - } - - void ConvertTo(const Handle& handle, const std::vector& kernels, Data_t out_ptr) - { - Run(handle, kernels, out_ptr, buf_handle.get()); - } - - void ZeroOutBuffer(const Handle& handle) - { - [[maybe_unused]] auto status = - hipMemsetAsync(buf_handle.get(), 0, tensor_sz, handle.GetStream()); - assert(status == hipSuccess); - } - - TransposeInstance() = delete; - TransposeInstance(const TransposeInstance&) = default; - TransposeInstance(TransposeInstance&&) = default; - ~TransposeInstance() = default; - -private: - void Run(const Handle& handle, - const std::vector& kernels, - Data_t out_ptr, - ConstData_t in_ptr) - { - assert(out_ptr); - assert(in_ptr); - assert(kernels.size() > kern_idx); - - kern_args[0] = out_ptr; - kern_args[1] = in_ptr; - - auto save = handle.IsProfilingEnabled() ? 0.0f : handle.GetKernelTime(); - handle.Run(kernels[kern_idx])(kern_args); - if(handle.IsProfilingEnabled()) - { - handle.AccumKernelTime(save); - } - } -}; - -class TransposeInstanceTagged : public TransposeInstance -{ - - ConvOperandTag conv_op_tag_; - -public: - template - TransposeInstanceTagged(const TransSolnType& sol, - size_t k_idx, - const MultiBufferWorkspaceTraits& wt, - size_t wspace_index, - ConvOperandTag conv_op_tag) - : TransposeInstance(sol, k_idx, wt, wspace_index), conv_op_tag_(conv_op_tag) - { - } - - ConvOperandTag GetConvOperandTag() const noexcept { return conv_op_tag_; } - - std::underlying_type_t GetConvOperandTagAsInt() const noexcept - { - using IntType = std::underlying_type_t; - return static_cast(GetConvOperandTag()); - } - - void ConvertFrom(const Handle& handle, - const std::vector& kernels, - const ConvTensors& tensors) - { - TransposeInstance::ConvertFrom(handle, kernels, pickTensorPtr(tensors)); - } - - void - ConvertTo(const Handle& handle, const std::vector& kernels, const ConvTensors& tensors) - { - TransposeInstance::ConvertTo(handle, kernels, pickTensorPtr(tensors)); - } - - TransposeInstanceTagged() = delete; - TransposeInstanceTagged(const TransposeInstanceTagged&) = default; - TransposeInstanceTagged(TransposeInstanceTagged&&) = default; - ~TransposeInstanceTagged() = default; - -private: - Data_t pickTensorPtr(const ConvTensors& tensors) const - { - std::array data_ptrs = { - const_cast(tensors.x), // NOLINT (cppcoreguidelines-pro-type-const-cast) - const_cast(tensors.w), // NOLINT (cppcoreguidelines-pro-type-const-cast) - const_cast(tensors.y) // NOLINT (cppcoreguidelines-pro-type-const-cast) - }; - - return data_ptrs[GetConvOperandTagAsInt()]; - } -}; - -template -auto MakeTaggedTransposeInstances(ConvSolution& result, - const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const CKArgsType& ck_args, - const Input1TposeOp& input1_op, - const Input2TposeOp& input2_op, - const OutputTposeOp& output_op, - std::optional& ck_buff_des) -{ - - auto input1_solver = input1_op.MakeTransposeSolver(ctx, problem, ck_args); - auto input2_solver = input2_op.MakeTransposeSolver(ctx, problem, ck_args); - auto output_solver = output_op.MakeTransposeSolver(ctx, problem, ck_args); - - // NOTE: In cases where the convolution updates only a subset of output - // indices, we need to first initialize the workspace buffer for - // output with the real tensor for the output and then apply the convolution. - // This is achieved by creating an input transpose op for the output workspace - // bufffer. - - using OutputInitOp = CKTransposeInputOp; - - auto output_init_solver = OutputInitOp{}.MakeTransposeSolver(ctx, problem, ck_args); - - result.construction_params.insert(result.construction_params.end(), - {input1_solver.GetKernelInfo(), - input2_solver.GetKernelInfo(), - output_solver.GetKernelInfo(), - output_init_solver.GetKernelInfo()}); - - if(ck_buff_des.has_value()) - { - MultiBufferWorkspaceTraits wt({input1_solver.GetOutputTensorSize(), - input2_solver.GetOutputTensorSize(), - output_solver.GetOutputTensorSize(), - ck_buff_des->ck_size}); - ck_buff_des->ck_offset = wt.GetOffset(3); - return std::make_tuple( - TransposeInstanceTagged{input1_solver, 0, wt, 0, Input1TposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{input2_solver, 1, wt, 1, Input2TposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{output_solver, 2, wt, 2, OutputTposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{output_init_solver, 3, wt, 2, OutputTposeOp::CONV_OP_TAG}); - } - - MultiBufferWorkspaceTraits wt({input1_solver.GetOutputTensorSize(), - input2_solver.GetOutputTensorSize(), - output_solver.GetOutputTensorSize()}); - return std::make_tuple( - TransposeInstanceTagged{input1_solver, 0, wt, 0, Input1TposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{input2_solver, 1, wt, 1, Input2TposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{output_solver, 2, wt, 2, OutputTposeOp::CONV_OP_TAG}, - TransposeInstanceTagged{output_init_solver, 3, wt, 2, OutputTposeOp::CONV_OP_TAG}); -} - -#ifndef NDEBUG // disable for release builds, enable for debug builds - -template -void DebugPrintVec(const char* name, const V& vec) -{ - std::ostringstream oss; - oss << name << " = [ "; - for(const auto& v : vec) - { - oss << v << ", "; - } - oss << "]"; - MIOPEN_LOG_I(oss.str()); -} - -#define DEBUG_PRINT_VEC(x) DebugPrintVec(#x, x); - -template -void DebugPrintCKArgPtrs( - const CKArgsType& ck_args, const ConvPtr& conv_ptr, ConstData_t x, ConstData_t w, ConstData_t y) -{ - - MIOPEN_LOG_I("CK Instance: " << conv_ptr->GetTypeString()); - MIOPEN_LOG_I("in ptr = " << x); - MIOPEN_LOG_I("w ptr = " << w); - MIOPEN_LOG_I("out ptr = " << y); - - DEBUG_PRINT_VEC(ck_args.input); - DEBUG_PRINT_VEC(ck_args.in_strides); - DEBUG_PRINT_VEC(ck_args.weight); - DEBUG_PRINT_VEC(ck_args.wei_strides); - DEBUG_PRINT_VEC(ck_args.output); - DEBUG_PRINT_VEC(ck_args.out_strides); -} - -inline void DebugPrintConvTensors(const ConvTensors& conv_tensors) -{ - MIOPEN_LOG_I("in ptr = " << conv_tensors.x); - MIOPEN_LOG_I("w ptr = " << conv_tensors.w); - MIOPEN_LOG_I("out ptr = " << conv_tensors.y); - - DEBUG_PRINT_VEC(conv_tensors.xDesc.GetLengths()); - DEBUG_PRINT_VEC(conv_tensors.wDesc.GetLengths()); - DEBUG_PRINT_VEC(conv_tensors.yDesc.GetLengths()); -} - -#undef DEBUG_PRINT_VEC - -#endif // NDEBUG -} // end namespace internal - -// packed size in bytes -inline size_t GetPackedSize(const TensorDescriptor& td) -{ - return td.GetElementSize() * GetTypeSize(td.GetType()); -} - -inline size_t GetCKAlphaBetaWorkspace(const miopen::conv::ProblemDescription& problem) -{ - std::size_t buff_size; - - TensorDescriptor input = problem.GetIn(); - TensorDescriptor output = problem.GetOut(); - ConvolutionDescriptor conv_desc = problem.GetConv(); - - miopenConvolutionABBackwardWeightsGetWorkSpaceSize( - problem.GetAlphaBetaCase(), &input, &output, &conv_desc, &buff_size); - return buff_size; -} - -inline bool CKWrwRequireWorkspace( - size_t G, size_t C, size_t K, miopenDataType_t data_type, miopenAlphaBetaCase_t alpha_beta_case) -{ - auto is_odd = [](size_t num) { return num % 2 != 0; }; - size_t C_per_group = C / G; - size_t K_per_group = K / G; - - return (alpha_beta_case == BILINEAR || alpha_beta_case == SCALE) || - (data_type == miopenHalf && (is_odd(C_per_group) || is_odd(K_per_group))); -} - -/// \todo move to a cpp file -inline size_t GetWorkspaceSizeLayoutTransformConv(const miopen::conv::ProblemDescription& problem) -{ - if(problem.IsLayoutNHWC()) - { - if(problem.GetDirection() == ::miopen::conv::Direction::BackwardWeights) - { - return GetCKAlphaBetaWorkspace(problem); - } - return 0; - } - - assert(problem.IsLayoutDefault()); - - if(problem.GetDirection() == ::miopen::conv::Direction::BackwardWeights) - { - MultiBufferWorkspaceTraits wt({GetPackedSize(problem.GetIn()), - GetPackedSize(problem.GetWeights()), - GetPackedSize(problem.GetOut()), - GetCKAlphaBetaWorkspace(problem)}); - return wt.GetSize(); - } - - MultiBufferWorkspaceTraits wt({GetPackedSize(problem.GetIn()), - GetPackedSize(problem.GetWeights()), - GetPackedSize(problem.GetOut())}); - return wt.GetSize(); -} - -template -ConvSolution InitInvokerFactoryNCHW(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& kernel_id, - const Input1TposeOp& input1_op, - const Input2TposeOp& input2_op, - const OutputTposeOp& output_op) -{ - assert(problem.IsLayoutDefault()); - - ConvSolution result; -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - auto ck_args = CKArgsType{problem}; - - auto conv_ptrs = DeviceOpType::GetInstances(); - - std::optional split_k = std::nullopt; - std::string id_string = kernel_id; - auto pos = kernel_id.find_last_of('+'); - if(pos != std::string::npos) - { - split_k = std::stoi(kernel_id.substr(pos + 1)); - id_string = kernel_id.substr(0, pos); - } - - std::optional _ck_buff_des; - - if(problem.IsDirectionBackwardWrW()) - { - _ck_buff_des.emplace(GetCKAlphaBetaWorkspace(problem), 0); - } - - auto ptr_iter = FindConvPtrByID(conv_ptrs, id_string); - if(ptr_iter == conv_ptrs.end()) - { - MIOPEN_LOG_E("PerformanceConfig kernel '" + kernel_id + "' does not exist."); - return {miopenStatusInvalidValue}; - } - - auto [_input1_tr_inst, _input2_tr_inst, _output_tr_inst, _output_init_tr_inst] = - internal::MakeTaggedTransposeInstances( - result, ctx, problem, ck_args, input1_op, input2_op, output_op, _ck_buff_des); - - result.invoker_factory = [split_k = split_k, - ck_args = std::move(ck_args), - sh_conv_ptr = std::shared_ptr{std::move(*ptr_iter)}, - input1_tr_inst = std::move(_input1_tr_inst), - input2_tr_inst = std::move(_input2_tr_inst), - output_tr_inst = std::move(_output_tr_inst), - output_init_tr_inst = std::move(_output_init_tr_inst), - ck_buff_des = - _ck_buff_des](const std::vector& kernels) mutable { - return [split_k = split_k, - kernels, - ck_args = std::move(ck_args), - sh_conv_ptr = std::move(sh_conv_ptr), - input1_tr_inst = std::move(input1_tr_inst), - input2_tr_inst = std::move(input2_tr_inst), - output_tr_inst = std::move(output_tr_inst), - output_init_tr_inst = std::move(output_init_tr_inst), - ck_buff_des = ck_buff_des](const Handle& handle, - const AnyInvokeParams& primitive_parameters) mutable { - handle.ResetKernelTime(); - - const auto& data_ctx = primitive_parameters.CastTo(); - - if(!data_ctx.workSpace) - { - MIOPEN_THROW(miopenStatusInvalidValue, "workspace pointer is null"); - } - - input1_tr_inst.AssignBuffer(handle, data_ctx.workSpace); - input2_tr_inst.AssignBuffer(handle, data_ctx.workSpace); - output_tr_inst.AssignBuffer(handle, data_ctx.workSpace); - output_init_tr_inst.AssignBuffer(handle, data_ctx.workSpace); - - // conversion operator applied here to convert to ConvTensors - auto conv_tensors = ConvTensors(data_ctx.tensors); - - /// \todo remove this when DataInvokeParams stops swapping - // "in" and "out" tensors for backward pass - if(output_tr_inst.GetConvOperandTag() == internal::ConvOperandTag::Input) - { - // this is backward pass, swap back input and output - std::swap(conv_tensors.x, conv_tensors.y); - std::swap(conv_tensors.xDesc, conv_tensors.yDesc); - } - WorkAroundHipEventProfiler pfr(handle); - input1_tr_inst.ConvertFrom(handle, kernels, conv_tensors); - - input2_tr_inst.ConvertFrom(handle, kernels, conv_tensors); - - output_init_tr_inst.ConvertFrom(handle, kernels, conv_tensors); - - /// \todo: Will need SetTensor() to properly zero out non-packed tensors - if(output_tr_inst.GetConvOperandTag() == internal::ConvOperandTag::Weights) - { - output_tr_inst.ZeroOutBuffer(handle); - } - - std::array tr_ptrs = { - &input1_tr_inst, &input2_tr_inst, &output_tr_inst}; - - // sort by tag in order: Input, Weights, Output - std::sort(tr_ptrs.begin(), tr_ptrs.end(), [](const auto& left, const auto& right) { - return left->GetConvOperandTagAsInt() < right->GetConvOperandTagAsInt(); - }); - - auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); - std::unique_ptr argument_ptr; - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) - { - if(split_k.has_value()) - { - argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, - tr_ptrs[0]->GetBufferPtr(), - tr_ptrs[1]->GetBufferPtr(), - tr_ptrs[2]->GetBufferPtr(), - data_ctx.alpha.GetAsFloat(), - data_ctx.beta.GetAsFloat(), - split_k.value()); - } - } - else - { - std::ignore = split_k; - argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, - tr_ptrs[0]->GetBufferPtr(), - tr_ptrs[1]->GetBufferPtr(), - tr_ptrs[2]->GetBufferPtr(), - data_ctx.alpha.GetAsFloat(), - data_ctx.beta.GetAsFloat()); - } - - if(ck_buff_des.has_value() && ck_buff_des->ck_size) - { - auto buf_handle = - handle.CreateSubBuffer(data_ctx.workSpace, ck_buff_des->ck_offset, 0); - assert(buf_handle.get()); - sh_conv_ptr->SetWorkSpacePointer(argument_ptr.get(), buf_handle.get()); - } - invoker_ptr->Run(argument_ptr.get(), {handle.GetStream(), false}); - output_tr_inst.ConvertTo(handle, kernels, conv_tensors); - }; - }; - - result.workspace_sz = GetWorkspaceSizeLayoutTransformConv(problem); -#endif - return result; -} - -template -ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, - const ProblemDescriptionType& problem, - const std::string& kernel_id) -{ - auto conv_ptrs = DeviceOpType::GetInstances(); - - std::optional split_k = std::nullopt; - std::string id_string = kernel_id; - auto pos = kernel_id.find_last_of('+'); - if(pos != std::string::npos) - { - split_k = std::stoi(kernel_id.substr(pos + 1)); - id_string = kernel_id.substr(0, pos); - } - - auto ptr_iter = FindConvPtrByID(conv_ptrs, id_string); - - if(ptr_iter == conv_ptrs.end()) - { - MIOPEN_LOG_E("PerformanceConfig kernel '" + kernel_id + "' does not exist."); - return {miopenStatusInvalidValue}; - } - - if constexpr(std::is_same_v) - { - ConvSolution result; -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - miopenAlphaBetaCase_t alpha_beta_case = problem.GetAlphaBetaCase(); - [[maybe_unused]] bool should_allocated_wrw_buffer = - ShouldAllocateWorkSpaceBufferForWRW(problem); - - result.invoker_factory = [split_k = split_k, - ck_args = CKArgsType{problem}, - alpha_beta_case = alpha_beta_case, - should_allocated_wrw_buffer = should_allocated_wrw_buffer, - sh_conv_ptr = std::shared_ptr{std::move(*ptr_iter)}]( - const std::vector&) mutable { - return [split_k = split_k, - ck_args = std::move(ck_args), - alpha_beta_case = alpha_beta_case, - should_allocated_wrw_buffer = should_allocated_wrw_buffer, - sh_conv_ptr = std::move(sh_conv_ptr)]( - const Handle& handle, const AnyInvokeParams& primitive_parameters) { - const auto& data_ctx = primitive_parameters.CastTo(); - std::unique_ptr argument_ptr; - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) - { - argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, - data_ctx.tensors, - data_ctx.alpha.GetAsFloat(), - data_ctx.beta.GetAsFloat(), - split_k.value()); - } - else - { - std::ignore = split_k; - argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, - data_ctx.tensors, - data_ctx.alpha.GetAsFloat(), - data_ctx.beta.GetAsFloat()); - } - - auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); - HipEventProfiler pfr(handle); - - if(alpha_beta_case == DEFAULT) - { - auto zero = 0.0f; - const auto& tensors = data_ctx.tensors; - SetTensor(handle, tensors.dwDesc, tensors.dw, &zero); - } - // use captured value, other wise getting warning - // "lambda capture is not used" since this variable is only used in assert. - (void)should_allocated_wrw_buffer; - assert((should_allocated_wrw_buffer && data_ctx.workSpace != nullptr) || - !(should_allocated_wrw_buffer && data_ctx.workSpace == nullptr)); - if(data_ctx.workSpace) - { - sh_conv_ptr->SetWorkSpacePointer(argument_ptr.get(), data_ctx.workSpace); - } - invoker_ptr->Run(argument_ptr.get(), {handle.GetStream(), false}); - }; - }; - result.workspace_sz = GetWorkspaceSizeLayoutTransformConv(problem); -#endif - return result; - } - else - { - ConvSolution result; - result.invoker_factory = [ck_args = CKArgsType{problem}, - sh_conv_ptr = std::shared_ptr{std::move(*ptr_iter)}]( - const std::vector&) mutable { - return [ck_args = std::move(ck_args), sh_conv_ptr = std::move(sh_conv_ptr)]( - const Handle& handle, const AnyInvokeParams& primitive_parameters) { - const auto& data_ctx = primitive_parameters.CastTo(); - auto argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, - data_ctx.tensors, - data_ctx.alpha.GetAsFloat(), - data_ctx.beta.GetAsFloat()); - auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); - HipEventProfiler pfr(handle); - invoker_ptr->Run(argument_ptr.get(), {handle.GetStream(), false}); - }; - }; - return result; - } -} - -template -ConvSolution InitInvokerFactoryFwdNCHW(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& kernel_id) -{ - - static_assert(ND == 2 || ND == 3, "Num Dimensions must be 2 or 3"); - - using Input1 = internal::CKTransposeInputOp; - using Input2 = internal::CKTransposeInputOp; - using Output = internal::CKTransposeOutputOp; - - return InitInvokerFactoryNCHW( - ctx, problem, kernel_id, Input1{}, Input2{}, Output{}); -} - -template -ConvSolution InitInvokerFactoryBwdNCHW(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& kernel_id) -{ - - static_assert(ND == 2 || ND == 3, "Num Dimensions must be 2 or 3"); - - using Input1 = internal::CKTransposeInputOp; - using Input2 = internal::CKTransposeInputOp; - using Output = internal::CKTransposeOutputOp; - - return InitInvokerFactoryNCHW( - ctx, problem, kernel_id, Input1{}, Input2{}, Output{}); -} - -template -ConvSolution InitInvokerFactoryWrwNCHW(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const std::string& kernel_id) -{ - static_assert(ND == 2 || ND == 3, "Num Dimensions must be 2 or 3"); - - using Input1 = internal::CKTransposeInputOp; - using Input2 = internal::CKTransposeInputOp; - using Output = internal::CKTransposeOutputOp; - - return InitInvokerFactoryNCHW( - ctx, problem, kernel_id, Input1{}, Input2{}, Output{}); -} - -template -ConvSolution -MakeSolutionGroupConvImplicitGemmXdlops(const miopen::conv::ProblemDescription& problem, - InvokerFactoryMakerNCHW&& invoker_factory_maker_ncdhw, - InvokerFactoryMakerNHWC&& invoker_factory_maker_ndhwc) -{ - -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - if(problem.IsLayoutDefault()) - { - switch(problem.GetInDataType()) - { - case miopenInt8: return invoker_factory_maker_ncdhw(int8_t{}); - case miopenHalf: return invoker_factory_maker_ncdhw(ck::half_t{}); - case miopenFloat: return invoker_factory_maker_ncdhw(float{}); - case miopenBFloat16: return invoker_factory_maker_ncdhw(ck::bhalf_t{}); - case miopenInt64: - case miopenInt32: - case miopenDouble: - case miopenFloat8: - case miopenBFloat8: - default: - MIOPEN_THROW(miopenStatusInternalError, - "3DGroupConvolutionImplicitGemmXdlops operation not implemented for this " - "data type"); - } - } - else if(problem.IsLayoutNHWC()) - { - switch(problem.GetInDataType()) - { - case miopenInt8: return invoker_factory_maker_ndhwc(int8_t{}); - case miopenHalf: return invoker_factory_maker_ndhwc(ck::half_t{}); - case miopenFloat: return invoker_factory_maker_ndhwc(float{}); - case miopenBFloat16: return invoker_factory_maker_ndhwc(ck::bhalf_t{}); - case miopenInt64: - case miopenInt32: - case miopenDouble: - case miopenFloat8: - case miopenBFloat8: - default: - MIOPEN_THROW(miopenStatusInternalError, - "3DGroupConvolutionImplicitGemmXdlops operation not implemented for this " - "data type"); - } - } - else - { - MIOPEN_THROW( - miopenStatusInternalError, - "3DGroupConvolutionImplicitGemmXdlops operation not implemented for this data type"); - } -#else - return {}; -#endif -} - -} // namespace solver -} // namespace miopen diff --git a/src/include/miopen/solver_id.hpp b/src/include/miopen/solver_id.hpp index ab824faa32..e69de29bb2 100644 --- a/src/include/miopen/solver_id.hpp +++ b/src/include/miopen/solver_id.hpp @@ -1,103 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MIOPEN_GUARD_MLOPEN_SOLVER_ID_HPP -#define MIOPEN_GUARD_MLOPEN_SOLVER_ID_HPP - -#include -#include -#include - -#include -#include - -namespace miopen { - -struct ForceInit -{ -}; - -namespace solver { - -struct AnySolver; - -enum class Primitive -{ - Invalid, - Convolution, - Activation, - Batchnorm, - Bias, - Fusion, - Pooling, - Normalization, - Reduce, - Cat, - Mha, - Softmax, - Adam, - Item, - RoPE, - ReLU -}; - -struct MIOPEN_INTERNALS_EXPORT Id -{ - static constexpr uint64_t invalid_value = 0; - - Id() = default; - Id(uint64_t value_); - Id(ForceInit, uint64_t value_); - Id(const std::string& str); - Id(const char* str); - - std::string ToString() const; - AnySolver GetSolver() const; - std::string GetAlgo(conv::Direction dir) const; - miopenConvAlgorithm_t GetAlgo() const; - Primitive GetPrimitive() const; - - bool IsValid() const { return is_valid; } - uint64_t Value() const { return value; } - bool operator==(const Id& other) const - { - if(!is_valid && !other.is_valid) - return true; // invalids are equal regardless of their values - return value == other.value && is_valid == other.is_valid; - } - bool operator!=(const Id& other) const { return !(*this == other); } - -private: - uint64_t value = invalid_value; - bool is_valid = false; -}; - -MIOPEN_INTERNALS_EXPORT const std::vector& GetSolversByPrimitive(Primitive primitive); - -} // namespace solver -} // namespace miopen - -#endif diff --git a/src/include/miopen/tensor.hpp b/src/include/miopen/tensor.hpp index 068e8ae676..e69de29bb2 100644 --- a/src/include/miopen/tensor.hpp +++ b/src/include/miopen/tensor.hpp @@ -1,320 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2017 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_TENSOR_HPP_ -#define GUARD_MIOPEN_TENSOR_HPP_ - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace miopen { - -template -auto tie_impl(T&& x, detail::seq) -> decltype(std::tie(x[Ns]...)) -{ - assert(x.size() >= sizeof...(Ns)); - return std::tie(x[Ns]...); -} - -template -auto tie_impl(T&& x, U y, detail::seq) -> decltype(std::make_tuple(x[Ns]...)) -{ - return std::make_tuple((Ns < x.size() ? x[Ns] : y)...); -} - -template -auto tien(T&& x) MIOPEN_RETURNS(tie_impl(std::forward(x), typename detail::gens::type{})); - -template -auto tien(T&& x, U y) - MIOPEN_RETURNS(tie_impl(std::forward(x), y, typename detail::gens::type{})); - -template -auto tie_pick_impl(T&& x, detail::seq) -{ -#ifndef NDEBUG - each_args([&](auto i) { assert(i < x.size()); }, Ns...); -#endif - return std::tie(x[Ns]...); -} - -template -struct tie_pick -{ - template - auto operator()(T&& x) MIOPEN_RETURNS(tie_pick_impl(std::forward(x), detail::seq{})) -}; - -template -auto create_tuple_impl(F f, detail::seq) -{ - return std::make_tuple(std::forward(f(Ns))...); -} - -template -auto create_tuple(F f) -{ - return create_tuple_impl(f, typename detail::gens::type{}); -} - -inline std::size_t GetTypeSize(miopenDataType_t d) -{ - switch(d) - { - case miopenInt32: - case miopenFloat: return 4; - case miopenHalf: - case miopenBFloat16: return 2; - case miopenInt8: - case miopenFloat8: - case miopenBFloat8: return 1; - case miopenDouble: - case miopenInt64: return 8; - } - MIOPEN_THROW("Unknown or unsupported data type"); -} - -template -std::ptrdiff_t integer_division_ceil(X x, Y y) -{ - std::ptrdiff_t tx = static_cast(x); - std::ptrdiff_t ty = static_cast(y); - - if(ty < 1) - { - MIOPEN_THROW("integer_division_ceil: y < 1"); - } - - return (tx + ty - 1) / ty; -} - -struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor -{ - TensorDescriptor(); - - // This constructor is only used in test/tensor_holder.hpp - // clang-format off - [[deprecated("Use constructor with lengths instead")]] - TensorDescriptor(miopenDataType_t t); - // clang-format on - - // It is preferable to use constructors with lengths and strides with the std::size_t - // data type, because in this format the data is stored inside the class - - // The delegation constructor should be placed above the target constructor in the - // code for better dependency tracking - - TensorDescriptor(miopenDataType_t t, const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, std::vector&& lens_in); - - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::initializer_list& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - std::vector&& lens_in); - - TensorDescriptor(miopenDataType_t t, - const std::vector& lens_in, - const std::vector& strides_in); - TensorDescriptor(miopenDataType_t t, - const std::initializer_list& lens_in, - const std::initializer_list& strides_in); - TensorDescriptor(miopenDataType_t t, - const std::vector& lens_in, - const std::vector& strides_in); - TensorDescriptor(miopenDataType_t t, - std::vector&& lens_in, - std::vector&& strides_in); - - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - const std::vector& lens_in, - const std::vector& strides_in); - TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, - std::vector&& lens_in, - std::vector&& strides_in); - - // Use only for external API - static TensorDescriptor MakeDescriptor(miopenDataType_t t, const int* plens, int size); - static TensorDescriptor MakeDescriptor(miopenDataType_t t, const std::size_t* plens, int size); - static TensorDescriptor - MakeDescriptor(miopenDataType_t t, miopenTensorLayout_t layout, const int* plens, int size); - static TensorDescriptor MakeDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout, - const std::size_t* plens, - int size); - static TensorDescriptor - MakeDescriptor(miopenDataType_t t, const int* plens, const int* pstrides, int size); - static TensorDescriptor MakeDescriptor(miopenDataType_t t, - const std::size_t* plens, - const std::size_t* pstrides, - int size); - - bool IsVectorized() const; - - const std::vector& GetLengths() const; - const std::vector& GetStrides() const; - unsigned GetNumDims() const; - - miopenDataType_t GetType() const; - // clang-format off - [[deprecated("Use GetLayoutEnum() instead")]] - miopenTensorLayout_t GetLayout_t() const; - // clang-format on - const std::optional& GetLayoutEnum() const; - static std::string LayoutEnumToStr(miopenTensorLayout_t layout); - const std::string& GetLayout_str() const; - - std::size_t GetVectorLength() const; - std::optional GetCastType() const; - void SetCastType(miopenDataType_t cast_type_); - - std::size_t GetElementSize() const; - - std::size_t GetElementSpace() const; - - std::size_t GetNumBytes() const; - - std::size_t GetIndex(std::initializer_list l) const; - - template - std::size_t GetIndex(Ts... is) const - { - return this->GetIndex({static_cast(is)...}); - } - - bool IsPacked() const; - bool IsContiguous() const; - /// Checks all lengths and strides. - bool AllDimsFitIntoInt() const; - /// Checks only lengths. - bool AllLengthsFitIntoInt() const; - - bool operator==(const TensorDescriptor& rhs) const; - bool operator!=(const TensorDescriptor& rhs) const; - bool operator<(const TensorDescriptor& rhs) const; - bool operator>(const TensorDescriptor& rhs) const; - - std::string ToString() const; - - // For vectorized layouts storage_layout must be without the ending 'c' - // \todo make private - bool IsPossibleLayout(const std::string& storage_layout, const std::string& layout) const; - // Layout could be NCHW, NHWC, NCDHW, NDHWC, NCHWc, ... - bool IsPossibleLayout4D5D(const std::string& layout) const; - - static std::vector find_permutation(const std::vector& lens, - const std::vector& strides); - - // storage_layout must be NCHW or NCHWc for NCHWc, CHWN or CHWNc for CHWNc, NCHW for other 4D - // layouts, NCDHW for 5D layouts - std::string GetLayout(std::string storage_layout) const; - - friend MIOPEN_INTERNALS_EXPORT std::ostream& operator<<(std::ostream& stream, - const TensorDescriptor& t); - - friend void to_json(nlohmann::json& j, const TensorDescriptor& descriptor); - friend void from_json(const nlohmann::json& j, TensorDescriptor& descriptor); - -private: - TensorDescriptor(miopenDataType_t t, - const std::optional& layout_in, - const std::vector& lens_in, - const std::vector& strides_in, - bool use_strides); - - TensorDescriptor(miopenDataType_t t, - const std::optional& layout_in, - std::vector&& lens_in, - std::vector&& strides_in, - bool use_strides); - - void CheckArgsAndInit(bool use_strides); - - std::vector lens; - std::vector strides; - - bool packed; - std::size_t vector_length = 1; - - miopenDataType_t type = miopenFloat; - std::optional cast_type; - std::optional tensorLayout; - - // For GetLayoutEnum() - mutable std::optional cached_layout_enum; - mutable bool cached_layout_enum_calculated = false; - - // For GetLayout_str() - mutable std::string cached_layout_str; - - // For GetLayout - mutable std::vector cached_permutation; - - // For AllLengthsFitIntoInt() - mutable std::optional cached_lengths_fit_into_int; - // For AllDimsFitIntoInt() - mutable std::optional cached_strides_fit_into_int; -}; - -template -constexpr auto GetNCDHW(unsigned spatial_dims, const std::vector& data) -{ - if(spatial_dims == 3) - return miopen::tien<5>(data, 1); - else - return std::make_tuple(data[0], data[1], static_cast(1), data[2], data[3]); -} - -} // namespace miopen - -MIOPEN_DEFINE_OBJECT(miopenTensorDescriptor, miopen::TensorDescriptor) - -#endif // GUARD_MIOPEN_TENSOR_HPP_ diff --git a/src/include/miopen/tensor_view_utils.hpp b/src/include/miopen/tensor_view_utils.hpp index 1b095affb7..e69de29bb2 100644 --- a/src/include/miopen/tensor_view_utils.hpp +++ b/src/include/miopen/tensor_view_utils.hpp @@ -1,81 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MIOPEN_TENSOR_VIEW_UTIL_HPP_ -#define MIOPEN_TENSOR_VIEW_UTIL_HPP_ - -#include -#include -#include "../../kernels/tensor_view.hpp" - -namespace miopen { - -template -inline tensor_view_t get_inner_expanded_tv(const TensorDescriptor Desc) -{ - auto dims = Desc.GetLengths(); - auto strides = Desc.GetStrides(); - - tensor_view_t tensor_view{}; - for(size_t i = 0; i < N; ++i) - { - if(i < dims.size()) - { - tensor_view.stride[i] = strides[i]; - tensor_view.size[i] = dims[i]; - } - else - { - tensor_view.stride[i] = (i == 0 ? 1 : strides[i - 1]); - tensor_view.size[i] = 1; - } - } - return tensor_view; -} - -template -inline void slice_tv(tensor_view_t& tensor_view, int32_t sliceCount, const int32_t* slices) -{ - for(int32_t i = 0; i < sliceCount; i++) - { - int32_t dim = slices[4 * i + 0]; - int32_t start = slices[4 * i + 1]; - int32_t end = slices[4 * i + 2]; - int32_t step = slices[4 * i + 3]; - - if(end > static_cast(tensor_view.size[dim])) - end = tensor_view.size[dim]; - - auto len = end - start; - - tensor_view.size[dim] = (len + step - 1) / step; - tensor_view.stride[dim] *= step; - } -} - -} // namespace miopen - -#endif // MIOPEN_TENSOR_VIEW_UTIL_HPP_ diff --git a/src/include/miopen/utility/transposing_solver.hpp b/src/include/miopen/utility/transposing_solver.hpp index e7ca3bdc7a..e69de29bb2 100644 --- a/src/include/miopen/utility/transposing_solver.hpp +++ b/src/include/miopen/utility/transposing_solver.hpp @@ -1,602 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#pragma once - -#include -#include -#include - -namespace miopen { -namespace solver { - -template -inline static std::array GetNCDHW(const std::vector& values) -{ - const auto cast = [](auto v) { return static_cast(v); }; - std::size_t n = 1, c = 1, d = 1, h = 1, w = 1; - - switch(values.size()) - { - case 5: std::tie(n, c, d, h, w) = tien<5>(values); break; - case 4: std::tie(n, c, h, w) = tien<4>(values); break; - default: MIOPEN_THROW(miopenStatusBadParm); - } - - return {cast(n), cast(c), cast(d), cast(h), cast(w)}; -} - -struct TransposeProblem -{ - TensorDescriptor input; - const char* layout; -}; - -using OldStyleTransposeProblem = std::tuple; - -struct TransposeInvokeParams : InvokeParams -{ - ConstData_t in; - Data_t out; - TensorDescriptor in_desc; - TensorDescriptor out_desc; - - TransposeInvokeParams(ConstData_t in_, - Data_t out_, - TensorDescriptor in_desc_, - TensorDescriptor out_desc_) - : in(in_), out(out_), in_desc(in_desc_), out_desc(out_desc_) - { - } - - std::size_t GetWorkspaceSize() const { return 0; } - Data_t GetWorkspace() const { return nullptr; } -}; - -struct TransposePseudoSolver -{ - virtual ~TransposePseudoSolver() = default; - virtual std::string GetTranspose() const = 0; - virtual ConvSolution GetSolution(const ExecutionContext& ctx, - const TransposeProblem& problem) const = 0; - -protected: - TransposePseudoSolver() = default; - TransposePseudoSolver(const TransposePseudoSolver&) = default; -}; - -template -struct AnyImplementation -{ - AnyImplementation() : buffer(), copy(nullptr), p(nullptr) {} - - template - AnyImplementation(const Implementation& impl) - { - static_assert(sizeof(Implementation) == sizeof(Interface), - "Implementation must be stateless"); - static_assert(std::is_base_of{}, - "Not derived class of the interface"); - copy = +[](const Storage& src, Storage& dst, Interface** interface) { - new(std::addressof(dst)) Implementation(*StorageCast(src)); - *interface = static_cast(StorageCast(dst)); - }; - - new(std::addressof(buffer)) Implementation(impl); - p = static_cast(StorageCast(buffer)); - } - - AnyImplementation(const Derived& rhs) : buffer(), copy(rhs.copy), p(nullptr) - { - copy(rhs.buffer, buffer, &p); - } - - AnyImplementation& operator=(const AnyImplementation& rhs) - { - if(&rhs != this) - { - if(p) - p->~Interface(); - copy(rhs.buffer, buffer, &p); - } - return *this; - } - - const Interface* get() const noexcept { return p; } - const Interface& operator*() const noexcept { return *get(); } - const Interface* operator->() const noexcept { return get(); } - - ~AnyImplementation() noexcept - { - if(p) - p->~Interface(); - } - -private: - using Storage = std::aligned_storage_t; - using Cloner = void (*)(const Storage&, Storage&, Interface**); - - template - static T* StorageCast(S&& s) - { - return reinterpret_cast(std::addressof(s)); - } - - Storage buffer; - Cloner copy; - Interface* p; -}; - -struct AnyTransposePseudoSolver : AnyImplementation -{ - AnyTransposePseudoSolver() = default; - - template - AnyTransposePseudoSolver(const Transpose& s) - : AnyImplementation(s) - { - } - - AnyTransposePseudoSolver(const AnyTransposePseudoSolver& rhs) - : AnyImplementation(rhs) - { - } -}; - -struct UniversalTransposeSolver : TransposePseudoSolver -{ - std::string GetTranspose() const override { return "*-*"; } - - ConvSolution GetSolution(const ExecutionContext& ctx, - const TransposeProblem& problem) const override - { - auto sln = ConvSolution{}; - - { - auto transposeKernel = KernelInfo{}; - - const auto tensor_space = problem.input.GetElementSize(); - const auto cus = ctx.GetStream().GetMaxComputeUnits(); - const auto group_size = std::min(tensor_space, cus); - - const auto build_params = GetDataTypeKBP(problem.input.GetType()); - - transposeKernel.kernel_file = "UniversalTranspose.cl"; - transposeKernel.kernel_name = "UniversalTranspose"; - transposeKernel.g_wk = {group_size * 16, 1, 1}; - transposeKernel.l_wk = {group_size * 16, 1, 1}; - transposeKernel.comp_options = build_params.GenerateFor(kbp::OpenCL{}); - - sln.construction_params.emplace_back(std::move(transposeKernel)); - } - - sln.invoker_factory = [](const std::vector& kernels) { - const auto kernel = kernels.front(); - return [kernel](const Handle& handle, const AnyInvokeParams& any_params) { - const auto& params = any_params.CastTo(); - const auto& lens = GetNCDHW(params.in_desc.GetLengths()); - const auto& in_strides = GetNCDHW(params.in_desc.GetStrides()); - const auto& out_strides = GetNCDHW(params.out_desc.GetStrides()); - - // clang-format off - handle.Run(kernel)( - params.in, params.out, - lens[0], lens[1], lens[2], lens[3], lens[4], - in_strides[0], in_strides[1], in_strides[2], in_strides[3], in_strides[4], - out_strides[0], out_strides[1], out_strides[2], out_strides[3], out_strides[4] - ); - // clang-format on - }; - }; - - return sln; - } -}; - -class SegmentedGpuBuffer -{ -public: - SegmentedGpuBuffer(const Handle& handle_, Data_t memory_, std::size_t offset_ = 0) - : handle(&handle_), memory(memory_), offset(offset_) - { - assert(handle); - } - - SegmentedGpuBuffer(SegmentedGpuBuffer&) = delete; - SegmentedGpuBuffer(SegmentedGpuBuffer&&) = delete; - SegmentedGpuBuffer& operator=(SegmentedGpuBuffer&) = delete; - SegmentedGpuBuffer& operator=(SegmentedGpuBuffer&&) = delete; - - miopen::shared operator()(std::size_t size) - { - const auto align = GetSubbufferAlignment(handle); - offset += (align - offset) % align; - - auto subbuffer = handle->CreateSubBuffer(memory, offset, size); - offset += size; - - return subbuffer; - } - -private: - const Handle* handle; - Data_t memory; - std::size_t offset; -}; - -inline std::string SyncLayoutDims(const char* from, const char* to) -{ - if(strlen(from) < 5) - return ReplaceString(to, "D", ""); - return to; -} - -template -struct ProblemTensorTransposeDescriptor -{ - using DescriptorGetter = TensorDescriptor& (Problem::*)(); - using ConstDescriptorGetter = const TensorDescriptor& (Problem::*)() const; - - DescriptorGetter descriptor; - ConstDescriptorGetter cdescriptor; - TensorDescriptor InvokeParams::*rt_descriptor; - - union - { - ConstData_t InvokeParams::*as_input; - Data_t InvokeParams::*as_output; - }; - - const char* to; - bool is_input; - - template // to deal with constParameter invalid warning - inline void Transpose(const Problem& src, Problem_& dest) const - { - const auto& desc_from = (src.*cdescriptor)(); - auto& desc_to = (dest.*descriptor)(); - desc_to = Transpose(desc_from); - } - - inline void Transpose(const InvokeParams& src, InvokeParams& dest) const - { - const auto& desc_from = src.*rt_descriptor; - auto& desc_to = dest.*rt_descriptor; - desc_to = Transpose(desc_from); - } - - inline TensorDescriptor Transpose(const TensorDescriptor& in) const - { - const auto labels = tensor_layout_get_default(in.GetNumDims()); - auto derived_strides = std::vector{}; - tensor_layout_to_strides( - in.GetLengths(), labels, SyncLayoutDims(labels.c_str(), to), derived_strides); - return {in.GetType(), in.GetLengths(), derived_strides}; - } -}; - -class ProblemTensorTransposeInvoke -{ -public: - template - ProblemTensorTransposeInvoke( - SegmentedGpuBuffer& allocator, - const ProblemTensorTransposeDescriptor& descriptor, - const Invoker& invoker_, - const InvokeParams& invoke_params, - InvokeParams& transposed_params) - : invoker(invoker_) - { - // Transpose runtime tensor descriptor - descriptor.Transpose(invoke_params, transposed_params); - - const auto& orig_descriptor = invoke_params.*(descriptor.rt_descriptor); - const auto& transposed_descriptor = transposed_params.*(descriptor.rt_descriptor); - - // Allocate subbuffer in the workspace - const auto e_size = get_data_size(transposed_descriptor.GetType()); - const auto buffer_size = transposed_descriptor.GetElementSpace() * e_size; - buffer = allocator(buffer_size); - - if(descriptor.is_input) - transposed_params.*(descriptor.as_input) = buffer.get(); - else - transposed_params.*(descriptor.as_output) = buffer.get(); - - if(!descriptor.is_input) - { - // Prepare output transpose invoker - const auto& out = invoke_params.*(descriptor.as_output); - - transpose_params = - TransposeInvokeParams{buffer.get(), out, transposed_descriptor, orig_descriptor}; - return; - } - - // Transpose input tensor - const auto& in = invoke_params.*(descriptor.as_input); - - transpose_params = - TransposeInvokeParams{in, buffer.get(), orig_descriptor, transposed_descriptor}; - } - - void operator()(const Handle& handle) const - { - const auto time = handle.GetKernelTime(); - invoker(handle, transpose_params); - handle.AccumKernelTime(time); - } - -private: - miopen::shared buffer; - Invoker invoker; - AnyInvokeParams transpose_params; -}; - -class ProblemTensorTransposeGroup -{ -public: - template - ProblemTensorTransposeGroup( - const Handle& handle_, - SegmentedGpuBuffer& allocator, - const std::vector< - std::tuple, Invoker>>& inputs_, - const std::vector< - std::tuple, Invoker>>& outputs_, - const InvokeParams& invoke_params, - InvokeParams& transposed_params) - : handle(&handle_) - { - std::transform( - inputs_.begin(), inputs_.end(), std::back_inserter(inputs), [&](auto&& params) { - return ProblemTensorTransposeInvoke(allocator, - std::get<0>(params), - std::get<1>(params), - invoke_params, - transposed_params); - }); - - std::transform( - outputs_.begin(), outputs_.end(), std::back_inserter(outputs), [&](auto&& params) { - return ProblemTensorTransposeInvoke(allocator, - std::get<0>(params), - std::get<1>(params), - invoke_params, - transposed_params); - }); - - MIOPEN_LOG_I2("Executing the input transpose"); - for(const auto& transpose : inputs) - transpose(*handle); - } - - ProblemTensorTransposeGroup(ProblemTensorTransposeGroup&) = delete; - ProblemTensorTransposeGroup(ProblemTensorTransposeGroup&&) = delete; - ProblemTensorTransposeGroup& operator=(ProblemTensorTransposeGroup&) = delete; - ProblemTensorTransposeGroup& operator=(ProblemTensorTransposeGroup&&) = delete; - - ~ProblemTensorTransposeGroup() - { - MIOPEN_LOG_I2("Executing the output transpose"); - for(const auto& transpose : outputs) - transpose(*handle); - } - -private: - const Handle* handle; - std::vector inputs; - std::vector outputs; -}; - -template -struct TransposingSolver : Base -{ - using TransposeDescriptor = ProblemTensorTransposeDescriptor; - - static std::vector GetTransposeSolvers() - { - return {UniversalTransposeSolver{}}; - } - - static std::unordered_map GetTransposeSolversMap() - { - auto ret = std::unordered_map{}; - for(const auto& transpose : Derived::GetTransposeSolvers()) - ret.emplace(transpose->GetTranspose(), std::move(transpose)); - return ret; - } - - bool IsApplicable(const ExecutionContext& ctx, const Problem& problem) const override - { - const auto transpose_solvers = Derived::GetTransposeSolversMap(); - const auto skip_transpose_check = transpose_solvers.find("*-*") != transpose_solvers.end(); - auto any_difference = false; - - for(auto transpose : Derived::GetTransposes()) - { - decltype(auto) descriptor = (problem.*(transpose.cdescriptor))(); - const auto layout = descriptor.GetLayout_str(); - const auto to = SyncLayoutDims(layout.c_str(), transpose.to); - - auto specific_pair = layout + "-"; - specific_pair.append(to); - - if(!skip_transpose_check && - transpose_solvers.find(specific_pair) == transpose_solvers.end() && - transpose_solvers.find(layout + "-*") == transpose_solvers.end() && - transpose_solvers.find(std::string("*-") + to) == transpose_solvers.end()) - return false; - - any_difference |= layout != to; - } - - return any_difference && Inner{}.IsApplicable(ctx, Transpose(problem)); - } - - std::size_t GetWorkspaceSize(const ExecutionContext& ctx, const Problem& problem) const override - { - const auto transposed_problem = Transpose(problem); - auto ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); - - for(const auto& transpose : Derived::GetTransposes()) - { - const auto& descriptor = (transposed_problem.*(transpose.cdescriptor))(); - ws_size += descriptor.GetElementSpace(); - } - - return ws_size; - } - - ConvSolution GetSolution(const ExecutionContext& ctx, const Problem& problem) const override - { - auto transposed_problem = Transpose(problem); - ConvSolution sln = Inner{}.GetSolution(ctx, transposed_problem); - auto old_factory = *sln.invoker_factory; - const auto old_kernels_end = sln.construction_params.size(); - const auto transpose_solvers = Derived::GetTransposeSolversMap(); - - std::vector> in_transpose_ifs, - out_transpose_ifs; - - for(auto transpose : Derived::GetTransposes()) - { - const auto& descriptor = (problem.*(transpose.cdescriptor))(); - const auto layout = descriptor.GetLayout_str(); - const auto to = SyncLayoutDims(layout.c_str(), transpose.to); - - if(layout == to) - continue; - - auto specific_pair = layout + "-"; - specific_pair.append(to); - - auto transpose_solver = transpose_solvers.find(specific_pair); - if(transpose_solver == transpose_solvers.end()) - { - transpose_solver = transpose_solvers.find(layout + "-*"); - if(transpose_solver == transpose_solvers.end()) - { - transpose_solver = transpose_solvers.find(std::string("*-") + to); - if(transpose_solver == transpose_solvers.end()) - transpose_solver = transpose_solvers.find("*-*"); - assert(transpose_solver != transpose_solvers.end()); - } - } - - const auto transpose_problem = TransposeProblem{descriptor, layout.c_str()}; - auto transpose_sln = transpose_solver->second->GetSolution(ctx, transpose_problem); - - const auto kernels_begin = sln.construction_params.size(); - sln.construction_params.insert(sln.construction_params.end(), - transpose_sln.construction_params.begin(), - transpose_sln.construction_params.end()); - const auto kernels_end = sln.construction_params.size(); - const auto raw_invoker_factory = transpose_sln.invoker_factory; - - auto transpose_invoker_factory = [kernels_begin, kernels_end, raw_invoker_factory]( - const std::vector& kernels) { - auto segment = std::vector{}; - segment.reserve(kernels_end - kernels_begin); - for(auto i = kernels_begin; i < kernels_end; ++i) - segment.push_back(kernels[i]); - return (*raw_invoker_factory)(segment); - }; - - (transpose.is_input ? in_transpose_ifs : out_transpose_ifs) - .emplace_back(transpose, std::move(transpose_invoker_factory)); - } - - if(in_transpose_ifs.size() + out_transpose_ifs.size() == 0) - return sln; - - const auto ws_size = Inner{}.GetWorkspaceSize(ctx, transposed_problem); - - sln.invoker_factory = - [old_factory, old_kernels_end, ws_size, in_transpose_ifs, out_transpose_ifs]( - const std::vector& kernels) { - const auto inner_kernels = - std::vector{kernels.begin(), kernels.begin() + old_kernels_end}; - std::vector> in_transpose_invokers, - out_transpose_invokers; - - std::transform(in_transpose_ifs.begin(), - in_transpose_ifs.end(), - std::back_inserter(in_transpose_invokers), - [&](const auto& params) { - return std::make_tuple(std::get<0>(params), - std::get<1>(params)(kernels)); - }); - - std::transform(out_transpose_ifs.begin(), - out_transpose_ifs.end(), - std::back_inserter(out_transpose_invokers), - [&](const auto& params) { - return std::make_tuple(std::get<0>(params), - std::get<1>(params)(kernels)); - }); - - auto invoker = old_factory(inner_kernels); - - return [invoker, in_transpose_invokers, out_transpose_invokers, ws_size]( - const Handle& handle, const AnyInvokeParams& any_params) { - const auto& invoke_params = any_params.CastTo(); - auto transposed_params = invoke_params; - - handle.ResetKernelTime(); - - SegmentedGpuBuffer allocator{handle, invoke_params.workspace, ws_size}; - - ProblemTensorTransposeGroup transposeGroup{handle, - allocator, - in_transpose_invokers, - out_transpose_invokers, - invoke_params, - transposed_params}; - - // Execute the invoker provided by the inner solver - MIOPEN_LOG_I2("Executing the inner solver invoker"); - const auto time = handle.GetKernelTime(); - invoker(handle, transposed_params); - handle.AccumKernelTime(time); - }; - }; - - return sln; - } - -private: - inline static Problem Transpose(const Problem& problem) - { - auto transposed_problem = problem; - for(const auto& transpose : Derived::GetTransposes()) - transpose.Transpose(problem, transposed_problem); - return transposed_problem; - } -}; - -} // namespace solver -} // namespace miopen diff --git a/src/kernels/MIOpenBatchNormActivInferHIP.cpp b/src/kernels/MIOpenBatchNormActivInferHIP.cpp index d9b269d950..e69de29bb2 100644 --- a/src/kernels/MIOpenBatchNormActivInferHIP.cpp +++ b/src/kernels/MIOpenBatchNormActivInferHIP.cpp @@ -1,177 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS -#include -#include -#endif - -#include "activation_functions.hpp" - -extern "C" __global__ void __launch_bounds__(MIO_BN_GRP0* MIO_BN_GRP1* MIO_BN_GRP2) - MIOpenBatchNormActivInferSpatialEstHIP(const FP_TYPE_PREC alpha, - const FP_TYPE_PREC beta, - const FP_TYPE_PREC gamma, - const double epsilon, - const FP_TYPE* __restrict in, - FP_TYPE* __restrict out, - const FP_TYPE_PREC* __restrict bias, - const FP_TYPE_PREC* __restrict scale, - const FP_TYPE_PREC* __restrict estimatedMean, - const FP_TYPE_PREC* __restrict estimatedVariance) -{ - unsigned int tidx = blockIdx.x * blockDim.x + threadIdx.x; - // HIP runtime does not support launching non-uniform blocks - // So extra threads are launched to handle this. - if(tidx >= MIOPEN_SBN_BOUNDS) - return; - - unsigned int tidy = blockIdx.y * blockDim.y + threadIdx.y; - - unsigned int c_i = tidy; - unsigned int hw_i = tidx; - unsigned int c_offset = c_i * MIO_BN_HW; - - // Load the mean, variance, scale, and bias that is broadcast across the block - const FP_TYPE_PREC pmean = estimatedMean[c_i]; - const FP_TYPE_PREC pvar = estimatedVariance[c_i]; - const FP_TYPE_PREC pscale = scale[c_i]; - const FP_TYPE_PREC pbias = bias[c_i]; - const FP_TYPE_PREC invVariance = rsqrt(pvar + epsilon); - // Load the input data. This is done in a vectorized manner - FP_TYPE data[MIOPEN_READ_UNIT]; - -#pragma unroll 2 - for(unsigned int n_i = 0; n_i < MIO_BN_N; n_i++) - { - const unsigned int index = n_i * MIO_BN_CHW + c_offset + hw_i * MIOPEN_READ_UNIT; - // Perform a vectorized load of the input data - *(reinterpret_cast(data)) = - *(reinterpret_cast(in + index)); - FP_TYPE_PREC bnRes[MIOPEN_READ_UNIT]; - FP_TYPE_PREC actRes[MIOPEN_READ_UNIT]; -#pragma unroll - for(unsigned int i = 0; i < MIOPEN_READ_UNIT; ++i) - { - bnRes[i] = - fma(pscale, (static_cast(data[i]) - pmean) * invVariance, pbias); - } - ActivationFunction(actRes, bnRes, gamma, beta, alpha); - if constexpr(MIOPEN_USE_FP16) - { -#pragma unroll - for(unsigned int i = 0; i < MIOPEN_READ_UNIT; i++) - { - out[index + i] = static_cast(actRes[i]); - } - } - else - { - // perform a vectorized store of the output data as FP_TYPE and PF_TYPE_PREC are same - // for FP32 - *(reinterpret_cast(out + index)) = - *(reinterpret_cast(actRes)); - } - } -} - -extern "C" __global__ void __launch_bounds__(MIO_BN_GRP0* MIO_BN_GRP1* MIO_BN_GRP2) - MIOpenBatchNormActivInferPerActEstHIP(const FP_TYPE_PREC alpha, - const FP_TYPE_PREC beta, - const FP_TYPE_PREC gamma, - const double epsilon, - const FP_TYPE* __restrict in, - FP_TYPE* __restrict out, - const FP_TYPE_PREC* __restrict bias, - const FP_TYPE_PREC* __restrict scale, - const FP_TYPE_PREC* __restrict estimatedMean, - const FP_TYPE_PREC* __restrict estimatedVariance) -{ - unsigned int tidx = blockIdx.x * blockDim.x + threadIdx.x; - // HIP runtime does not support launching non-uniform blocks - // So extra threads are launched to handle this. - if(tidx >= MIOPEN_SBN_BOUNDS) - return; - - unsigned int chw_i = tidx * MIOPEN_READ_UNIT; - - FP_TYPE_PREC pmean[MIOPEN_READ_UNIT]; - FP_TYPE_PREC pvar[MIOPEN_READ_UNIT]; - FP_TYPE_PREC pscale[MIOPEN_READ_UNIT]; - FP_TYPE_PREC pbias[MIOPEN_READ_UNIT]; - - // Perform a vectorized load of the mean, variance, scale, and bias - *(reinterpret_cast(pmean)) = - *(reinterpret_cast(estimatedMean + chw_i)); - *(reinterpret_cast(pvar)) = - *(reinterpret_cast(estimatedVariance + chw_i)); - *(reinterpret_cast(pscale)) = - *(reinterpret_cast(scale + chw_i)); - *(reinterpret_cast(pbias)) = - *(reinterpret_cast(bias + chw_i)); - - FP_TYPE data[MIOPEN_READ_UNIT]; - FP_TYPE_PREC invVariance[MIOPEN_READ_UNIT]; - -#pragma unroll - for(unsigned int i = 0; i < MIOPEN_READ_UNIT; i++) - invVariance[i] = rsqrt(pvar[i] + epsilon); - -#pragma unroll 2 - for(unsigned int n_i = 0; n_i < MIO_BN_N; n_i++) - { - const unsigned int index = n_i * MIO_BN_CHW + chw_i; - // Perform a vectorized load of the input data - *(reinterpret_cast(data)) = - *(reinterpret_cast(in + index)); - FP_TYPE_PREC bnRes[MIOPEN_READ_UNIT]; - FP_TYPE_PREC actRes[MIOPEN_READ_UNIT]; -#pragma unroll - for(unsigned int i = 0; i < MIOPEN_READ_UNIT; ++i) - { - bnRes[i] = fma(pscale[i], - (static_cast(data[i]) - pmean[i]) * invVariance[i], - pbias[i]); - } - - ActivationFunction(actRes, bnRes, gamma, beta, alpha); - if constexpr(MIOPEN_USE_FP16) - { // In this situation, FP_TYPE_PREC is FP32 whereas FP_TYPE is FP16 - // So, we cannot perform a vectorized store -#pragma unroll - for(unsigned int i = 0; i < MIOPEN_READ_UNIT; i++) - { - out[index + i] = static_cast(actRes[i]); - } - } - else - { - // perform a vectorized store of the output data as FP_TYPE and PF_TYPE_PREC are same - *(reinterpret_cast(out + index)) = - *(reinterpret_cast(actRes)); - } - } -} diff --git a/src/kernels/MIOpenCumulativeReduction.cpp b/src/kernels/MIOpenCumulativeReduction.cpp new file mode 100644 index 0000000000..6a81f94598 --- /dev/null +++ b/src/kernels/MIOpenCumulativeReduction.cpp @@ -0,0 +1,143 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS +#include +#include +#endif + +#include "float_types.h" +#include "MIOpenCumulativeReduction.hpp" + +template +__device__ inline void CumulativeReductionScan(const bool& reverse, + const int& lid, + FLOAT_ACCUM* __restrict__ a, + Ts* __restrict__... b) +{ + // reduction + int stride = 1; + while(stride <= LOCAL_SIZE) + { + int idx = (lid + 1) * stride * 2 - 1; + if(idx < LOCAL_SIZE) + reduce_func{}.calculate( + !reverse, a[idx], a[idx - stride], b[idx]..., b[idx - stride]...); + stride *= 2; + __syncthreads(); + } + + // post scan + stride = LOCAL_SIZE / 2; + while(stride > 0) + { + int idx = (lid + 1) * stride * 2 - 1; + if((idx + stride) < LOCAL_SIZE) + reduce_func{}.calculate( + !reverse, a[idx + stride], a[idx], b[idx + stride]..., b[idx]...); + stride /= 2; + __syncthreads(); + } +} + +template +__device__ void CumulativeReductionForwardContiguousLastDim(const TI* __restrict__ input, + TO* __restrict__ output, + int* __restrict__ indices, + const uint64_t reduce_size, + const bool exclusive, + const bool reverse) +{ + /* + * input = packed tensor with stride[last_dim]=1, output: the same as input, indices: the same +as input + * reduce_size = input.size[last_dim] + * exclusive: TRUE to exclude input[i] when calculate output[i] + * reverse: reverse the operating order + * + * cumulative dimension = last dim + * blockSize = {1, LOCAL_SIZE} + * gridSize = {Number of input elements / input.size[last_dim], input.size[last_dim]} + */ + + __shared__ FLOAT_ACCUM otmp[LOCAL_SIZE]; + int* itmp = nullptr; + if(indices) + { + __shared__ int _itmp[LOCAL_SIZE]; + itmp = _itmp; + } + + int lid = threadIdx.y; + + auto xid = blockIdx.x * blockDim.x + threadIdx.x; + auto yid = blockIdx.y * blockDim.y + threadIdx.y; + + int idx = yid - exclusive; + if(0 <= idx && idx < reduce_size - exclusive) + { + idx = (reverse ? reduce_size - idx - 1 : idx); + otmp[lid] = CVT_FLOAT2ACCUM(input[xid * reduce_size + idx]); + if(indices) + itmp[lid] = idx; + } + else + { + otmp[lid] = reduce_func{}.START_VAL; + if(indices) + itmp[lid] = (reverse ? reduce_size - idx - 1 : idx); + } + __syncthreads(); + + if(indices) + CumulativeReductionScan(reverse, lid, otmp, itmp); + else + CumulativeReductionScan(reverse, lid, otmp); + + idx = yid; + if(idx < reduce_size) + { + idx = (reverse ? reduce_size - idx - 1 : idx); + if(output) + output[xid * reduce_size + idx] = CVT_ACCUM2FLOAT(otmp[lid]); + if(indices) + indices[xid * reduce_size + idx] = itmp[lid]; + } +} + +extern "C" __global__ void CumulativeReductionForwardContiguousLastDim(const INPUT_TYPE* input, + OUTPUT_TYPE* output, + int* indices, + const uint64_t reduce_size, + const bool exclusive, + const bool reverse) +{ + // instantiate the kernel + CumulativeReductionForwardContiguousLastDim( + input, output, indices, reduce_size, exclusive, reverse); +} diff --git a/src/kernels/MIOpenCumulativeReduction.hpp b/src/kernels/MIOpenCumulativeReduction.hpp new file mode 100644 index 0000000000..7323642e8d --- /dev/null +++ b/src/kernels/MIOpenCumulativeReduction.hpp @@ -0,0 +1,116 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef GUARD_KERNELS_MIOPEN_CUMULATIVE_REDUCTIONS_HPP +#define GUARD_KERNELS_MIOPEN_CUMULATIVE_REDUCTIONS_HPP + +#include "float_types.h" + +enum class CumulativeReductionOp_t +{ + Max = 1, + Min = 2, + Sum = 3, + Prod = 4, +}; + +#ifndef __HIP_DEVICE_COMPILE__ +static_assert(MIOPEN_CUM_MAX == static_cast(CumulativeReductionOp_t::Max)); +static_assert(MIOPEN_CUM_MIN == static_cast(CumulativeReductionOp_t::Min)); +static_assert(MIOPEN_CUM_SUM == static_cast(CumulativeReductionOp_t::Sum)); +static_assert(MIOPEN_CUM_PROD == static_cast(CumulativeReductionOp_t::Prod)); +#endif + +inline constexpr void update() {} +template +inline constexpr void update(T& a, T b, Ts&... c, Ts... d) +{ + a = b; + update(c..., d...); +} + +inline constexpr bool isgreater() { return false; } +template +inline constexpr bool isgreater(T& a, T b, Ts&... c, Ts... d) +{ + if(a != b) + return a > b; + return isgreater(c..., d...); +} + +template +struct reduce_func_base +{ + inline constexpr bool isbetter(const T& /*a*/, const T& /*b*/) { return false; } + inline constexpr void combine(T& a, T b) { a = b; } + inline constexpr void calculate(const bool keep_greater, T& a, T b, Ts&... c, Ts... d) + { + auto derived = static_cast(this); + if(!derived->isbetter(a, b)) + { + if(derived->isbetter(a, b) != derived->isbetter(b, a) || + isgreater(c..., d...) != keep_greater) + update(c..., d...); + derived->combine(a, b); + } + } +}; + +template +struct reduce_func; + +template +struct reduce_func + : reduce_func_base, T, Ts...> +{ + const FLOAT_ACCUM START_VAL = -MAX_VAL_ACCUM; + inline constexpr bool isbetter(const T& a, const T& b) { return a > b; } +}; + +template +struct reduce_func + : reduce_func_base, T, Ts...> +{ + const FLOAT_ACCUM START_VAL = MAX_VAL_ACCUM; + inline constexpr bool isbetter(const T& a, const T& b) { return a < b; } +}; + +template +struct reduce_func + : reduce_func_base, T, Ts...> +{ + const FLOAT_ACCUM START_VAL = CVT_FP32_2ACCUM(0.0f); + inline constexpr void combine(T& a, T b) { a += b; } +}; + +template +struct reduce_func + : reduce_func_base, T, Ts...> +{ + const FLOAT_ACCUM START_VAL = CVT_FP32_2ACCUM(1.0f); + inline constexpr void combine(T& a, T b) { a *= b; } +}; + +#endif // GUARD_GUARD_KERNELS_MIOPEN_CUMULATIVE_REDUCTIONS_HPP diff --git a/src/kernels/MIOpenGLU.cpp b/src/kernels/MIOpenGLU.cpp index f9881a1eb0..e69de29bb2 100644 --- a/src/kernels/MIOpenGLU.cpp +++ b/src/kernels/MIOpenGLU.cpp @@ -1,82 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS -#include -#include -#endif - -#include "float_types.h" - -__device__ FLOAT_ACCUM sigmoid(FLOAT_ACCUM x) { return 1.0f / (1.0f + exp(-x)); } - -template -__device__ void GLUFwdContiguousKernel(const TIO* input, TIO* output, uint64_t N) -{ - const TIO* inputFirstHalf = input; - const TIO* inputSecondHalf = input + N; - const size_t gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= N) - return; - - FLOAT_ACCUM val1 = CVT_FLOAT2ACCUM(inputFirstHalf[gid]); - FLOAT_ACCUM val2 = sigmoid(CVT_FLOAT2ACCUM(inputSecondHalf[gid])); - FLOAT_ACCUM val = val1 * val2; - output[gid] = CVT_ACCUM2FLOAT(val); -} - -template -__device__ void -GLUBwdContiguousKernel(const TIO* input, const TIO* output_grad, TIO* input_grad, uint64_t N) -{ - const TIO* inputFirstHalf = input; - const TIO* inputSecondHalf = input + N; - TIO* inputFirstHalf_grad = input_grad; - TIO* inputSecondHalf_grad = input_grad + N; - const size_t gid = blockIdx.x * blockDim.x + threadIdx.x; - if(gid >= N) - return; - - FLOAT_ACCUM inputFirstHalf_v = CVT_FLOAT2ACCUM(inputFirstHalf[gid]); - FLOAT_ACCUM sigmoid_v = sigmoid(CVT_FLOAT2ACCUM(inputSecondHalf[gid])); - FLOAT_ACCUM grad_v = CVT_FLOAT2ACCUM(output_grad[gid]); - - inputFirstHalf_grad[gid] = CVT_ACCUM2FLOAT(sigmoid_v * grad_v); - inputSecondHalf_grad[gid] = - CVT_ACCUM2FLOAT((1 - sigmoid_v) * sigmoid_v * grad_v * inputFirstHalf_v); -} - -extern "C" __global__ void GLUFwdContiguousDim0(const IO_TYPE* input, IO_TYPE* output, uint64_t N) -{ - GLUFwdContiguousKernel(input, output, N); -} - -extern "C" __global__ void GLUBwdContiguousDim0(const IO_TYPE* input, - const IO_TYPE* output_grad, - IO_TYPE* input_grad, - uint64_t N) -{ - GLUBwdContiguousKernel(input, output_grad, input_grad, N); -} diff --git a/src/kernels/MIOpenKthvalue.cpp b/src/kernels/MIOpenKthvalue.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/kernels/MIOpenSoftmaxAttn.cpp b/src/kernels/MIOpenSoftmaxAttn.cpp index 42fd90f86f..e69de29bb2 100644 --- a/src/kernels/MIOpenSoftmaxAttn.cpp +++ b/src/kernels/MIOpenSoftmaxAttn.cpp @@ -1,836 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -// rocblas operates with non-ieee FP8 -#define MIOPEN_FP8_IEEE_EXPONENT_BIAS 0 - -#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS -#include -#include -#include -#endif - -#include "miopen_cstdint.hpp" -#include "miopen_limits.hpp" -#include "miopen_rocrand.hpp" - -#define MIOPEN_ENABLE_F8_DEVICE_CODE 1 -#include - -// Some versions of amd-clang treats MIOPEN_ENABLE_F8_DEVICE_CODE as unused despite the fact that -// it's used in the subsequent include. Some other versions amd-clang failed to compile everything -// without MIOPEN_ENABLE_F8_DEVICE_CODE explicitely defined. -// Fake-using it. -#ifndef MIOPEN_ENABLE_F8_DEVICE_CODE -#message "MIOPEN_ENABLE_F8_DEVICE_CODE must be defined" -#endif - -#ifndef THREADS -#define THREADS 64 -#endif - -#ifndef OUT_TYPE -#define OUT_TYPE float -#endif - -#ifndef dO_TYPE -#define dO_TYPE float -#endif - -namespace { -constexpr float plus_op(float a, float b) { return a + b; }; -constexpr float fmaxf_op(float a, float b) { return fmaxf(a, b); }; - -/// Atomically calculates maximum of non-negative ordered values. -/// Produces wrong results for negatve values or nans, -/// but it is a final amax reducton step and we expect only non-negative ordered values. -__forceinline__ __device__ float atomicMaxOfNonNegative(float* addr, float value) -{ - // ordered non-negatve and even infinity values can be compared as integers - // NOLINTBEGIN - // cppcheck-suppress invalidPointerCast - return __int_as_float(atomicMax(reinterpret_cast(addr), __float_as_int(value))); - // NOLINTEND -} - -template -__forceinline__ __device__ float reductionFullWarp(float reduced_val, uint32_t laneId, Op op) -{ - static_assert(WARP_SIZE != 0, "WARP_SIZEmust not be 0"); - static_assert((SWIZZLE_SIZE & (SWIZZLE_SIZE - 1)) == 0, - "WARP_SIZE and SWIZZLE must be a power of 2"); - - if constexpr(SWIZZLE_SIZE == 1) - return reduced_val; - - reduced_val = reductionFullWarp> 1)>(reduced_val, laneId, op); - - constexpr uint32_t warp_msk = (WARP_SIZE - 1); - - float tmp; - if constexpr(SWIZZLE_SIZE >= 64) - { - // swizzle can handle only 32 lanes, switching to bpermute - uint32_t idx = laneId ^ (SWIZZLE_SIZE >> 1); - - idx = idx >= ((laneId + WARP_SIZE) & ~warp_msk) ? laneId : idx; - int itmp = - __builtin_amdgcn_ds_bpermute(static_cast(idx << 2), __float_as_int(reduced_val)); - tmp = __int_as_float(itmp); - } - else - { - // butterfly reduction based on __shfl_xor - // swizzle () - constexpr uint32_t xor_off = 10; - // constexpr uint32_t or_off = 5; - constexpr uint32_t and_off = 0; - - constexpr uint32_t field_msk = 0x1f; - - constexpr uint32_t and_msk = warp_msk & field_msk; - // constexpr uint32_t or_msk = 0; - constexpr uint32_t xor_msk = (SWIZZLE_SIZE >> 1) & field_msk; - - // clang tidy does not like that (or_msk << or_off) is zero - // and cliams that it's redundant, but it's required for - // __hip_ds_swizzlef_N reference. Menawhile swizzle_op generation - // must be a part of hip intrinsics, because it depends on ISA - // like __hip_ds_swizzlef_N - // For some reason NILINT doesn't work. - // NOLINTBEGIN - constexpr uint32_t swizzle_op = - (xor_msk << xor_off) /* | (or_msk << or_off) */ | (and_msk << and_off); - // NOLINTEND - - tmp = __hip_ds_swizzlef_N(reduced_val); - } - - return op(tmp, reduced_val); -}; - -template -__forceinline__ __device__ float -reductionBlock(float local_val, Op op, uint32_t lid, uint32_t laneId, uint32_t warpId) -{ - static_assert(NumWarps <= warpSize); - static_assert((NumWarps & (NumWarps - 1)) == 0, "NumWarps must be a power of 2"); - __shared__ float reduction_tmp[NumWarps]; - - float reduced_val = reductionFullWarp(local_val, laneId, op); - if(laneId == 0) - reduction_tmp[warpId] = reduced_val; - __syncthreads(); - - if(lid < NumWarps) - { - reduced_val = reductionFullWarp(reduction_tmp[lid], laneId, op); - if(lid == 0) - { - reduction_tmp[0] = reduced_val; - } - } - __syncthreads(); - - return reduction_tmp[0]; -}; - -template -__forceinline__ __device__ float reductionCommon(const float* __restrict__ line, - const float* __restrict__ bias_line, - const float init_value, - const uint32_t seq_len, - ReductionOp&& op, - ElementOp&& eop, - uint32_t lid, - uint32_t laneId, - uint32_t warpId) -{ - float bias = (bias_line && lid < seq_len) ? bias_line[lid] : 0.0f; - float reduced_val = (lid < seq_len) ? eop(line[lid] - bias) : init_value; - - if(bias_line) - { - for(uint32_t loop_lid = lid + blockDim.x; loop_lid < seq_len; loop_lid += blockDim.x) - reduced_val = op(eop(line[loop_lid] - bias_line[loop_lid]), reduced_val); - } - else - { - for(uint32_t loop_lid = lid + blockDim.x; loop_lid < seq_len; loop_lid += blockDim.x) - reduced_val = op(eop(line[loop_lid]), reduced_val); - } - - return reductionBlock(reduced_val, op, lid, laneId, warpId); -}; - -__forceinline__ __device__ bool doDropout(float dropout, rocrand_device::xorwow_engine* state) -{ - return (dropout > 0.0f && prng::xorwow_uniform(state) < dropout); -} -} // namespace - -extern "C" __global__ void __launch_bounds__(THREADS) - SoftMaxWarp(const float* in, - OUT_TYPE* out, - float* __restrict__ M, - float* __restrict__ Z, - float* __restrict__ bias, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ scale_S, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float descaler = (descale_Q ? *descale_Q : 1.0f) * (descale_K ? *descale_K : 1.0f); - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler = (scale_S ? *scale_S : 1.0f) / (1.0f - dropout); - const bool save_stats = M && Z && (laneId == 0); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x * NumWarps + warpId; gid < nhs; gid += gridDim.x * NumWarps) - { - const float* line = in + gid * seq_len + laneId; - auto res = out + gid * seq_len + laneId; - - float local_val = - (laneId < seq_len) ? (*line) * descaler : std::numeric_limits::lowest(); - - if(bias) - local_val -= *(bias + gid * seq_len + laneId); - - float r_max = reductionFullWarp(local_val, laneId, fmaxf_op); - - local_val = (laneId < seq_len) ? expf(local_val - r_max) : 0; - - float r_sum = 1.0f / reductionFullWarp(local_val, laneId, plus_op); - - local_val *= r_sum; - - // It is supposed to be maximum of absolute values, - // however we do not need abs() because expf() above produces - // non-negative value. Plain max() is enough. - r_Amax = fmaxf_op(r_Amax, local_val); - - if(laneId < seq_len) - { - *res = static_cast(doDropout(dropout, &rng) ? 0.0f : local_val * scaler); - } - - if(save_stats) - { - M[gid] = r_max; - Z[gid] = r_sum; - } - } - - if(Amax) - { - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - SoftMaxBlock(const float* in, - OUT_TYPE* out, - float* __restrict__ M, - float* __restrict__ Z, - float* __restrict__ bias, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ scale_S, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float descaler = (descale_Q ? *descale_Q : 1.0f) * (descale_K ? *descale_K : 1.0f); - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler = (scale_S ? *scale_S : 1.0f) / (1.0f - dropout); - const bool save_stats = M && Z && (lid == 0); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x; gid < nhs; gid += gridDim.x) - { - const float* line = in + gid * seq_len + lid; - auto res = out + gid * seq_len + lid; - - float local_val = - (lid < seq_len) ? (*line) * descaler : std::numeric_limits::lowest(); - - if(bias) - local_val -= *(bias + gid * seq_len + lid); - - float r_max = reductionBlock(local_val, fmaxf_op, lid, laneId, warpId); - - local_val = (lid < seq_len) ? expf(local_val - r_max) : 0; - - float r_sum = 1.0f / reductionBlock(local_val, plus_op, lid, laneId, warpId); - - local_val *= r_sum; - - // It is supposed to be maximum of absolute values, - // however we do not need abs() because expf() above produces - // non-negative value. Plain max() is enough. - r_Amax = fmaxf_op(r_Amax, local_val); - - if(lid < seq_len) - { - *res = static_cast(doDropout(dropout, &rng) ? 0.0f : local_val * scaler); - } - - if(save_stats) - { - M[gid] = r_max; - Z[gid] = r_sum; - } - } - - if(Amax) - { - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - SoftMaxCommon(const float* in, - OUT_TYPE* out, - float* __restrict__ M, - float* __restrict__ Z, - float* __restrict__ bias, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ scale_S, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float descaler = (descale_Q ? *descale_Q : 1.0f) * (descale_K ? *descale_K : 1.0f); - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler = (scale_S ? *scale_S : 1.0f) / (1.0f - dropout); - const bool save_stats = M && Z && (lid == 0); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x; gid < nhs; gid += gridDim.x) - { - const float* line = in + gid * seq_len; - const float* bias_line = bias + gid * seq_len; - auto res = out + gid * seq_len; - - float r_max = reductionCommon( - line, - bias_line, - std::numeric_limits::lowest(), - seq_len, - fmaxf_op, - [descaler](float x) { return x * descaler; }, - lid, - laneId, - warpId); - - float r_sum = 1.0f / reductionCommon( - line, - nullptr, - 0, - seq_len, - plus_op, - [r_max, descaler](float x) { return expf(x * descaler - r_max); }, - lid, - laneId, - warpId); - - for(uint32_t loop_lid = lid; loop_lid < seq_len; loop_lid += blockDim.x) - { - float local_val = expf(line[loop_lid] * descaler - r_max) * r_sum; - - // It is supposed to be maximum of absolute values, - // however we do not need abs() because expf() above produces - // non-negative value. Plain max() is enough. - r_Amax = fmaxf_op(r_Amax, local_val); - - res[loop_lid] = - static_cast(doDropout(dropout, &rng) ? 0.0f : local_val * scaler); - } - - if(save_stats) - { - M[gid] = r_max; - Z[gid] = r_sum; - } - } - - if(Amax) - { - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - ScaleReduce(const float* __restrict__ in, - OUT_TYPE* __restrict__ out, - float* __restrict__ Amax, - const float* __restrict__ descale_S, - const float* __restrict__ descale_V, - const float* __restrict__ scale_O, - uint64_t nhsd) -{ - const float descaler = (*descale_S) * (*descale_V); - const float scaler = (*scale_O); - - const auto gid = blockIdx.x * blockDim.x + threadIdx.x; - const auto step = gridDim.x * blockDim.x; - - auto in_ptr = in + gid; - auto out_ptr = out + gid; - const auto end = in + nhsd; - - float r_Amax = 0; - - while(in_ptr < end) - { - const auto res = *in_ptr * descaler; - - r_Amax = fmaxf_op(r_Amax, fabsf(res)); - - *out_ptr = static_cast(res * scaler); - - in_ptr += step; - out_ptr += step; - } - - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - ScaleRowReduceWarp(const dO_TYPE* __restrict__ dO, - const OUT_TYPE* __restrict__ O, - float* __restrict__ out, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_O, - const float* __restrict__ dropout_P, - uint32_t d, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float scaler = (*descale_dO) * (*descale_O) * (1.0f - (*dropout_P)); - - for(uint64_t gid = blockIdx.x * NumWarps + warpId; gid < nhs; gid += gridDim.x * NumWarps) - { - float local_val = 0.0f; - if(laneId < d) - { - const auto dO_ptr = dO + gid * d + laneId; - const auto O_ptr = O + gid * d + laneId; - - local_val = static_cast(*dO_ptr) * static_cast(*O_ptr) * scaler; - } - - local_val = reductionFullWarp(local_val, laneId, plus_op); - - if(laneId == 0) - { - out[gid] = local_val; - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - ScaleRowReduceBlock(const dO_TYPE* __restrict__ dO, - const OUT_TYPE* __restrict__ O, - float* __restrict__ out, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_O, - const float* __restrict__ dropout_P, - uint32_t d, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float scaler = (*descale_dO) * (*descale_O) * (1.0f - (*dropout_P)); - - for(uint64_t gid = blockIdx.x; gid < nhs; gid += gridDim.x) - { - const auto dO_ptr = dO + gid * d + lid; - const auto O_ptr = O + gid * d + lid; - - float local_val = 0.0f; - if(lid < d) - { - local_val = static_cast(*dO_ptr) * static_cast(*O_ptr) * scaler; - } - - local_val = reductionBlock(local_val, plus_op, lid, laneId, warpId); - - if(lid == 0) - { - out[gid] = local_val; - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - ScaleRowReduceCommon(const dO_TYPE* __restrict__ dO, - const OUT_TYPE* __restrict__ O, - float* __restrict__ out, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_O, - const float* __restrict__ dropout_P, - uint32_t d, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - const float scaler = (*descale_dO) * (*descale_O) * (1.0f - (*dropout_P)); - - for(uint64_t gid = blockIdx.x; gid < nhs; gid += gridDim.x) - { - const auto dO_ptr = dO + gid * d; - const auto O_ptr = O + gid * d; - - float local_val = - (lid < d) ? static_cast(dO_ptr[lid]) * static_cast(O_ptr[lid]) * scaler - : 0.0f; - - for(uint32_t loop_lid = lid + blockDim.x; loop_lid < d; loop_lid += blockDim.x) - local_val += - static_cast(dO_ptr[loop_lid]) * static_cast(O_ptr[loop_lid]) * scaler; - - local_val = reductionBlock(local_val, plus_op, lid, laneId, warpId); - - if(lid == 0) - { - out[gid] = local_val; - } - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - BwdAttentionWarp(float* __restrict__ QxK_S, - const float* dOxV, - OUT_TYPE* dS, // may overlap with dOxV - const float* __restrict__ M, - const float* __restrict__ Zinv, - const float* __restrict__ dOxO, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_V, - const float* __restrict__ scale_S, - const float* __restrict__ scale_dS, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - float scale, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler_dropout = 1.0f - dropout; - const float scaler_inv_dropout = 1.0f / scaler_dropout; - - const float descaler_QxK = (*descale_Q) * (*descale_K); - const float descaler_dOxV = (*descale_dO) * (*descale_V) * scaler_dropout; - - const float scaler_S = (*scale_S); - const float scaler_dS = (*scale_dS); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x * NumWarps + warpId; gid < nhs && laneId < seq_len; - gid += gridDim.x * NumWarps) - { - const float M_val = M[gid]; - const float Zinv_val = Zinv[gid]; - const float dOxO_val = dOxO[gid]; - - const size_t idx = gid * seq_len + laneId; - - const float QxK_val = doDropout(dropout, &rng) ? 0.0f - : expf(QxK_S[idx] * descaler_QxK - M_val) * - Zinv_val * scaler_inv_dropout; - - QxK_S[idx] = QxK_val * scaler_S; - - const float dOxV_val = (dOxV[idx] * descaler_dOxV - dOxO_val) * scale * QxK_val; - - dS[idx] = static_cast(dOxV_val * scaler_dS); - - r_Amax = fmaxf_op(r_Amax, fabsf(dOxV_val)); - } - - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - BwdAttentionBlock(float* __restrict__ QxK_S, - const float* dOxV, - OUT_TYPE* dS, // may overlap with dOxV - const float* __restrict__ M, - const float* __restrict__ Zinv, - const float* __restrict__ dOxO, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_V, - const float* __restrict__ scale_S, - const float* __restrict__ scale_dS, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - float scale, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler_dropout = 1.0f - dropout; - const float scaler_inv_dropout = 1.0f / scaler_dropout; - - const float descaler_QxK = (*descale_Q) * (*descale_K); - const float descaler_dOxV = (*descale_dO) * (*descale_V) * scaler_dropout; - - const float scaler_S = (*scale_S); - const float scaler_dS = (*scale_dS); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x; gid < nhs && lid < seq_len; gid += gridDim.x) - { - const float M_val = M[gid]; - const float Zinv_val = Zinv[gid]; - const float dOxO_val = dOxO[gid]; - - const size_t idx = gid * seq_len + lid; - - const float QxK_val = doDropout(dropout, &rng) ? 0.0f - : expf(QxK_S[idx] * descaler_QxK - M_val) * - Zinv_val * scaler_inv_dropout; - - QxK_S[idx] = QxK_val * scaler_S; - - const float dOxV_val = (dOxV[idx] * descaler_dOxV - dOxO_val) * scale * QxK_val; - - dS[idx] = static_cast(dOxV_val * scaler_dS); - - r_Amax = fmaxf_op(r_Amax, fabsf(dOxV_val)); - } - - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } -} - -extern "C" __global__ void __launch_bounds__(THREADS) - BwdAttentionCommon(float* __restrict__ QxK_S, - const float* dOxV, - OUT_TYPE* dS, // may overlap with dOxV - const float* __restrict__ M, - const float* __restrict__ Zinv, - const float* __restrict__ dOxO, - float* __restrict__ Amax, - const float* __restrict__ descale_Q, - const float* __restrict__ descale_K, - const float* __restrict__ descale_dO, - const float* __restrict__ descale_V, - const float* __restrict__ scale_S, - const float* __restrict__ scale_dS, - const uint64_t* __restrict__ seed, - const uint64_t* __restrict__ offset, - const float* __restrict__ dropout_P, - float scale, - uint32_t seq_len, - uint64_t nhs) -{ - static_assert(THREADS % warpSize == 0); - constexpr uint32_t NumWarps = THREADS / warpSize; - const uint32_t lid = threadIdx.x; - const uint32_t laneId = lid % warpSize; - const uint32_t warpId = lid / warpSize; - - const float dropout = (dropout_P && seed && offset) ? (*dropout_P) : 0.0f; - const float scaler_dropout = 1.0f - dropout; - const float scaler_inv_dropout = 1.0f / scaler_dropout; - - const float descaler_QxK = (*descale_Q) * (*descale_K); - const float descaler_dOxV = (*descale_dO) * (*descale_V) * scaler_dropout; - - const float scaler_S = (*scale_S); - const float scaler_dS = (*scale_dS); - - rocrand_state_xorwow rng; - if(dropout > 0.0f) - { - const uint64_t idx = blockIdx.x * blockDim.x + threadIdx.x; - rocrand_init(prng::hash(*seed + idx), 0, *offset, &rng); - } - - float r_Amax = 0; - - for(uint64_t gid = blockIdx.x; gid < nhs; gid += gridDim.x) - { - const float M_val = M[gid]; - const float Zinv_val = Zinv[gid]; - const float dOxO_val = dOxO[gid]; - - float* QxK_S_ptr = QxK_S + gid * seq_len; - const float* dOxV_ptr = dOxV + gid * seq_len; - OUT_TYPE* dS_ptr = dS + gid * seq_len; - - for(uint32_t loop_lid = lid; loop_lid < seq_len; loop_lid += blockDim.x) - { - const float QxK_val = doDropout(dropout, &rng) - ? 0.0f - : expf(QxK_S_ptr[loop_lid] * descaler_QxK - M_val) * - Zinv_val * scaler_inv_dropout; - - QxK_S_ptr[loop_lid] = QxK_val * scaler_S; - - const float dOxV_val = - (dOxV_ptr[loop_lid] * descaler_dOxV - dOxO_val) * scale * QxK_val; - - dS_ptr[loop_lid] = static_cast(dOxV_val * scaler_dS); - - r_Amax = fmaxf_op(r_Amax, fabsf(dOxV_val)); - } - } - - r_Amax = reductionBlock(r_Amax, fmaxf_op, lid, laneId, warpId); - if(lid == 0) - { - atomicMaxOfNonNegative(Amax, r_Amax); - } -} diff --git a/src/kernels/MIOpenTensorKernelsHip.cpp b/src/kernels/MIOpenTensorKernelsHip.cpp index 6b299aa02f..e69de29bb2 100644 --- a/src/kernels/MIOpenTensorKernelsHip.cpp +++ b/src/kernels/MIOpenTensorKernelsHip.cpp @@ -1,183 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#ifndef MIOPEN_DONT_USE_HIP_RUNTIME_HEADERS -#include -#include -#include -#endif - -template -__device__ T miopenAdd(T a, T b) -{ - return a + b; -} - -template -__device__ T miopenMul(T a, T b) -{ - return a * b; -} - -template -__device__ T miopenMax(T a, T b) -{ - return ((a > b) ? a : b); -} - -template -__device__ T miopenMin(T a, T b) -{ - return ((a < b) ? a : b); -} - -#ifdef USE_1D_TENSOR_GENERIC -// N -extern "C" __global__ void Op1dTensorGeneric(const MIOPEN_TYPE* a, - const MIOPEN_TYPE* b, - MIOPEN_TYPE* c, - const uint64_t Aoffset, - const uint64_t Boffset, - const uint64_t Coffset, - const uint32_t a_nstride, - const uint32_t b_nstride, - const uint32_t c_nstride, - const MIOPEN_TYPE alpha0, - const MIOPEN_TYPE alpha1, - const MIOPEN_TYPE beta, - const uint32_t total_work, - const bool use_beta) -{ - const MIOPEN_TYPE* a_off = a + Aoffset; - const MIOPEN_TYPE* b_off = b + Boffset; - MIOPEN_TYPE* c_off = c + Coffset; - - const auto gid = blockIdx.x * blockDim.x + threadIdx.x; - auto a_ptr = a_off + gid * a_nstride; - auto b_ptr = b_off + gid * b_nstride; - auto c_ptr = c_off + gid * c_nstride; - - const auto step = gridDim.x * blockDim.x; - const auto a_step = step * a_nstride; - const auto b_step = step * b_nstride; - const auto c_step = step * c_nstride; - - const auto c_end = c_off + total_work * c_nstride; - while(c_ptr < c_end) - { - const auto res = MIOPEN_TENSOR_OP(a_ptr[0] * alpha0, b_ptr[0] * alpha1); - c_ptr[0] = use_beta ? c_ptr[0] * beta + res : res; - - a_ptr += a_step; - b_ptr += b_step; - c_ptr += c_step; - } -} - -#endif - -#ifdef USE_2D_TENSOR_GENERIC -// NC -extern "C" __global__ void Op2dTensorGeneric(MIOPEN_TYPE* a, - const int a_nstride, - MIOPEN_TYPE* b, - const int b_c, - const int b_nstride, - MIOPEN_TYPE* c, - const int c_c, - const int c_nstride, - const MIOPEN_TYPE alpha0, - const MIOPEN_TYPE alpha1, - const MIOPEN_TYPE beta, - const unsigned int bitmap, - const int work_per_wg, - const long Aoffset, - const long Boffset, - const long Coffset, - const int num_wg) -{ - int gid = blockIdx.x; - - MIOPEN_TYPE* a_off = a + Aoffset; - MIOPEN_TYPE* b_off = b + Boffset; - MIOPEN_TYPE* c_off = c + Coffset; - - int o_n_div = (bitmap & (1 << 0)) ? 1 : c_c; - - // num_wg: the number of workgroups should be launched - // MAX_NUM_WG: the maximum number of workgroups actually launched -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" - if(beta == static_cast(0)) -#pragma clang diagnostic pop - { - for(; gid < num_wg; gid += MAX_NUM_WG) - { - - int lid = threadIdx.x; - int o_c_gid_off = gid % b_c; - int o_n_gid_off = gid / b_c; - - int bindex = o_n_gid_off * b_nstride + o_c_gid_off; - MIOPEN_TYPE operand = b_off[bindex] * alpha1; - - while(lid < work_per_wg) - { - int o_c = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c; - int o_n = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div; - int aindex = o_n * a_nstride + o_c; - int cindex = o_n * c_nstride + o_c; - c_off[cindex] = MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand); - lid += blockDim.x; - } - } - } - else - { - for(; gid < num_wg; gid += MAX_NUM_WG) - { - int lid = threadIdx.x; - int o_c_gid_off = gid % b_c; - int o_n_gid_off = gid / b_c; - - int bindex = o_n_gid_off * b_nstride + o_c_gid_off; - MIOPEN_TYPE operand = b_off[bindex] * alpha1; - - while(lid < work_per_wg) - { - int o_c = (bitmap & (1 << 0)) ? o_c_gid_off : lid % c_c; - int o_n = (bitmap & (1 << 1)) ? o_n_gid_off : lid / o_n_div; - int aindex = o_n * a_nstride + o_c; - int cindex = o_n * c_nstride + o_c; - c_off[cindex] = - MIOPEN_TENSOR_OP(a_off[aindex] * alpha0, operand) + beta * c_off[cindex]; - lid += blockDim.x; - } - } - } -} - -#endif \ No newline at end of file diff --git a/src/kernels/activation_functions.hpp b/src/kernels/activation_functions.hpp index 256494f90e..e69de29bb2 100644 --- a/src/kernels/activation_functions.hpp +++ b/src/kernels/activation_functions.hpp @@ -1,541 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2024 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef MIOPEN_NRN_OP_ID -#define MIOPEN_NRN_OP_ID 0 -#endif - -#if MIOPEN_USE_FP16 == 1 -#define FP_TYPE half -#define FP_TYPE_PREC float -#define EPSILON static_cast(0.0001) -#ifndef HALF_MAX -#define MAX_VAL 65504 /* max value */ -#else -#define MAX_VAL HALF_MAX -#endif -#endif -#if MIOPEN_USE_FP32 == 1 -#define FP_TYPE float -#define FP_TYPE_PREC float -#define EPSILON static_cast(0.000001) -#ifndef FLT_MAX -#define MAX_VAL 3.402823466e+38F /* max value */ -#else -#define MAX_VAL FLT_MAX -#endif -#endif - -#define PPCAT_NX(A, B) A##B -#define PPCAT(A, B) PPCAT_NX(A, B) -#define TWO 2 -#define FOUR 4 -#define EIGHT 8 - -#define FP_TYPE2 PPCAT(FP_TYPE, TWO) -#define FP_TYPE4 PPCAT(FP_TYPE, FOUR) -#define FP_TYPE8 PPCAT(FP_TYPE, EIGHT) -#define FP_TYPE_PREC2 PPCAT(FP_TYPE_PREC, TWO) -#define FP_TYPE_PREC4 PPCAT(FP_TYPE_PREC, FOUR) -#define FP_TYPE_PREC8 PPCAT(FP_TYPE_PREC, EIGHT) - -#define MIOPEN_NEURON_PASTHRU 0 // x -#define MIOPEN_NEURON_LOGISTIC 1 // 1 / (1 + e^-x) //Sigmoid -#define MIOPEN_NEURON_TANH 2 // beta * tanh(alpha * x) -#define MIOPEN_NEURON_RELU 3 // max(0, x) -#define MIOPEN_NEURON_SOFTRELU 4 // log(1 + e^x) // bonomial normal log likelihood -#define MIOPEN_NEURON_ABS 5 // abs(x) -#define MIOPEN_NEURON_POWER 6 // (alpha + beta * x )^gamma -#define MIOPEN_NEURON_CLIPPED_RELU 7 // min(alpha, max(0, x)) -#define MIOPEN_NEURON_LEAKY_RELU 8 // alpha * x | x <= 0; x | x > 0 -#define MIOPEN_NEURON_ELU 9 // alpha * (e^x - 1) | x <= 0; x | x > 0 -#define MIOPEN_NEURON_TOTAL 10 - -#define kBNLL_THRESHOLD static_cast(50.0) - -template -__forceinline__ __device__ void ActivationFunction_PassThru(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = data[i]; - } -} - -template -__forceinline__ __device__ void ActivationFunction_ReLU(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = data[i] * (data[i] > 0); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Sigmoid(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - // y = 1/(1 + exp(-x)) - res[i] = static_cast(1) / (static_cast(1) + exp(-data[i])); - } -} - -template -__forceinline__ __device__ void ActivationFunction_TanH(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T beta, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - // y = beta * tanh(alpha * x) - res[i] = beta * tanh(alpha * data[i]); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Abs(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = fabs(data[i]); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Square(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = data[i] * data[i]; - } -} - -template -__forceinline__ __device__ void ActivationFunction_Sqrt(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = sqrt(data[i]); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Linear(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T beta, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = alpha + beta * data[i]; - } -} - -template -__forceinline__ __device__ void ActivationFunction_Power(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T gamma, - const T beta, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - // y = (alpha + beta * x ) ^ gamma - T arg = alpha + data[i] * beta; - res[i] = arg <= static_cast(EPSILON) ? static_cast(0) : pow(arg, gamma); - } -} - -template -__forceinline__ __device__ void ActivationFunction_BNLL(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - // y = log(1 + exp(x)) - res[i] = (data[i] > 0) ? (data[i] + log(static_cast(1) + exp(-data[i]))) - : log(static_cast(1) + exp(data[i])); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Leaky_ReLU(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = data[i] * ((data[i] > 0) ? static_cast(1) : alpha); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Clipped_ReLU(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = fmin(static_cast(alpha), fmax(static_cast(data[i]), 0)); - } -} - -template -__forceinline__ __device__ void ActivationFunction_ELU(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - res[i] = (data[i] > 0) ? data[i] : (alpha * (exp(data[i]) - static_cast(1))); - } -} - -template -__forceinline__ __device__ void ActivationFunction(T (&__restrict__ res)[N], - const T (&__restrict__ data)[N], - const T gamma, - const T beta, - const T alpha) -{ - static_assert(N <= 4); - if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_PASTHRU) - { - ActivationFunction_PassThru(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_LOGISTIC) - { - ActivationFunction_Sigmoid(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_TANH) - { - ActivationFunction_TanH(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_RELU) - { - ActivationFunction_ReLU(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_SOFTRELU) - { - ActivationFunction_BNLL(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_ABS) - { - ActivationFunction_Abs(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_POWER) - { - ActivationFunction_Power(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_CLIPPED_RELU) - { - ActivationFunction_Clipped_ReLU(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_LEAKY_RELU) - { - ActivationFunction_Leaky_ReLU(res, data, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_ELU) - { - ActivationFunction_ELU(res, data, gamma, beta, alpha); - } -} - -template -__forceinline__ __device__ void -ActivationFunction_PassThru_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T* /*bot_data*/, - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i]; - } -} - -template -__forceinline__ __device__ void ActivationFunction_ReLU_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i] * (bot_data[i] > 0); - } -} - -template -__forceinline__ __device__ void ActivationFunction_TanH_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T* /*bot_data*/, - const T (&__restrict__ top_data)[N], - const T /*diff_scale*/, - const T /*gamma*/, - const T beta, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - // dy/dx = alpha * (beta - y^2 / beta) - T y = top_data[i]; - bot_diff[i] = fabs(beta) <= static_cast(EPSILON) - ? static_cast(0) - : (top_diff[i] * alpha * (beta - y * y / beta)); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Sigmoid_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T* /*bot_data*/, - const T (&__restrict__ top_data)[N], - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - // y = 1/(1 + exp(-x)) - T sigmoid_x = top_data[i]; - bot_diff[i] = top_diff[i] * sigmoid_x * (static_cast(1) - sigmoid_x); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Abs_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i] * ((bot_data[i] > 0) ? 1 : -1); - } -} - -// Compute dy/dx = beta * gamma * (alpha + beta * x)^(gamma - 1) -// = diff_scale * y / (alpha + beta * x) -template -__forceinline__ __device__ void ActivationFunction_Power_Diff(T (&__restrict__ bot_diff)[N], - const T* /*top_diff*/, - const T (&__restrict__ bot_data)[N], - const T (&__restrict__ top_data)[N], - const T diff_scale, - const T /*gamma*/, - const T beta, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - T arg = alpha + bot_data[i] * beta; - bot_diff[i] = - arg <= static_cast(EPSILON) ? static_cast(0) : (diff_scale * top_data[i] / arg); - } -} - -template -__forceinline__ __device__ void ActivationFunction_BNLL_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T /*alpha*/) -{ - for(uint i = 0; i < N; ++i) - { - // y = (log(1 + exp(x))) - // dy/dx = 1/ (1 + exp(-x)) - T expval = exp(fmin(static_cast(bot_data[i]), static_cast(kBNLL_THRESHOLD))); - bot_diff[i] = top_diff[i] * expval / (expval + static_cast(1)); - } -} - -template -__forceinline__ __device__ void -ActivationFunction_Leaky_ReLU_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i] * ((bot_data[i] > 0) ? static_cast(1) : alpha); - } -} - -template -__forceinline__ __device__ void -ActivationFunction_Clipped_ReLU_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T* /*top_data*/, - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i] * ((bot_data[i] > 0 && bot_data[i] <= alpha) ? static_cast(1) - : static_cast(0)); - } -} - -template -__forceinline__ __device__ void ActivationFunction_ELU_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T (&__restrict__ top_data)[N], - const T /*diff_scale*/, - const T /*gamma*/, - const T /*beta*/, - const T alpha) -{ - for(uint i = 0; i < N; ++i) - { - bot_diff[i] = top_diff[i] * ((bot_data[i] > 0) ? 1 : top_data[i] + alpha); - } -} - -template -__forceinline__ __device__ void ActivationFunction_Diff(T (&__restrict__ bot_diff)[N], - const T (&__restrict__ top_diff)[N], - const T (&__restrict__ bot_data)[N], - const T (&__restrict__ top_data)[N], - const T diff_scale, - const T gamma, - const T beta, - const T alpha) -{ - static_assert(N <= 4); - if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_PASTHRU) - { - ActivationFunction_PassThru_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_LOGISTIC) - { - ActivationFunction_Sigmoid_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_TANH) - { - ActivationFunction_TanH_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_RELU) - { - ActivationFunction_ReLU_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_SOFTRELU) - { - ActivationFunction_BNLL_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_ABS) - { - ActivationFunction_Abs_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_POWER) - { - ActivationFunction_Power_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_CLIPPED_RELU) - { - ActivationFunction_Clipped_ReLU_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_LEAKY_RELU) - { - ActivationFunction_Leaky_ReLU_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } - else if constexpr(MIOPEN_NRN_OP_ID == MIOPEN_NEURON_ELU) - { - ActivationFunction_ELU_Diff( - bot_diff, top_diff, bot_data, top_data, diff_scale, gamma, beta, alpha); - } -} diff --git a/src/kernels/gfx90a.kdb.bz2 b/src/kernels/gfx90a.kdb.bz2 index 89716c1bb2..e69de29bb2 100644 --- a/src/kernels/gfx90a.kdb.bz2 +++ b/src/kernels/gfx90a.kdb.bz2 @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:da274b54b9e036d2b8aebb640785cdb2084b0b031ab914dba959cb7a792ee831 -size 233781575 diff --git a/src/kernels/gfx90a68.HIP.fdb.txt.bz2 b/src/kernels/gfx90a68.HIP.fdb.txt.bz2 index 75c876cec7..e69de29bb2 100644 Binary files a/src/kernels/gfx90a68.HIP.fdb.txt.bz2 and b/src/kernels/gfx90a68.HIP.fdb.txt.bz2 differ diff --git a/src/kernels/gfx942.tn.model b/src/kernels/gfx942.tn.model index 3ac6e8da27..e69de29bb2 100644 --- a/src/kernels/gfx942.tn.model +++ b/src/kernels/gfx942.tn.model @@ -1 +0,0 @@ -{"architecture":{"class_name":"Functional","config":{"name":"tunaNet","trainable":true,"layers":[{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,18],"dtype":"float32","sparse":false,"ragged":false,"name":"input_1"},"registered_name":null,"name":"input_1","inbound_nodes":[]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense","trainable":true,"dtype":"float32","units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,18]},"name":"dense","inbound_nodes":[[["input_1",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"re_lu","inbound_nodes":[[["dense",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_1","trainable":true,"dtype":"float32","units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"dense_1","inbound_nodes":[[["re_lu",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_2","trainable":true,"dtype":"float32","units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,18]},"name":"dense_2","inbound_nodes":[[["input_1",0,0,{}]]]},{"module":"keras.layers","class_name":"Add","config":{"name":"add","trainable":true,"dtype":"float32"},"registered_name":null,"build_config":{"input_shape":[[null,128],[null,128]]},"name":"add","inbound_nodes":[[["dense_1",0,0,{}],["dense_2",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu_1","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,128]},"name":"re_lu_1","inbound_nodes":[[["add",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_3","trainable":true,"dtype":"float32","units":256,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,128]},"name":"dense_3","inbound_nodes":[[["re_lu_1",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu_2","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,256]},"name":"re_lu_2","inbound_nodes":[[["dense_3",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_4","trainable":true,"dtype":"float32","units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,256]},"name":"dense_4","inbound_nodes":[[["re_lu_2",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_5","trainable":true,"dtype":"float32","units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,128]},"name":"dense_5","inbound_nodes":[[["re_lu_1",0,0,{}]]]},{"module":"keras.layers","class_name":"Add","config":{"name":"add_1","trainable":true,"dtype":"float32"},"registered_name":null,"build_config":{"input_shape":[[null,64],[null,64]]},"name":"add_1","inbound_nodes":[[["dense_4",0,0,{}],["dense_5",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu_3","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"re_lu_3","inbound_nodes":[[["add_1",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_6","trainable":true,"dtype":"float32","units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"dense_6","inbound_nodes":[[["re_lu_3",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu_4","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"re_lu_4","inbound_nodes":[[["dense_6",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_7","trainable":true,"dtype":"float32","units":32,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"dense_7","inbound_nodes":[[["re_lu_4",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_8","trainable":true,"dtype":"float32","units":32,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"dense_8","inbound_nodes":[[["re_lu_3",0,0,{}]]]},{"module":"keras.layers","class_name":"Add","config":{"name":"add_2","trainable":true,"dtype":"float32"},"registered_name":null,"build_config":{"input_shape":[[null,32],[null,32]]},"name":"add_2","inbound_nodes":[[["dense_7",0,0,{}],["dense_8",0,0,{}]]]},{"module":"keras.layers","class_name":"ReLU","config":{"name":"re_lu_5","trainable":true,"dtype":"float32","max_value":null,"negative_slope":0.0,"threshold":0.0},"registered_name":null,"build_config":{"input_shape":[null,32]},"name":"re_lu_5","inbound_nodes":[[["add_2",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_9","trainable":true,"dtype":"float32","units":21,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,32]},"name":"dense_9","inbound_nodes":[[["re_lu_5",0,0,{}]]]}],"input_layers":[["input_1",0,0]],"output_layers":[["dense_9",0,0]]},"keras_version":"2.14.0","backend":"tensorflow"},"image_data_format":"channels_last","input_shapes":[[18]],"output_shapes":[[21]],"tests":[{"inputs":[{"shape":[18],"values":["eMzhP2jhzD6Tjno/y2oPQCQM7z/iLnq//zhzP2L9Gr5oZNO9+DnSPiiAEz6iJbo/XtNCP8Aw+T0LQuM+XdeqPvw9vz8CFVK+"]}],"outputs":[{"shape":[21],"values":["kaKswc9vlECrja/A5mNAwbMVXkDQaVhAa+2kwdg+sT9hvtHAFe6YwVCwhUCHaKLABRkKwtESjMEPzGjBRIQKwZtUgsCpmJhAePFqwZZBXsANtkPA"]}]}],"trainable_params":{"dense":{"weights":["Hyozu0y6tb67sRE9EW5PPFUG0D10L6E8GLSevFv/Gr5tv5O5wD6DvldrAD5zPeK8JykJvmFC9zxregy99b4gvakgLbwgGTC9MoAmPTHqST1UvEs8zVESPupitTgSNoK9u0D4PUzikzyX4am+wj8tPGZ+qT2H1EA9E+yAPTnZqryoYy68sV9CPayrsD3Qlhy8WFOwvtmM9jvR2OA9aPhOvCVtMzxVVbk9NHCePP4rXD2BC1C8FIotPV3jzjoYijo9BqOmPlCo5DxKO2I82EESPbGGeD3BC4I9B2jTPZWLzjyXntY6GhrjvroAZj30srO9xjuDPQ5ghj1idUw6dZiova5KgT7fe5K9XNaKPmKZL7yOF1A+37JiPhna0z0Qow29CxRpvcJqSr5i87K9qDt0vsBA/72jKOG9mvf5PPR40D2uBDS++Nw1Pqy1mj4lzYk9wdN6vkwtwD6w0XA8Jat5vZNvTztIkRs9tvOTvaLBzj1R1Pm83pR6vmPriT03g/e8zYkAvvJKsbu7SDC9YCnLPDf5kb0j4i0+Z7tAPo62cD1p3MK+OhiLvoZmtj0c5aG+bstlPU9b4jvejGg+URcAPoxQ97zUscI+huVRPgbgnD24bzi+uL2RPfa7tT1Y7bM9dEp4vYnfyz3PPO09qiKDPv1BBL4UaAK+TbmnO+8VP77JbRy+lQIdvOQrmj5YWwc+EJWSPhPoeD44+2K+vhi5vbiXQT4sT6C8gwTRPcUDgT0/9Qo+gTNhPTRVNL6Vv6i+eRazvmVRoj2tMSk81ltnvgpOnL2PlOQ+1KKivQxvTL5oNyW9B1LIPO5bvT2XUJC8xI+zPWWCgb25D/692VmRvlvHnbx6M1y+tlt9PRG6qD3YUuU9f4CjvJaYwT3VYyq+Cs5bPmcmBb4dhuK9OiTjPn3miL5yfAw+zuPePtYCmr2WQei8vzgAP3HIYD6R2rA95T48PkBwurt9Tg++HHWZPcVBjD34kg++Y/X9PaR74L38wAU+vFgAvpsGLjqJqRg+","xYlTvCuLOT7QasU90N5cvcb/Zj2teQy8RFAYveJIwL53S6a6d89JPSvJdTyu9yq8oC8yv/a+1L7k9xa+6RLbvAbOB71iATu+IknQPJcZpjyN0Ec8Xys2PDtqGT0M5JQ9nRSSvcz4qT22u7M9id6WOyiUQT6DHlY919gmv0V9Xb1RybG7QQmiPdJrbT0hXzY9vfyWPNJCkT00qS2+OVqyvQl/GrryxCU9/hYDvN8iPr8FTZC88prdPclkjD1fO6c9GmSGPaxxbD0IcSI9vlejvbyiJr/jPQ+/iFXuuwvQujwxuLQ6bzGVPQwEFTxdGQq+9fj6PGQY6rtynp46Cb9UPK6oLr0Uzcw8ebKQPJupyTyI8XK6B6gNvgIxfb42XF09jZsiPMq3UL1/3f09nI4Dv/FivL3mGYs9ABY7vrb98j3t45c+kVJDvvf7oD56yPI9ZA1nPZBCWb2DfSg+ktqVPYwLGT0KiUq9eoxUPf06e70wf7A8yYWSPpser7wc6Re9krxKvY+o/D38iQc+pSSoPVaXED5h6yO+eVadPXujHT0f2c++P5q6PkNdh70Puks+ypz9O61+hT2F6Gk8SWOUPRdF4Tzd5a+8PHvlPY/jJb6swwA+D/iNPTjKij05NDY+Kz0tPUpJAD6Dvs08Si06PMaUHT6TXEo+ZoDYt5AYWD6DrBU+LoLfPR9qsD14Avm8dxMWvlW6oL3026A+v4DYPWld0b3EBiM+fC/8u2jXhz3zq7g9+SE2PP/IqT456N6+BrPEPoO3HL2VJ4W+zi5GPoEGmz5n6JA9CH5KPboJfz4t38i7Lhvyu5Xxo73EbBU9Gdo8PeB7gj1l18Q9BocJPFSkt70Z4GU+pGEePAEHur2cJQm+R0eDPZ/EHD75iWg+W/7aPiCvFT5SXSw+AWVdvhejj75xX5c90G4Nvt0Ffz5K3Bs88FRLvqXnxzutTDk+AuLxvSMisz0GJ24+J5v0PTnwLb0fUyi9SUnYPYvzOr7kcNY99SsaPvcxWzp9jAC+","zjk1PubUrb3Aac49WT3jvHufoD01PrQ9SbAqPpE5Lr66Aou+YTxjvmi6Lr6xZU++F8etPcJ0UT0hDze+tKn5PTq/Er+sUO27xMp8vlSWkL7qMhs+2bEsPYPXCj0769y9HhGkvV4M0rxxr+48qOcOPca/jbxoe2C9H7utvc666707VMG94YztvYzAfj5N+ym+cY8Avt2+MbwcpBw+fXuGvtUpTz73zQy9tzkzvrvOC76K+hS+KmooPkMd5b3CYdm98ASFvM3bKz6cLgO8K1MVPoucur0q/sW+OCWbvheMkD3cRo28B6QzvXxj6jsVjvW8/PMJPkp7h752+hI84imdvaOvCz7KVQ8+lUzVvfvppbzR9Rq+8eoEOpTemr73L9Y8lEKwvlH2azyVpAa+/+tIPkqL1L0VhsC8tmQOvjMKUr5Tr7a9zRQJvlpLhr468Ce9MMNTvsU89D0soBk+AxOqvrYxf76gdNe4fWt9vu//B76pwwg+gBEhvmBSD73tFRs+Hr0HPX9H+b3SFdU8F0vAvpHinL1qyUk9OQnAvMUPED3pm1y+tEW7PTaVHz6ZzyU+hHejvXcGgj7jBA0+5YAAO5xg+72QjA29kMJZPRUhS75cH4o9GnfoPa3TOD3L/Vw9l/ajPJwqtb1pHaK7UOngPPlxYb1xvpw97SKRu3HnZT1VuTk+7QGWPg7JZD1dtCq+shNJvoomqL1tAx6+58AMPjFGaL1bNrA+JPJEvgara74J53C9eJh8vWNIa75/vZ69YCFdP4lV/zzQqb8+IAlHvgmM17zwBw4+JnIHPtwp5j0cfu29UHGCvVa2+byNQZ49Q0tNPKsneD428Na9hMIcvdq1Cb1U7dy9Oqu7vVxGgb7UNJ89sJDeO5rYBT2rNAC+u+KiPeenTD5aMT0+/jyQPXo5sz3le7M9OzlPPvRsoD4WyXq+x/bvO/BDgDyZ3Ms9c99ou98M+T52cvc8rnxavP7emzxqQiI898U8Pc2kdz1PFC89KUuPPY/wFrzG9RG+","3LspPiIwX76Wq5c9TL6jPYazjT5YPQA9uQ+0PfGLWr0f+pE9H2rPvSPwZz4SLUs+fWGgPXXk87c1sIM+epD1Papt5r4xkdU9FRR/PqYXCD255r89Th02PkL3Fr7lhYI+AeWDvW4Xf73l3Wg+V7QBvV9/ubwwDlo+KPpkPqFQqbzkMpW9+H2PPpy7ET6LDTc+ELgIPsaMeD0/CyY+RZQZvtevgb2RoY++AwrqvcmbTb339OO8TJedPlJYk73D0S++Xv4HvSAxvD7STT+9uI5KvvhlPb3lH7K+HAsXvunXqDyZA5q8bq10PRT4iT0Kmgq9DLF5vSn8BL3kQb87SmzzvYkwnLx3RbS9JKodPbKIGD5YbmQ+GWjUPNk04Tvd3JQ9Lc0tPuhlDT4ALf09lXhaPSlBF724248+VTKvOzPg6T0R/qQ+XsPqvV1ISb73uIG9lkYGvhzCF776Vb89vRW4PoIZ0b2gAhe+8FntvetF170b4wU8BQT+u/znAz2mRpM9wBeVOxv/ur2WqNQ9oKsQvrcUvL0v6xq9l3eSPX732r09fTm+uP3wvOWp/r2Q95O8BiT5PeyDaT5pFxW+41EfvTAh+rv1Dhm+u+hxPUxR0D1oEqc7tqx4vjiLBz6c7oU9uXY9vsnqmr3Dl3S9H1shPuHMHL7YWeQ92IMKPIy7bT4s6E88XBpSPEre8jxiiEG9j8dmvmquLb2sXEU9e0AVPWK0Sr6v8F6963S6PcIZ9rzNd509Nn9yvsxCzj1hGrm9Z3vivovFxz1s3fY95A3mPWGhTT6DeVI9yLxQPXKPeb6OHHE99RqsvZst8D3VdSK9ljX+PBsCybwExO48xoPQPLtqrjyOKce9vYHxvcdUuD2tFZ49PFFnvHKUIr7tvvK9pYNIPvfZ6b3qlRw+IscBPc4KBr6h61K+XvToPXsQML5Brhk9ZMnROyuqEz0GbF28/0TaO170dT4w1Xu9B9S3vT2XPD6bjKQ9Zv7gPEEsQb719iY+Zn3PO7mECrycZXW+","bI7QvcxX6b2UMku9W5FxvJQypjzqZiG8/goTPVC/pL0tM5Y9Azqfvg1kvr0BVCg9Wox8vCSXTz4tZKU9OMZmvq+Y374ISQO7aM4jvjBJv7tATZi9dyr2vDVKq72aTOo8II35vZLO7jwVwVO9RpYyvIcdXr0WpCy+1SEJPlzWKT1gwrs8edZuPb2Roby7llg9qd8JvTUHtD3q9oA8qaJfPm3ofr1wHyS+X3OavYcXrj1bdKO9dcN+PcklR75n6eM9TF5sPK5PXr7lg9Q7Xt2NPuAehj0LEIo+K7aEPW1MqDxsVoI+JHY+vt258Dv+EUI+6pkkviM82zyZhlA+9JIEPl+lE70tIc89Wb7IOtdKKz1L93293q36POgPrr0zy1k9DX+FvV3GFT5tzTc9SSOpvIxpZzwXT1G+pIf+vY2VVz4I+zo+o07QvbFAv72hBmY9WPUHvSGHx70cGrI9xBw0vh1sPT442AC9+kwBvdw9ibvOqeg8hK8FvbdNO77c0gI7796uPAMwBL75XZq9UHKgvCIb8bzbfNm9l53ovc9Xw73wol89diouPp2tBzxcj9K9yBwlPcNTXr4b+RU+cjJkvqBNOT2xgZ090qSXPMZycr6PGz+9MrulvsdGo70sV8y9fAaCvn4POT75kSq9NlmQvubnyD3FkvC9FndOvkRLpL1J4sa+FJf+vYEfKD3q0Au874QkvYeeBb90OU6+ISAzvSj8TzvjfLS9GzETviXErztqxro7NoGBvZUuzL3OZMe7UJBBvRYAQL74chy8F5BavQmmNr8wBfW9GA6MvYokRr0SIW089RDUPC+Fq75uLgi/pA/AuxqC3r7hfrA8aQmWvnU2Trt2Sui+MgAdPYYQA76LLPw8k+aTu9sMwbrRCMa8kKVPvz1BDD1UtQI9XpG8OwLBCDzTxh88KZW+u37Mm7x5L469zri8vmQB6jxiF42+RXhVPAUaOrmwgdy+IvtmPCaAvr+qbN6+q84nPfS2hL5yPQO/+8HjO4Cctr8gCA29","/J2HO2mmIb0blrs6w3KQPu5TnL3KNGg9zYmOPI44BDwACjW4ogWZvFBiQ74Piug5O7VIPDu18Duoo4E93V52PYo4ID3M0CQ7qlGFPgHJkT03lJk8uxryvdSdm71+/Ss+wAgVPItOpjwy2TY8HWXDO6WZ1r0BgnS+qCdeOhXsl7230DW9+7AjO4cmL7wp9eM89yKSvshnwL5RzK87N+S1PdTMDL34mDI/Tj7RvmqcEzx2kzc8g2b7vWCYPz0lqnY+FDnHPKpy5r2vfnC7VXVuPn4WD7s7Qko8uO4VvcU2I7tHdWY560idPZH4rr1ecDu8V4gCvo+UPT7SWGu4LqYPPFm17btyi4i+uTYYPUjN7L5o4XU+SsRavs8OSb8kMfo+0R6POhvkJD+92pm+HfwlPttFBbs72RG/lhwtPwzuGD07C/8+rzxkv7lSF71lAwK/NnezPd1LETwm8cQ+IUYGvywdHz0Q25k8pa7zvNv1jD74xWE+Pf2jvihXizzM/sK8EFvhvErSO7xAmJC+AZcIvyjkeT2HLCi9WIkQvm9y0z4o+Co8vhX7PkNdPT2btbs8AtiVPVxApb3f/JA+2w4tPpDYUL5vFv48BfUevu+h4z5rSpK7paniPWupCL46DYa+IpzvuUkWub2Fnwy+NwchP25q2D0uBQu/eNdvODZLLz8QE6u80+OCvSNHrDzpV6u9D0msvZ0BD75MUhe+3ihhu4B7FD2DASa+BR/9vLhjKLqQgZI8lP7MO8rSuL1HHnK6hB6LvSgiob39Tpa93O3aO6W6fzyIVRo8OXwVPMK+obvqJC+8V3+xva2EjjwheIc8iS6dvc3hxDueMUE8P2hBOw0sHDwSUl69rT2duyHS5TuxzwE9kgLMOhZd1ztWVTQ8q/S4ueUlADstkk07aKr4On3dMTv0jMi9Ibi3vfrnqL05W0u9Ndpaush1rL1srY489fkEPD+zljzawyQ9u3kLvjToijohNym9LIUxPcWEhLyOP7K90Ug5vkdYwDjdwua9"],"bias":["YRm9vp+tlr65oI6+GnrJvvRaF766Cpu+6nFlvmuSpr5f4j++UTRXvutGkb4s+cG+Du8Ev0i9h761ILC+Y/+yvrQvvL4n24a+sxYOvtzDub72QNm9T4oSv8DpqL4al7e+RFqlvuEBI741mpS9r/tJvsORDL7hk4a+0M+1vlqRE74o6xy+DYV1viNobr7Jq92+WUv1vnAbbr5+WvS9ye7svn/mib6RQtS+eM9Xvk+our4pZdq+8+zkvmXza748F0G+PWq+vqvO+b6Awoa+wQKtvj84qb4l3Kq+b+qdvkSFO75/Gdq9XOmNvsrD873Z3Vu+lb98vkfP575ckt69pQKcvg=="]},"dense_1":{"weights":["HqdxPVYZQLsYfYM9BcppvaPz3LwxEu87WRoFPEGUA76FhU68cSCCPRQ3yLyd5Eu+1bGOPPvHBL7yxR6+Vc+SO7VeiL3Ka6++a8amPJG77Tw6sCW8N8nxvTkCerw3SZk7lx+mvFP/ub1KI8e++lEpvcjz/j1Uf5C9BkB+vLuOU74PKFm8PdYpvqwnTr6lqn48yo+mOp2HJbxDoSG9lt0Dvr/vLzy68ci+EvUYPOKIAjxPIAS89ITDPI7uTL2wuZK7K1qnPL7UIL0hpTE9foMVPDWZEL0DikI9bYIKvGSmk7wICd88s8jivI8dtr2g1Hi8CbpJPaA1fzxR9kc8LHMNu5i9Nb5Sdx89E0gcvYYhZL514g0+agVlPHO9azxgYgO9MxD/vY7RUr3BVSM8PlB6vOrsBz2PD0S8N/KMu+vcQrxpZfU77BkhPWLJzDp1ktW7cUE6vLFMHjzWpIk6FiyzvWAeSbsJFT+8PW3RvHWiDbw3Tr28FS8mOzLzGD3avtg85V7Bveh1hjzXU5E87IpDvnrjIL1phG68jR82vS1TWTzanmG+i77evQuLHz0nSYE8v9Q1vkaK9Ttzx7A8gNzRPNtU9DzH1WE9mq7yvGj5OjzVSbW7bjELvZzPwbsHAzI8nxGxvBAPEj04AMo8Klp5vjILy769M8W7QxADvMyVKDyFjzW6zNz1POMhlLzIPY695DIuPdLQnz2jRTE9umK9OxAfYr2OZEY9zUfVvcn6wbu1Ffi8w7jmvLbUS70734k7oPURPexPFb0fcjE8wtnzvP7bnTzivAs8TXpvvQ4GvTwoMQ69eRLIuu7EUjx0uI89q5xxPeXTyjyUbbM7sJL6vBT/hju774Q80XP/vEK7mLxabtw7aJl7vPB7XruSxbi8G8jKu177nbt4W8Q7IIsGvXe2mr1EuvM7vGUMPcHw8jvYNnc9ZTUWvM9hwzxT9FC6UJm6vGcQnrsQ3zy8rLbmPfP0er3kegC9+w8uPbWhjTtnRiW9YR6XO/EFiL0lRVQ9","PW4ovQk42T2RlTq95YYDOz3OjrxdSr47pldJPSgWpTqiOWe4Y6U2vWmpsLuA/3A8SN6hu0eY1DtKQRY8OChHvEiZsb0SPJi8TZUEPUonr7oXUC87XGeHPSMfaDw8Gbo8LjS4PI9Yrzo63ze99St0PEe1BL53bR69JxAUPS5CKr3uIpW8+iHeO3MLVzxctga5Ka6KPKWOwjv6wh28BjRXPfy/WT3nns68KGROPDa7gbzn9sG6QN67PGU1djy5shG9lgjrPNr3G7qOALC+AOwIOmlmg7zwbQe8r4KbPBLJQ7wzUoi8Qt+2vKbb6zzm9lS7AtgNPT9pP7xeEkA7KluLPB8c8j06+fS7+qhYPcpIGL0+Xxe+j/KHPADS0r3Tks68Cgu1PMwuzro+Z3m9Pa4Pu9X12bt+jZq8r+0rvYOxfL278ZO9GWuZPdZLtzzcBRU96w2ive1whb2wWfA87CQOPXAY0ry/+Bm9k+knPMRndz16iY09ii5wvUMXo70w4wa9TSYIvpAVJr2Qiu68O+/gO8szv7svyBc8DmgMvk1Eib0m65a7nlqjO8CCvDxqSTM99A0APN9ahj0KN8K8IlfQPGlb0zx1Xg6816sDPPaR3r3bUau7XU7HPEOSVb4k3p68RKnjOwYEmb0jPcC9+eBQvjvSRj2Q+YW9njpIPKNasrxUsry7wrMFvPELjD1rXv87fePaPfAvuTuf9Ka8CwjYvRZR3LwgNYO7jh1wviEpG70WRH08FC3JO8/xWr2bE7O9UpfaPLyHxrxRfG48Ah0fvcT9djv7cUo9iKkiPJ6mtjx5XSi+yXnrvEnhQzxJGFk8pxiNvJ8LHbsLpuY8sxekPNy92r2arMe9OU0LO6hQPb0fbg6+s1/bvfkMe71UrSC9ndGfveHcEb3S9US9oBXnPMGpgb2lqEq+zbmOPdImNT3hG0Y8Bw02O55Oejzs1F89rrKGPd3ppL3DLIs9iB2sPLdwA74Cz6+9OavhPAd9frv0uLy8JTVAvHb1JDvm2eG8","E3QgvD9jL76m05C8nQ6Ouw3XMLtWMXa+WSNqvMFZXzyh4SY7VsgJvE+3ez0/r4Y7xY2svF4pubtHtJq8wFmcuuRz/Lowlc27h2ClPBKtxjx8SJo5U69gvM9Zhb5QuyW8DTNnPF0FELxJ/wS7erQ2PWSxjjwkJ4w8AkrUOtLU3jwfjge80EulPFwag7wClFy8qWgxu4/EuLv2KGS7Q3+pu0Jz8brq48m7YvfNupB3Hzx5Kcy9xkI6PEp4pTsE4qc7fxAhvVhos7uU5ny7bzeKuuTPIr5kzYg9U/L1O0EvKr2LApQ76aD5PGgb6Lx2faO7huHZOyd1jzy1ncU9FdsKuvlKuToNayS+lZ6VOpJ+FDxcCBG9CIhUu5obWTwZsW26hw68O/mzi72IJ9u7RNIbPRZAQbsAXtY8UoFyuwJaEbpO++S9d3I3PD3+QbzQJqI7DN4vPcUter28TCG8QgwKOuPFEbyor4S+BPmVPJeqQjsn94a8EgDyO8n5Ar2hwo68oen1vdGw1Tvr5lU7X2AyvPdIADw1OSW8VZIPvIlznbwUMKE7UHIJvPAENrvHfZU8vRDfOxxDlbxQmTM62e+hvXKblb2P4Ja9Dxz1PLcmzjzCLck88Hs+PLtMtLxY0hy+pG3vu/lZfruN8se6HZWRPPz9cjvznqY7Du+auoEClb2eUf661CnPvKIOzbvKIrc72CxNPf+74TyjTDk8YO2yPCf/oL1djDI7v1I8PGvnEjeA4Ag9rGrXvDzgKr2TKvi7DRyiPN+iubycMTY7jdgRO/rFxjtZzSG9Bc2wu6OFSbs/Yry9udqPvaq4rTtTDpK6Ss2OPCTRNrxIN1a8w+6SvK8GLb3/S1C8SRLDO/oRdjyslhe9FH5yu7MvEDvxMxY7Es7+vK30NTxHqsC658wLPGTeOjtFfNW8tMcHPZN3y7w33X68X86OvFzOjrvYrdu6puSBPffn9zxN1CA8znKVO98/kTpop368wczMPDLzU7xvhtU8QhgbPZLu9btDr5K9","GZp+vRdPUL3jQMe7v2THOxNd3LzgOIC8spyhPHyySTx6/Fu8iD4APZllebxLXcW9SIaBPKVTGLyfltS9EKEpvDDEL71QpwC8+3MaPETAPzwclrG7gEvuPAyQljwxua88YQ0BPLeEHLx5lQi9PZoPvQrZ+7ugHf08LAZGO/VVLD1O3Cm81Dm1vVANCz1BhV8701wwvPeWKjwr7qo7KQb7PEix0julddu7nfXcvc3VmbxjOiK9EshoPPOyy7vcpo69FJVxO+jpM7xM3Ig8JADePJZ95bvbsiu7vxUwvSNhwrzAQKG8MJgCvZaQUL1ue5M8ChwbvGcLg7zX4qc8JtYMPKwV5Dwbq7y6II49PUWqJT0YiZ08Hk4nvSXbHb1wo9g89taBPYj1ejxFWQK502/zvG/9qT3KAWI8KUZEPVZOADxM0Hi95Wj2vN+d9LwBHWO9MLO4POEqsL0V2Ak99A7SPF16ezzQlRM80tY6PFV5fr0OykU8mIsmvQUTlDzlAfG8aVO4urzXpbyj/2O9MmyzO0q30DzwUd88FTesvZQbhLvquIg7iZUlPZH9bLzIHpI8hajiPaplnT3j9Fc9tgdEPK87gbwrIaS9W2KhPBYijrtMzp89RwtHvfHGy73ciQS+BAeMvZNyyD3AD9W8V3XFvZOVMbrzGJ08DbslvSjlqDsZuIu8MzyBPQdBtr0C1O+8YZPvPLzKhzwTmM+8YTa6O8Sz+btf2eU8H9MYvTpMzjzudQA9gV+hvAhHUTxEgH28ebWfPIJb5jyxq3o8YQ6Jve626r2YuV09zbTOvJNkMr2rgvy8EXYDPW+KZzuHLHm6J8FkvJpRIj2G4iG9jfN1PUMc3bwlUSS+TzaEPSGcNbz2QZm9NVGIvb7dsL2He7E9lUuIO/OsCr0R2Xw8LQQrPP3glrzmylK+t+kfPdLQ1Tx9Cku9HaQfvZUNfj0P2ZM8uoEMPA92XL3qkY69jtqFPIVKg73KQee8Nu+PPACuuzvhtoe9NQWlvax4gTgi9BO+","a1OxO3E477xS94K8nGbROyn1xTpFWk27NB1Ou3EPMLung9u7OfXCOxgNEz4hbn27vcIQvxRWrjv0CSO6d2rROoUdvLtkGz46Weo1u0OrMDsIFHC7d8iKu786l77e9hA8ifWOO51Opbqwg8i7i7zzvgn3vr0QJew79+eyOz5yg7v+kCU7gP8OO076nLv6/Ku7WY0QuklZkjmALA27U9cFO4DSN7su4HQ7kgm/vlz/GrnXwJS+GjOIOmh3MbtF0Hq7CLPPuw1RLDuArIM75On+uRYEtzz0cq85nlXVuqvLor2c9X47e1tQO2Q7bjvReXg7i5lnOZ+zszxcX9O+uS1GOrZuaTuTTKO+q8K3uvyCtDoL15+8YfUNu0iqBrwZ27K6VLYDur9Xj7uL81g7QjPIulwj+brouDG8nMkROwL4KzvRsn6+yOWUuZGdM7XKNUa6RlXwPZPfLrxsd5e+KWXoum7ulLvoPtq7kyuduz/NqbpqRYu+96/eO3WuZjpcea++O3efPFdJl7u4l3q8X8d5OqI2TDtDjig8xQfDvO1IL7wveVM7qz+MPI2YAbtvy3i65mH9ugbQwrrbLqM7ExaUvuxX97ujreC+nYcWvfGF4TtiwK+7JT48u3Xc5juZbuU69fpeOiMBcDshlzm7vrkdu/A5TTvn89U7dSOauuzTxL7MpR887gFJPNHEXTwlsL67pet7vFvGhLuBLN47cskzO3H2c74LRR692c/SPJAatLuGfz27jzeDO3XwDrwz5Vi5hJJDu5K0QLwwFw48zrwjvEGLyDwfMS+7fJUBvD1c27qKzjq+1LSvu6vglzvsnVK6/QGTPBHegTtcbEo7yWP8uyrLwjvDU3e8UDQQPQKTFbq26tk7Ca6avDYsWTw17Tu8A5DHO8oK1DsxPUE8316Cvo0Eo7uTJIe8kmarvi9zU72Kogq7koMaO/DgD7y2YPo51RUcPB9qa75VVI08A1YXundgPLzmQ4i+Gj0wPI0bdTzdS6y92vkhPBvahTw44SG8","XsAhvJ9ForpWL/y8nLSOOzZa7z3BSia9+Dzvvb3WnztQBb47xg2bvDIpLbzRNi++VE4yOlmQtzszLQ08mOhxPDgoxrqi6Rc9C9/fvOIdgLsy0a+8wzJivSQFljzkco+7tWQNPKxzGDuYFi28TkVqOyxEDr13PqY7o+2Mvdt9hzyjv3k8wbwQPK6jtLkjBYK7Hn7dO7X69rv2fC08e9tqvg41h7rW3m08SxmxvsZ2Ab7DaTi+RPEWPNo9nzsYVSu8hU66O3yPYLwcK0W8IkkPvEDpzr6kgEU7yrYzOj9g4rsnrmi7zDcNPNHVM75tqwS8eQagu8ELhbrCX4q7FXAsvHzSYLzicWm9psiHPBIRirylp4I8M1z+vB4tuLwJix692HIGPagOHz3zFiM9CvPwOFVhhrwPtAY9RowPPHiEaLthcF28B9A9vGqAFjxIrpu8ftZvvJazdDxSw1G82/UGPD0BoTwtH5C8pY6ePBHPbzyU4wQ+ld8OPLnZMTvJlaM9hB98umRoZD2QCDI99HpLvPZFGrxLLzY8ANwEPD7J3LulBxW85QATPmEKbTwFFGe8V3wfPFBGGj2GRty7jM8JPW4NQr05yZK88D4Au+N0SDuqjei8QegZvfXR7ryuWuq8J2U+PDnWpDudmYY86eErvKrCGD2qNq+9AqUjvVTVsrw28dK8YjB7OxabhDujLiI7t01eO53/FLrJEYK9yLCwOBC8Cj3jXhc9fjTSOxmUCD4Cyqm7AzPPu+5Q8rzYy7u71RtlOhk7iTwWsbu657HWu7m1NTwgfwQ7RYgkvLjI9Ts+C5m8aPg9vB0Jnr2ALcq8/mzvPFhc2DuyT+e8HhwJvL8BOL3NN4g7o8D4u0OvE72yZCy8/G4wOhfvVzzQD4C8sNjiPf47GD3rwpK72/KJO4W73Dwzq227yaWbO4eo7ju+S5S8V1+1urwnL71z7YS77n0bvdT2wzupitm8+40aPTgFgrxUB4G6gMAkvZ686j0GUSS9sZOqvQkpx7qqdpI7","pn7kO4naH7wCYXE8ukVHvCYg0rl9IhC7a3W1u1yugzr//ie+kULJvU4qETtiFKI68ghsu7EeETxeHe87r+jJOlLzyDsZZrA7pL7ovDT2trtRuym8rh1xvPklCbtmshG8JHG+vuTzfjpgvkA69ZYHuzON4DsOXOu6p63YulfsczyM8oq7bveIvPT1Ebzulwm8FZLmOuVaRLvFvVU7rpPkOpp4pzoOFw+8ntoOOyZ4K77wxAq7FEfsOx7F5b5GFOa9RO/DPJsI3buxWRC8La2OOisMtbti3z2+b97yO5LAALzOE4k7FhhZvlZBUzwyVEA7F0M4vm3NtTtm+DE6UMZ/O5IR3b1500a7WUWzvtb9QLt0V8a6Uv4RvN777b46hWe4wHqjOiCvK7uR4nS7D0RWvhGMLDv2dXE7temDO8tV/jm3ppa7nxyIvAXHVzyVd/q6/XMIOR2r6LoPEFA7oSf9un5h/rtTICw86oqyOgdk8Tr+K5I7EtEEPE27Ob7tvZw8Kznot5Bpfjq+nUy81U0LvBCXWzupL0876BmauhER2r7jHIk85erjO9MWE77PNTq9y0KHvpERCTwIqyi78SrHulpMSzl7yUk7cn8dOiSwHTuAQKa8Bor/u2BS6DxK6Rs8eT6JvHcPNrrgBTO+28b7OmfJvbqBKXS7tv+POFcz5DvTV1C7T+NrPG7UbrpWiAe5Km+yu+ZD37t3oKQ8Q0nQO2YWTjzKI429T3ExvXH5ADo1GSi8xEjGPL5tGD3ohJM7T9C7OyV1UzzYpUO8N4gOvPNICTsKMam6KdO5vBBWVTyIixe7r82pu+92ITsZckE81MlPu4ITZLxp/647MpmkOxbThDu+wcM8AZXKPAJ+Qr3Yl4K6C34kvMj0C7oomFk8A1zuu3I1bzznJ6M8PcTVO8Jr47xhb7m7AQmzu4cw2zv/1Ig8SpvYO1RtPbynzAo7d9GbPFEnhbtGwOe66FfYvSapR74DEoQ26sVfPItMGbxmLcu7c+VPPEmrNjw7QNO9","+ggzu3699zvjXiQ7OLyCuTO6mDt3vse6auKBPL08BjubLAE8XslGvBgpHTqCgiU9zcpxOihiiTw0t9A7wH/kO3WyYL3oPmG7xA5CPLK8QjxzUzu9IAzyPFsjZ7xamtA8iMeCvNeD2js4u5I8TsWPOxPVNzsoK669q/LHOqEKJDwB9ji6urWBPEi5pbnmeR08uNzkPIBKGjwQmCM919mhPCTCaTxNYVs8pyR6PIiirLzpfg67PfWavPGw3DsQ+Ws8iFuRvJSBR716i708158EuxrlrrxHsV48fmJBveEUez2wr488CPAJPN7E77oExyU8q6KNOxloQb4khVg8VXfKu5HM3TtKkY683FrJu25uTDzW+fI8lUgavBXTars4Ioy8FasMPGsCT77C0ls899pmu7qeGLwBDyM7//JyvxvaBLo1We07NpWzvFN1h7xo3EO9lJOGu+//2TwV2Bc6J0mJPCs2Jz1MMrk7eoo8vEY9Ojo4rbI7Aqf5O4dAPTz3K7m8YDWXvMqmjLxtj4y7+lbxugGcaDsKslM83+4cPFNk6Ds8VnS8NC0fPDVYSTwACcq8A2NAPBr9hTvvVnY8TekAvHCFUbymsUO5+ko/PTXsXrtC5M67sFF1PdVTzzsHCGS7jaNcvAP75bujG4Y8u9Ouu6b2CTtqYp08wWoLPAzjpTz8LLA8P2LZOy/zSr925na7fDohPFnuYjxoHdi8faOtOuWCATxrJMC60xsGPGyevDuISaw7kyQDPFxCljqiqQG7umyFPDJhb7x7kAu89p/Auwc8srwYbAo9LLMKu7Xqibv04pw6y2QYvGDQHjza7/A7uEeQvLimVb2DRFs6TQdmujcGp7ySER08H3Q6u8+1HTsv87Q7gBXxuoyrHDtwB0o90SqdOs9R+ruIkuW6xUz6vFrTyLwcoI47yoEHuiYovTxuECM7htDSPKLnDzw0NiC88CQIvSnNajw7Tze9uL0JuzTLWrzN/AK8rLIuPJGq2ryt/Wa8JAqtPA0dAbyQ1QM8","RAE1Pj3RcjwWgJk9KWHNvf+8RT0s1+m9YUegvdbSzrxGDeW7YKGgvd7L3L3LdME+JgQ7veAtkj0GkL8+2RXzOJ9jYD4A7pa9yTS3PaUrvTxSJRE+GIMlPqptMT0rWXI9qHoivryuFzweS+y9z18OPt3tHj1ptpM88DjxO58SN7z9aoE7i+4nPYi+Db2GoHI9QVJXPV1orb00pDQ9SyzMvf1btDyitQ69KJWTvaQIRr2zid896xyLPesnqDwzGIe8ZvQzPviGajySHD+8lSoLPMkzpbpnZYs7UxL9vCmmJ73FjdA9Cb13PbzDyTx7koO9/BiKPcUAkD7tuDi9BJ3wPfwlY73FHtO8tQGIvMMjZTzjk609NUe2PFhKqD21VAy9AjdgPQGvb7zjp/c65GIbuVf3iL0Fhyi9QituPR/Qhz0WNHQ8c+0NPoAQZDz+SXg9UYOAvdLuAr9D+Zw8VOkSPpDkCDy26ZG9pIn/PcR3oDxjeTc9o/t8PYcoT7xUYJs75/PiPOflIz6VmoW9TaNPvb4Z4L08d7E9SbWkPY/6Hz188oC92l9MvYtNlLsud/C8z08nPOhvd70J6ls9xppNPeVssT3bGh+90q6nPKcXhrrcJIA9mClNu/NtxL2iw7W8h/2RvNkRhbz3CUu8Yws6vT2H/77WFZk9XTFMPKOuQT1gtTY7WHNYPHcuIryM0Yu8XAPcue1E5zuJjtM7ZJrWO96yHTyk7pY60pNDPe6QgDqzzmK+up1qu6yeQDwX9j+73RTtOoKPZDjnykW8nIW7PB6cuDv93bE7DpT3vVJzDDxEk0C8xc+OO3oUbjvFA42+icgFvUudn7sn9MK7F879OiSF9btYmnY71NFHPHxuNTxWLIO5W6qRO3y72rvZvUk7Nba7O4zLhTvECoW+2qJnO95NF77nd4M7eeWuPCYx7TvqLgS8/6hWu9JH9rxPQu06iVqGPF1bk7zvT+Q72DfXvkuk3Tt3Upu8d3Amuuo/QDtMDAc7drqTPEQgxL54p4C7","oys7vGZnj73G3CU9dI81vE34ory2JWM8szIyvJXkJbslrwc6ZFCFPDTJsjmcaBw8uzMfOxFOUDwHJgG7NzNKu1fpkb1Ks5A7CU36OzjbuzkfV549RnbDu8bANb46Lwu7TY9vu5WEzjytogy8x3/nuxXkf771De88VD0bO8LAxL5isTk8F0fju0NoqLwyVUc7P8tFvPCMnbu8RWK9Klq7O1bufjsNmcq7TXVKvFp9N7taFYa8m4viOzvvDbz19Ga+wrgCPMbUQb7h5Ka9z2fLvCP4Hrw3vva6EHAyPNEbfjyRs1U7NZtsOtfFEbtvbEW8/lzuuzIAxjwvOaS6v7+ivodD/Lojsxa6LlAauzpL0bsIuN26EQKBOzm+NTz+piC8uMeHvjY4aL7fUQG8RYqNuxa2vLuW8tc7OsUju2EGwTlyA406T2vhOwCmHbtb7fW6+HB/u/5zcrzPhkc7Q+XZOxJRkr675/26dXYWOePXyLumO6c715VEu7hRPzu/3b27Lxp1OX00Djx64zI8ZFHEuzQUhDsSA/m73hT3uoKRNTuRmd86QFlwO2FTgTtoTM2+J/j8uhXPqruldam+9lTkvCmPkLxnoeq6K7UYPFy0BDlDX3s78Llcvq1O2DrFL7O78GjcO3ckxL70BPY7aTzwu5z9pr6UmiY8nWFBu+6UkjviVNa9rINuOzNcjb2FTMI6amGOPPJRf7wKqti+BXS7unkxaLtrxQ48px+tO8X7cr1J26S6e1VkuusOQzpL5jM7j7gDPFPtzrvpMXe63S84u1wRuLuJBmI8LyjtOlc9q7mqBxO7KG+Ouxmh0rzjl6s6kQknu2IBNrvp4zy+8bZjvIQT4brgkhM7/+rfO8M3RzsGnJ26CcA9uVmORTkw+q2+OfuAuPccXrv96rm+qe9vvQrtXb5TAoy4MEi8Ox5owLqJBwS8+fMIPBi5AzvDoAu88mkcv6v2jLvxivG8a0c+PKRGsztIbNS5AXyPvsf7JjtJVT88umlXOhmpijk2fgi5","AOpQPETswr1p0fQ8E9rAOxSZ3jzel4U8AZAaPIEsmzvDnRs9np3tvcF1Jz2Wb2I7mQ16vcCXnby1Lc++UFMcvPanZbx0r3M7CW1KvbLR9rw7Tca7CyNePYzLLb36SnG7z8tBPb0SBT1LPqc8NpLBu308g7wLDi24pv9iPKobP70XbZa8vfRzvMJh5TwDioG7jN1gusV587tu0rc8nhJ9utV6k7zIB5o8taPDPCuF8ryZ7fM8YemKvOVLQz2lNaK7DuUjPZN0qTtmdIc9qkxdu5hbET3+08G89hPROxHShTw2MxI8qUCMvNLzzDvRjU68pyCaPUewSz1KIDm9KKKQvB+3WzxSd4W9vX2nvhOJBTzEixM+38GtPGv1eL0jEQC89h+Ku1VAFb0eAaE7+htAvCb+SLywApg84LktPDpU2zzJmyq9g/jfvHDj7DzyeLG8k2Jivf3eSD3Qw/g8hT0YubM37jpxPuq9jdPUPJT91rwRHnO91FKTvR7ti7x1JSC975paPd6igLyFeU+8jHTEvL0pAT2Fdni7zw7PPJuvbTx93O28pci5vBG9abwMt8q8y9FLPWJ0Jr3uYtM50d5evatEgjwjxSO91EQAOpEmXjuPOic8Ye37POkzv7xFkCi9M6VtvQHU3Dvsddo8Yr5Kvc7IOr286wg8hwH4u+rdgDyYWs08H2wwPUD8pTuJ+zY8Rqz8u0c4DzwQZJE8L4ewOy5sSr8mI9e8qUh0u2wW9rlrF1k7t1hIOzidxLs6SVq6VdAHvAsptzvNXh6/F2hNu2rqEbug4Fu9ZSk7vHQjTr0XZdi++PnbO1/jKTyjjQS8IKgMvFpySLv7drk6l0YEPMgwXzsi1HA6NRW7u+FAajvahTO7UQ6NO1v3Bjy2VAq7A/nvOuA1j7u/mja8lyudvs/ClrtRfwO9dja7vWAG9rvwExm8mtOTOgHlqbrWgxg75BZWvHhqx72qKss89ADiu3v21jkh74y9Fk8sO03+v70wNBu+LRNcvKGBe7yWN4k8","S51xvX8vdztHHhW9D1zFOBI2HD1kfqG91QukPb5h57phviW77W8Uu6bqSTwk81K90YiCPLXfiLu9I407oEMxPD5zsbtyeXe7Ja4gvmPVOLqS1lI77Rghv8/dLrzd/rK7HKOKPMTAdDqKWcG+RxdvOlEFETvJ1z07zOkivfgUn7vrX+k7z9bsO9b1fjyjmA+7EHTSu1II1zvKmHI7sXgUvVRqXrvU4C88kAVzPbrp4b1T4o+9v5CFOiE4K7uDv+A7I6eAuo0YhTs6vIW62o1fPNLKPb6I1qQ7x4UbvsH3kjs5RFk9uxoBPI+bxjsGWhw8ybNBPCEzpzvogxW6GiNiPLRhz7uGuFo73sGQPL4c0jrJxR84NOl3O29mA7urzEO6ByoCu+JWATy1cHY95ZYHO7cxzL68aBg7g0jduV0B9zlWfdK737mKujFnazuu6sg50aiCOEhJv7uSDty+v/iYOrOUp7oR4c+6yfrOt0U2GL8VdKC9tyKgOt3QKrsYFny6f6ExO64F3Tu94uu6D7DyOsyaA7pgeg056o9XOhyWvjp3cq87KwZuO8d3CL/f6kS7OMTCvpTabbtHTs67RYjWOroKuLvHJDq7khSWvJp04rmRhQM9LZwLvPsaBLu45w+/8FsMuxMNAruxFbo7WKodu66THrrdAjg82p6jvhT1DbtL3147xbaGvmQzYbwV1nC6ByhiO1g0ErwnsTA7KgUsublkxzrOnMI6QH7BOer8BTyqaOq6vN4CPAHGB7vEn2a7jtzhvjBfCbol1ze6fYtkO2WYZz05RHq7YQI/vttiUzt+MlG7a2b+u6HRhbuO/K06g7q4vi/9ZTo4DVe8sEfxvg9fVbyDv8y63jBnuojvKzuxdqQ6vDjcujSROL32xfi4EVHHO6OifTw7+5w7hcznuqPXczscJU67RQtZuvHWzb5l0gU9egyAvr6erryX/hq7JeJtuj4Cp7tQKCg8TmMJO+PIfjiClUU7IvoTu472wLsOdXU5YWpLu77IQLqQlam+","8+PGPKNelTw0ChY9+QzKPYunJrtCD5m86MJzvJZfOL3mCiM9LekCva48iz0NV4U80TnMvJJxOzwIJPq8n9MAOxm/+rxozlU8s5i+vNeXCb3hcrs8roKzPTl0Ijtp98m70rayvGLa+jyDkhQ9WIMHvBgCOzzFH5C9pWwXPd0mCz3qSBS8r9ORveVeZT3p2SY8siHou6iYpbzc5Wi832JUuviHsbvtTLi6pAPoPI6mKb2oUcM87BpQOzn4izsv0EM7woWxvV/oRjrh+JY9rDuuu28Eij1ccHq6pT47vXCrNzyLXKg8NlukObr/Xzxl0Ya87FGAvdoDpD1xefs8FYlput/8Krw09lY8zK2evZRiSLgL82S8E3wlPQHUJL3K1Ww7WnuivDBHVb31Zrg7lkBbPN3rg7oGYRC92j1FvStYOz1zVi69i34GPMifeDzhj5e9FmWmPBfIIL65O8K8xfulPNgGuTsaFF+9/AMNPbHPvLw66iy9NQE8vaxnwTwWdOy8AbylPdg3kTzwvKO7zyQPvBNjvbzckdA63H9hPFVymjwlTlu7Q68zvM6T5bzLDhC8cyA8PdMGxrtxFAG8e/SqOwtXFTumgSU9YjhFPG4vm7slLEI8RSudO3Cy372cxKS8XZmpvGKHbbwIQzS8xq7tvDoo/DvU4La8blLLuy3UXDs+7Ry8TO6iveCElzzslkG5ikaEu/8hG765bYW8vW6Yuz45i7uxOBc8h6MBvZ8qEzrzZB+9NmwFubrIM7ktjjQ708yNO0YGzrpM2AW8OAubu8YarruEevq7r97kvQdmRzv/GSk8wL+WuvpCtbuaid29816BvDXH1TuJ1Zy7VVPsO69D3LphdMG7eslCPM/wXL0qLjA8qmzzOpw8jTs0M4s8fiJgO8fP2juCTwC+ZzvQuvAynL7ALx48Z0eeuwZH6TtY1Rc8EtLCO8DsjLueT3g6zUS9vjIKMzvcRVS7GKKfPDAohruMRKi70a+yO7VZxzzw10Y8RBr+PITucr3E5pg7","BX0OPD9CSr6tprE6ctFmO6sGtrzP2Aa8v3QrvOZ4eToTcLu4+UISO6+nzDvS7So8QbgWOxqLtDsevp47Q7bKueOp0L3BdQ28A/YVPANucjyTn5g+HP5mvGWnDb5WZUm8xgu/O7YmoDxcDuK6mQAfOwsoZL1ITKq9+/BeO/58XL0dh4I8V6wePNwvh72oENa7ihs+PCTz2LtjNlO8iaQFPTDvibs7nqM7CnituzVrbTtvxss7hgQwOwrT6TsC9aK93Ko9v/nCg763b5W6Xemeu+bmIbs5B448aIHePL5svDsp6pC61FzWO+gAcrvl5Q88BcrwOhww/TvfRyW6dYsNvSxT+T1ASTi7SaMDPVGE17sOuNs8e9CsPEdoGzu1Hjk81WRwvbu5JL2rr6c8foPXPAytKryj3YK9BWBSvWSxybsVSF07RB0euzB/hr0uUhe9n2VsPbZsE73WJpg9/mgDvFCsH7wWNM48yWf6u3tyez0sYcW8ybqHusuoHj3MGWi8QWkAvNeMN70eqsc71IrlOr987Dpy+FQ9FUwLvXf2G719hIW7MmOGPCUxFD1GeDi+Po/vvBdEXjzCVmW9kkI3vLjz+rxX9S494mq2u7lZkjreu/C8tFDHvfJUa73LSks9QVroPNFcML00ewG87PjVvFrdnb2PGVs9pvq3vCk7ST07lj888AlBPez4jr2VlT68d+SbPCjR8btrRo29tIE7O+LWdzyKFyQ9bZe5PIiDvjuHp7I7ZzYru49uHD39ePY8U7hZPXojP71At7K8vPPMPEMmiL0bd7+9Nf4SvOLOMLy9HVm9Hg56PBvnw73eegs8SLyIPCHlN71knh49EYawvOHGKDzGNCG9qFsTPT3pmrz/9ys8WuOSu4rwvLw7StO9stRGPH4dmDuGIPI9T7Wvvcx3Gb2AfRS6Wj0rvUZlEzofZgS7eiRHvXTc1jykCvM7G6/KvRv8KT3FLh8+EjKaPWo3EL0Yaro8j5IyvTdbxjsC7j68EkcgPR1AJrtvelg9","TlwVPvUh1zq2mOU9EXpRvkOJdr7oYDC76gwgvmfu070RydK8wP+SPI944r0FuUq+gga8u20AEb7etv+9hYJnvpgzRr7pJxg8OKrWu5OjnTvTUk6+C1yYvbBZZjxn/mM8YqooPMKUOb729K+7h9liPHkpSD1lWBy+XUHevTT9aT2SOgK+H77tvZi+p70O36O5+KxwPGcrGrwYBR6+9XRRvtMvyjtd3si8kq8+vFTXmLwFx/c6f0oOPjmMnLxx2rc70+3MvORYCL7GucM8F0nfvfX/Rbw/BxU9Ppzevc+uyTyFnww8/UY9PbRJg77/CgK+YusjPM+ogb2EyNg5mlWfPAMdL76225I7qKU0vUbgjL2d+Wc9haJYPJWwSDyuDEy+4FjfvRNt5buSZdG9eUKCvBOl0Tvc2VG+3rHlvb3yH775tr27+/FxPBuwXLyopJ+95DfGvL0SMD0doSg9tZMWvdHC673KudM48UKNu4KHMbxu+g68uBjvPJJ+oTwCVj28iblDvkzqgr6lE+Y7L47QvfVS0b0HYji+GeJavr91nTy/t3W96/WWvcBNXDxOg288IS/iPAwih76cAuY93EQKvN4wGbyW/Tw99HgxvcqG5z24+0q7GJ00vkJTvzvj9e67UuhMvoxmeb4lBK26aTBuvYOL4r0rsHS8HY68vJjFLzsIDHC7F6NMPANm2rahZzG9c9/+OxszD7xiSIU6OkZYvHV2PT3/8uS6gLWYuzRkbDz8DS0815favHW42jyTKTe7GMLgPHLCrbzhzo08W79rvEUjtTw9cye9n/byu6BLKLzipx2+xAgOvTgQhLx5XeY88kYaPTp4rDv4nsQ4kx2JPCvhAb3T6LC8NI5mvdAJbDsRB7W7mAi4uxDQQbp1zkm80z6evMmKH7xwI8o7Tjl+PdWy9LomzfK7cQ7CvJ6aAL35ED28zVEnvcrxqjzPu0k7jDiqO3LXMT1TtRk8L7+rPC3ewTzjQKq9WcuEPIeUFz1xPio9slqwPetr6TzfJLE8","kurivXBgiDwzP2u+XP52u5ILUbxZ7BA8XYtNvmUDyzuMiKI8MhlPvExH/bzb9vq9wyBQu23Ksbp3n4M7IuDFu4CLIzyQaE69iBJ/PB6EY7x4vkg8gLpmPNpdjbwL+JA7cUVlvE0sY7vdZrw8wNmQvHJJM7vXG5O8qMfNvKnuBD20mha8zQsbvGlJvLyTPpe7xCpdu/zss7us9tE5FQYWvWFtBj32DRg8yVOQvbrVJLw7Uqu+VSrOu01oubudJyE8mqMkvOsbhbqGQSu9lb8JPeNNW73eaIM7ttF7vZLHFDoArio8JwP2Ou7oODy5oqi8R72WvKnaUD36qzA80V3BO+DjbLxYYKS9kd+1u8b7pbxOwfM7inZfvo73Ij20c6k8gLQ0PBbRIbpS9BU8QuMdPMqeP76OtiC8dlivvBAJDbtXEaS7GKNpO4sMy7wa3ZC7IDKdu4cRSDyvPxG+5VKNunaXGby3fW65ZJ4dPIqN1buKGZA70lfvPDxhebs1LCM8qaIGPK57CDvYagM8qMbZuw8pVjmKZoC7yVywO5ZjfLrecOM7wlPOOlfLaL0P70q7O26GvqIuULzVwna80h4UvGnYzjy5mP052xbDO1cw5Lqa8z6+WnW4PHSVIzxBi586A+pbvKhcATy3ddi7Tud3PAyLIDyv9R49nXU4vgcP4zs9aTS7t6gHvjO0XLvp5z48IdNxvIs26Tv+04o8PTFGO85FazwT+gS9UO0jPPZ+Bz2hUSO7PaUfO3ZzPruul/G5nchkvq6pAbvbJlw7aCmWOzVpEz5kmj+9NneOvbYKjTsuK4+7swUQvgevzzuauTC8lD2bvckIfjwpfbA8idUfvhg8x7vwQWU7pNLcOh3Iubp6rCU8pj0svFnnoryE44K8s442PC4XBL39BDS8Y7VIO59k5jxWorG8BmsSvO/ACr6fFwO+OSpWvtOxhjwhMwA9WbhsO6g6IDyh0qU84mEKvso2ljvMaho7kjujO2AUtDskN2M83/3rOsGlJbvVXxW+","or1HOXSSZL2+ODm73VeSu75E4TtINtG8HKPlu91agbqnhco75Ob8PAA8A70ERqI7yeUdvG2LsrtfLDW8HK9ru89enLvJqwS7gw67PNFscrxntPY6vbV9O7+4XDxbn0u8vgoRunc6Nbzra407z+Kmu/7+Kjuo3R07hCcrO6Uw0jss0KU7OG4EvDUbG7xIbgO91RfbvgZtcbvJAR48dVi3ujwtoDxCBhW8c878O6CduTrlU6e742kLPFGUGrvlzKK6f6PHOMTZzLrUfwK8OxFwOgtDwrwZaNA85ZnkOm0XHTw5LbK7JwMmvMXpSju3pnm8X0OUvGgRtzzwP3i8f/XJvptHSbzDloy85vAHO/nwarvLehe8BMI2PPNiJ7zDh1G7D2qrOxiCR7wWSVW7sOLdu34cATyGNbu7nAGGvSsFgjtdqom8kPNwPZ+bhL2Qqke86eeiPXBFuLvDez48BAh8vO41AjswlbS85AUjPQnel7trHU68XhMlvOcL07x9Bvs7+nqkPHpkaLv4mYG9tQZLvFbfYLtRbae6qWucu/UhD7ya9A48ftIePP3IEbzf5KU8KaRROjAWkbp4iN46qS00PAw5KzxsQmW8NomwuzhKJbu1oB88YNQju6nMJ78aKKI7RPBMvI63prqsLae5hj5UOoeTZTt4Esi9+eIhPRPKq7zhosQ7pK9ivS9ahjtPbya8vEUsvGPJsjvfsIa7NuLCvHDDcjuRf/68pDRaO6oHyzzDa9G7R+P6vDPOHjz694y7I40fucfsurtTSJG8LuOOvJsCGTzBzHI7JMOnuxiVHDwTzcU80kDmvFTLZrzKgkS7gIe1u/jSNbobWKG7vkTXO1scwTuZP8w8pqvUvBiZSzylAB+9TLb4uW8PS7yU/TI67eUNu7K8aDwvZg48JuE+vdJb1ju5GYU76mTovPCZL7v5qpq8cNrGO71xuLyYHiu7wnWPvBX+zDtPE3O7MSbpPBtXDDwLJqq7ogpOvIvtdLzL4Vi9JvAcPVD+cLwGl586","DTrZu8a7Mr2bdDE8HJspvP5oo7zII8Y7gvNHvD/qZrvvDw47f8vSvGLumrwQg4a8M4UFuXZwGrvBY2K8MRRCOztcljw4onS8j4lLvGSoqznE5Qm9x3uYvXQZ/LuZy228HJ3gu6DriTwVUy2+BO8gvdwv+zwxGi6+TRJ7vEY37LvJ4ya8uMgUvUYgILteUwS8R6u/u5kxM7z54nM8alTIPD5rrjo/f6e70DRKvHskubwAb1283tsHvJKbqzni2JI835gLvN+O+bugp7+8w1FEOsS7JT0RM5Y2HnySvrMfwL3UHps6SYi7O0oH0rz4yKS7jtsavHEBB76dWmA8JAKXPI3bVjw50dC8tVQqOxE0Qb0Cl/c8wMWKvRfVBzx2IVe83xQsPAtChTx5YWy806TrO69zKr1fF1q4Ac9qPY34RDp23Jg8MlxwvN04zL3kKBu98o0kuxC+hjttqr+8Gs4ZvXEFCL3tU2+8YMeivCiitzwDll87S9iRPGakC7yvNyw8/EYdvMJMqzwCX9S8PaCHvNsSYLtnPVQ82RXaPDTNbzu0OU87xlN3vHmWlD1ESww+Lv+7vU86KLn99Pa8MwFyO9R6m7tjzUo7qO/ZvEFrVztHRoa9cAG7OzOwVD1tKbw8L+pHvFVHmbw/seE8/PlJPK+qu73mdgO8c6EqvYi2mrxg2jE8zQCJPZxHKTySbf0758xVvBJ7ATwrWNK8Speiu805bTzUgPy899sFPUrB3z1DvaW7+Ai2vAqc9zvX5rY7tKYtPTmrZL3o2VY8wS2YvAFGJb5M/Dm9N2DvPGt1mzxQUcQ8rqPLvaYwB74w5hG8Qx92vQdhLb7q5qQ8E1EvPTufdLw9eW+7BzzsvLhTj7yD6kQ8W6zqPKTSmzyBPCA88GGyO6vjjTzZXIq8VcNKO0s8iDxMkZ26MPxkvOm47zz8yL69Bn8tPfpDLr3rqfU8OJhNPSQ/UjsohuK9K1gPvncUjrx5vYa8QpbLOla3CjwXhCK8vURQvCKb/DsMEoe8","1qzdvPROl7yQy1s7+tu8vM2vQbtmWDm8RDFoO2pnMzwsqxa+GLV9vkyyC7wJT+I5rAydvARzTruQE+G7Uwy4OsWWZjz8tLS73Gxrvvyg1zxmEda3HLRGvDDLxjyHQmG80PSCvYUfATzr/VK8IA+NPMEkmbw2sWC6/xc4u1GjxTxn+DO7gdqWPFf/3LsS+x68uV8fPLrEhbtGUbS7TKd3u7vpW7sCSCI7dymAut+5573b+Fs7QYQyPDE9ir1X97e83u8uvLbgZjvlw4+82A1xOxwuVT0lJRW+vfWdvLs0xLy4nFc74i8AvqT2hLs42To7yOgevvxS8zwtLWi8wGtROyqJWz1xLyE73Pilvd9N/zvxepU8fH5avBkIlb01lRG7l5vuO2gqKr3QtEA88QqsvbroNLyTZSc81uJjO5ZZk7vl9TW86AEVOkKigL1bFV07CCRzvZP6gb5v4mK8YHc1OTOwd7x2AbG84GY2vlEoRLxo+dK653UOPQU/q72xbrK8gVSVvB3w4DnqPDa8bKq+uvboKjyIDEa7on5hO0XZuTvBaiS7eeVdPFalTD3UyRK+hjupvaWQp7xccDI8VGIAPH7dgDwaJqk78zo2PHegArzrRMI95YDvOl5rfD28LwY9tFU6O/Ignjxc8ju+FC09uxAXJj2vi/Y6Rep8u2aujTv+U1273lxBvYVfLLzxqC28bOCDPXu13rxuhds6JyBCu48+SLwSlDe9LubQOrPSG7tD8Lc8yn/yOz2g0rxorUY8C8MwPHlk4To3CES8e30bO5ClA7ynkva70gqCPEZmnDtlt0u9M9hLvHBUMbwAMcG7Wr67Oy2DFTpunHc6zkZrO+ycKL1+Ioy7uQrLu8llxzoG3m68u6WrO2zY7DtNyQw8IOXpvHLNzbuVIgq8BIQIPY2EbbxP7Xi7FgwCPTh2Rryf+oi8iV2zOxQFd7yJFCW5ZJRXPF9FuzxzrWE7oaopO8nYHzzKgaI7wWp2ux9xtLtvJpG881muPF6sDTyBu3M7","WrVUvNmlN7yt27q9f0kDvILgArxTsJO7dPkXu70xg7oNbrE6BWeGuYxzLbzdAVI8yzKPuTu04zqthqO8ea5Bu/zt2jtMpCW8QmRaPHV+DbojIQe8zAvDvGuAUbthtAA6wvw/PHF5+LgM6ys9jrUTvgUwGjwuR229xAxovYp46DscYWY8EWJFuuGGgTv0BeG7T3VevJ5iWDtdp4u88ukOvHlewbqqlvK7+h4ovHUHjzx+SBW8LGLRuwkZDLvPtbw6FhdAu1a6UjodWw28YlT0uncIhzzp+xo8qN/KvToSzTwa+Io3f1e1OtCQi7riDRM8A1duO+7YPjzQfcY8u5IiPGlrkrvvt8I8KXXeun3jQjy0r9E6wjoUPQYKsTtU/pq7TM3/uYsngLzM0T++Upd8vFK/5r31NSU8LgiwOzVnKjuS7cC68vmgvN6wvDwNd6K7DGHXu3qHFb3MAEW9CtvHu/9Nyrw41c67cpEDvYNZZ7x3WFe8oN5SvMDeRjxxeEy8e744vAR5fTyfAdE85o/TvPeDiLrRkIK7yVcKO7P7hbxZcDG7J8RWvI2+OzzL7jy7/RnFvGxnpDxVSni7RVZqvGKDi7zadPE79/OyPCXBIzq2MkQ8+y0HPMWHv7wX6dO9pwLnvQc0FTxju8a8jow/vIomyjzf3D099voSvbezvLpHxS68rW1FvJ5VeryDs+K6AB3QPTmKlLlk4Uy5kIUmO5Y4Wrx2ug28T00vPJFG8Tpzlio8pcmDvURvF7oFaeS79ZLSvBfXNDxi2LS6oB/tOuj43rw5bI08unjavMhLujpXJoI8bpV4PEC4RTwXfB+6WOOnvf5+azvGcok62ftPvhsd5TuRecY8N7l5vJVrPjxbvqw71LQmPHC1HLzQqxm93vlWPC1Osb0hXJw79TVwPF1CYDwdDbY6uvqEuk7Jyb2iG8w8q8YiPNWpg79On6s8h4gOvdOFwrwae5e9aZ0qPRWWrztDKcs779AuvPBeAr1fkKq57Tuiu1x12btfuQ89","IvOvvO7HnzwKq+C6GG+JvK7uuTxO2bq7wsURu+YDKb2eko08uEL7vB4r8D1noqI9mx1ZPXZHoT0FnLI+e1H0u3EuvzyfuE29UQiCvZlQnj3Lpms9D9Zfu4oP3zwc+0O7/i4wvW6Y/jxinqK9wmzfuRjdMD0XHig9Tbktuxg8Pjznr408ZteevR6IpLvLJVe9SDSxveREM71Cu6Y90G5WvKmnazxEyJu8JxWdPei9oL1a9xS95ZtYvCPFLr3CzgS9qWplPI0RnDxxudq8b7kcO+cfMLzE8jS8LUuJPTQlMT5cCGk96Q8CvTxTCT16KkM9LSWTPf0ioj5DOBu+wTDvvG+567t82Kw8EqjlPbdcoLz85Ai+H4pcO6BGzz3kZye8ySS+PKHq07uD0ZU8FBq+u8moV71zG6u97RoxPAGEDDy68fM9uesePv/Mf71voLE9eN9TvZBfCb7mfB69bPe1PYoMSb3P9+m9w1gaPsMZb7qjSjs9cwYmPsqstrwCNZ698VeHuwibNT36z3W8mpK/u7KS4DxMikw6tqTIu8Zve7vTSmC9e54cvQL7i73PL6m9Y5owPa6H5Lwn69i89W4VPZz9yDtxwAO+TA5LvYpSUDu8RZC8TdKsPAYU9r0Mg+U97uOFPFkIgr2IQR27Mpi7ucMSEb6q9Dw+HkWDO8jOKT0rk8G8qAwqvCfNm7t2FXS6FHzPvIvbTLwMOiE71KftPAhq2bzxRx+9+psKvUdPKztwrPg8yONYvWTcCr5bvAC8OO3aO/f14DwLIIK9n6jGvBWL/rocDR087WvdvJ1JWjoK3bA8fCtxPLmDGTx1I2O9IfCFPGU1F7w2q4C78vvlPMXoJzzuZ2q6XxtGveLPYr0Qaqg7rD+WvHMoDTxTGJc8YCSVvBuI57wRvC09WpsKvT2l27wM3227+/K3PEucfzwiB5c7576/vKJ14Dw2sQc7om+lu3tOq7tzVK28JdvXPH/V+jsK+Ai8rXsMvSwiqTzlNdk66IgvPvGXALxpHs66","+kvrOzu0jL34s2q91O6hPOeq4T3R0Hm7mLoNPRuXcTq15HA88m3AvHJX4juNwMy7N+dNvI1m0bvH6Rq9b4zhOreAT71+ql+8t0KSPENjoLy9ZAe9GiFHvem3hDs6e8e7BteauzCAPL2JEJs6BiiCPH/+TD105/S8rywOvKJsmb36cke8+6VoO316Hb2fsRq7EAAsvb/BbTxGHOK8L746Pbh7m7yJsss8mV/KvMTELzyRC4Q88gBEPIhuiDs75o48Fz5fvdxQRzxAmFs9yoaOPDApUr07yEw6TSj0vc/01rx++8q6lJC7O1KG5DvquKS9Bx0PPcSZXT0En0a6zpeaPBP64zwUjw69GBO6PP842Ty+Lxy70hIGvjWvgjwdpFW8LRLyPPGdRL3e5608qBNcOwZOPrs/qg29c9/WvcVKgTmRYIO7Mwyvu+e04jsqm8c8Q2SOPNQJGLtRuM67CB3dPJKS9jxOdzq8UJ0vvF4GmDtRU4u7nkaVvOFCd7yrGmy6it7jvDdRprxoMtu7cKLEuwsR/zvjWNq8ErmevLsLS7xVsWy7sPkZO1zPbzy28pM8sz67PDElwjt1+Oi8xmX5PKLOnrsK0Lk7jOL9Oz4Zr7vcsME8e5DqvEE5fr1xfJC936DRvHqbq7yOgY66xBqnu2kFj7uP1so9GWpwvRfGO72LOjk8h+qgPAr7zL3PBsE8FAPnPVZwmru6dcc89+SFvIk/bzzqggM9C4GOO2UgwTxa40+8in1aPBetSjtnUi28IPmYvffFl7rh4EO8CQqUvINr4b0PTPc9gLwOvKPQAb1QTEs9Ov3/uSEOHb0gDKC8vpu3vBImDr01xBK9CP7PvK2bo71N06O7ZxE2vE+TdrzyIWe8Br4AvO8VfjyvfYc8MqacPFuiEz3F+jE9CDiUvLwuCb0Xe3m7q87su9pRBD02/UI9eRiuvYY8GrxqUM87wc0TPQ+e7byc0/u9WeYRO1CABr2YRGM8hvGrvIkHObyDPVO9AKB9vAZp87pdxcY8","n+TevCA0Jr1T4+u7U/C8u0L6Mz1707Y8g3NYvR01QT0LHVq9qIxnvKe4y7yLcdQ8PSjYPN2JwLx3IzQ8wDLDOoTKGzytgDg8XRuNvg+pJT27JD+7v8FivQd+BjuZ67K6uj5EO/51M7xDzOm8DOYWPawwibzoOic8Ul+VPPtEE7ymKQg8xRxkvZfIXjs5kru9fY8OvaBzvDsFYBM752bSvMNcDLxj0Ea8OmZTPWBBTjzBDAe98WP0vFunNT2Lcyo87mB0Oz+MobrJ24e8U/25O0lqKb740VU9ctXEPCVHrj2qFZq8vys8PVSQDT0RnYu64D78PI0PqT2IL5I8iRMMPcvcTj2q4Yg92GqbPY4ztTzErgq+X3tevJCUiT0hKBY899AIPdOJKjxNFRE9vOGXOw0QM7s5kiq80PG7vMBOubvBIW29O2rRO0u6xbyf+1E9EqwGPtlMxr3yI628xaDsOdzB4zxWLKM8h06nPGiKk7qIIFi97H40voacJj1k8409379KvXctmLzRIbS9j7SSPLWf0jw3G7y7P927uyY37bxeTlo8uKeHvO6mvjrkg0m9wqH3Oziy9Dy0rvw7KlmTPO74iz2pDq67HTNku4BJQzvMRP47tAqnvLgK3L2wZoQ8FR8tvJPNTbwgZgM95mUGvZQVNj2z4/E8YUokO+KudbwTaO87//yKPOAZ/ruicba8Gc2PPFdGP72qSyK9NYTOvUa/qjtDH9K846ukPGgiNbjp0w6921Nyvd8uEj2hoqE7UU7MvPFTyruI39m7QiBWPEuGATw6LfG7gwcbvYVdEroxFe488ddBvEUcybyJwJm9CHp6PZNc5rxhcAi8oPzRvE4iYjxoaMm8mOHIvPWOBzy3kA+82Jrju1CBkLz6G6I7gAdUu47UNL3W2aS9KD+zOdvAjryIEi287Vu6vB4iaDwd2qI8aptmvHuHiT22SMO5sLoSPQ+Ya7tMX/g8JkOOPe30gr1hnce7DcKFvBxfh7wV6aE7B3SwPEnpVb2PUim7","4CZ+NpAr3zw25fk9Ud01vA/gHj0xsGk8LFIGPH3Xrjr17QU8T4uRPLeVSzzCP+c7lmVzvPOde7vLKpU8O8VTvHJ+VTyyZKY7EfXfvE6sYjuuRLO8JMnjPA3RDz09TiS90PluvNbkArvuNfQ8YggOPNKja7xpsES7n+sxPD183b2DXCw9DVJAPe/X5zyNjV69tr6kvHhXwDuiU8q50YKHvRB4jryR2KO8FXmCPOrhvTyT4Ts7o33PO6DUobzTOoG8+DsTvZAxErzw5KU8LDirvCv6Y73W8KC9PZvOOnAkQz1eWza7ftrwu8O0Mrsfk5q9KULJvJxqODxVOuu7tHRqvNVRVTxlC0y9gtcJO3wN8DtVOcI6/28av01VtrzTdYW87tJuvMyXnTtSFXQ92fR5vP9lDL5LPkw7Wv8YPGG4kjqjhHu68ZVSu0l3qbwr+W68GhXOu/Ea/rvUT4K9l7QGPBDZCDxDWWu7IedROrEQQL0ocYw8JTEpOx9qDbwi+gY6GXUBu3UKmLzDAzW82FfjvDQ3MDxlCVA8+rNGu6St47tYzWc7aZBKuzIcKb2zIOa6ZEXDvp2Lj7zjSVq8GiQzuyftfTupxpW6G60+vHGrRTtbaFe/IVonPDOTJDwlGZm9lHSZu80bRruLoCW8iuQZuzdssbsLP108J+YOvtbkJ7yFjFo6KLCWvm2uP7wvfoc7xFhbvLGsw7sWOS28+vDEuuvpeLuUYZm89GwbPJbBnDwOhAA8T1fPu9p6Cbh6tji76ho/vgVjCrzjjBu8RVeAuuYBOD71TJm8qQrnvUIB0LuuU/E7YPutvPzZyLsFhBu7JavGu7kOp71jRtI8oAoOPaD8Mrl8kfk57rqtv/ql4jvmhtc7vSLUumk9mzwNxdK8aTMAPFDpsDwO+SS88yHEukUMQzwYbKu7TwpIPC0NEL3+ghq//mz6vWOCCL6VxzK8y1kJPKUNIjwd+D08xFSQvPBLSjhRgws8YWGHu+c4IbwwMqc7DxecPPe6FrpARoe9","CM/APHAVT7xgzmq7UdLeO7TCRL0uxk49v+N3u8cjFL2sLQ0+syA2vbM0sz0764U9ZDB7vf5cBT3vuZM9B8fIOyPJbTpwBoa7DqeSvfRkzT0eABS8npd1PTF+nL1sXPK6JgkQPQDqPLsjYrc6bQCAPN8liTrFu30856kLu/lDGT0d2+a8l6tKPceTG7sD0K28Sd3+O8VmSLwernS8xEMMPU8l/DtIft27AfuEvKfxmLrEMyc9ahsJOwbci71C+QO9omaavKSiSLt27bu78fAmvE9sN7zoMCM9QO0bvWN9bz1Mhae8iVOUvB7mkzy5Czw8cb8IPGe5Fb6w6BO7J86lPAiJADybjow9IsShO8nMkTsgU+s88VYVuxmd0zzYzmS6EawQPUEBn7tfeaw8BG9kPMxoz7w8xAK9jesmvG+W4Dvs6f87egEHPdqlm70t/5+8CXhBPd93Br4yeYk8p5AkPGDA1LxUeDc8qfXSO83XertDcJ48uus0vaKB/zwUgxm96xhZvcL0sbxqDic9cYedvWDqLb3HALU86OcsvXgGHL01Fve6QSrcPGiZPb01g1W9QpmQPBixnryVTbu8ZdUVPVwEQ7yZYDk9mXyOvZDYtzuGL7q9CxsuOzuw1j3gI1Y9guSbvR46nDxIphq9VncAPGd8yb3rqIm97nWVO1ddZL3Ik4W8ecDDOlERB7rg7Eo60vo+PLO5xzyfJz28qNawvHgKjjuiF9O8dK2/vH6Z7juVWAq+/Xu4O0F+ob3c/RY7uYMcvQxCXbwmrvS8tdKqvN/cZjxkb/k8+XF9uy32QLsNtH09LKdqvJxEljxlxQ89q/u3O9IGojpRB9+7MajxPGCcfrwHuJM7iUwzPP+CmLxb14+8L8E1vB7WRzxCjOy7Mzg3uy+fYTpZgsK7Z1CGvJHlnLpHKA27yVsRvcynubzoqTC9JAGqu3243jxqtEO7D8wRPRPAA7y5lOm8JKoVPVZFBb0Gvmu8AH4Pvco5SDyRjvI89mX4PbX+5ru2iSe9","wXz3O8eaNLwHBKQ6bXmUOnI2TT30dA47v5iHvHPy+Tmzxv06hOlZPSVTHrxhZww9XoGpvD0UrbtKN/a8ewWZPD7wjb25tra8QhkxO+0xyTp7iNO8l0YHvBA7Dr0gqf87CnABPPR4/Dw+uYi8FBA5vZQsQ7wg3qu9eQ4EvFkQ+LxYmQy9FgmevEclGb3PpVO8twRyPA71XbykwRg9UGPtOzhN+LyFZTA8lvQivQfmPLwdehI9hQdMvLFwirubWwK8JdfgvPk9DD3kxye+xaDhu95voz2cOj28Sb9GvXmiRD3VDZO7npgWu6Eghr1/p7G85+5evEuPVb24NMA7vzLLvAbFbD46s8s8S/7RPGr7Rr5fzKw67iqDOxgv6LuFAF+9VG4BvFBYN71NxhC+vA5TvWBpRbqJdy+9+54fvE/mOLxAGXu9vIchvRoIs7sGW/48b40dvVXkAL7+RqQ84H2CPFvl2zwolSm80QiNvUUAUryiy689VdawvUl38DngaSa9EzBPvBHvAb6qZou9uH12vIaX4jvAIBM7aIo4PYRp1734Ifo7N+AFvSa07rw4bwO91AnxvHN52z2OYe676g3NPIE76zy5WkU8OIETPSnWGTyRsHm9J5T5u6nAzjw7UF68YxAIPODQqTxm1jI7ddXKvVXatzxHYES98cIpPXRAmrsfVha96Mwxuvd9wD0hCMm83xCTPQWtXTwpT088uf9uvH2IL76Y06Q83cCZO8gaqDy9TZ67KbGfvWDAkrsCDQS+w6IpPTxSNbyU5xa66/XZunsP+7x3fEU9rA+KPLL/Yjt557Y8jlOGu8s4TzuSICG8LyauPFLcCjsWJFo8+WRpvUGiY70TuQa9YjzTO+XRur1ISC09/5+nvTPWFr6ajBo9BovRvJ1AAL4xC6A7bI07u/A6xzzjIjO8J1xSPTBrPDsTIYM85P6kPIsFyby6arc9gBQdvUYXGb7omvA8e/ypPG8LorxGBcW8RuU4vKe0w71IiFe9zVDoPGV/wLpr4YA8","7F9YPAmvWT1IXzS6GkzGuyEA27rh57Q7bfL6uoFpz7t4PPC92VX3vRkPmbuslw68oISvvCy+1ru/DTy828u/OXuYC7zawfE7Vxmlvpfe0TsUM8K793YivUPTsTuesrm86IO/vU+FhrvnV2E76gWKvI4wDrwDHuk7idjMuwZd67vjThO7FMWDu8nEBjzEnHo8XrdgvW4iuLxSIoG7/6p4u8t2VDvtBLg7nQrtu0ssxL55KIa6iIjwO778LL7WMug8gMf2vA42rTuB3Ws7XNqDugVLvTz1d++9HCUsPF5rTrzmtKC744s3vDAqM7wYCEO9sOYyPD6wCju4Acg6wy4yO3RPUD1sfHO8BOwVvdp/CDsn+l+8K50YvTiyNrzwc6Y5IQsOu7AbrTs6wXe7ILOqPaFjoTxiN+o7wo7aOxT08bprmXE89jcNPWjNCb9BsBS8ZmEGPJ+iU7/FwxA87Qy3u3EFRjyIuCm7ADObvigs2LpIjyC8s9AIvNfHGL1Emxy7Bg6Zu5pn2joxyVE8Hco5PC/mQbws5HM7uIbPvDZYvry5nwe8uLISvEwXGL0cb8q9jiOOPBkZELy1gxy7aQFdOT5xl7wcn/U8x+GePEQIkLu2FwC9A0nuuwMZeL9v+KO4WZbEO4RjQjwDk0E9pXwYO6saNThgVpW6GPpKu3PrRbyl/tw9FT0wvTgDqz233bO6IaKCvDy/vbutHTU98wzrPC5LRT6Hx227JTMLPuTdWT0Z2vG8TM1GvXsoFT1pyY47INm3vJcwoDwleQI+S6LWOgmKNT2rk0c+4WbAvFcFPbvF+Jm8VWJzPSv0TD2lQxa+66JcvegixzzpEGA9SKyhvYj9PTxgSBY93mNdOpnLUTz4b6G8trcZPeXrRT1QFA49LlzuvC77Gz1i87c9zsuIPYSM0j0k/hM9CP9dPg6t0rxhtx09cA+LvPHInT0ocsy7NGW1PS10njyuJdS76x/evJeRPT3cwG29lL6HvNzSMD0jeKC8QLgSPpnLAb1SnBY9","x8ISvewpJ7sBdVe8ZIFNPW4UAT6UnkQ9IiAXPRMROLxChgY9bvA6vSi3Bj1f7QO9iOK/O62bXjz8qY68T/YWPfHU+717yLu94yDZPbavTD0U1xc+p+MzvkENVz0UVdM6Kx9jvDHCYjrPkrg8vkwoulLFebyRLN8821OWvQRzMr1oIMk9JmmmPboppT20at+9W8iqvXNBMD183dQ9EKTGPTAwwb3Ssrw9fPwEPfIgJT0Xq7E9f/avvDZ1kTvVSvE7u7AWPeS3pr3gKDO9vlO6PMlvdjw4lMI9fGZzvUjeib0r/f+7s3S6vDnvnD0jkFQ9bjIivhA2ND1SnqY6Iby1PGVYObtHyGk6MD25ugjmLjmPcIc7YHK4OuQuubo34ZQ7dgz7Oj2QeLu7n4M777oRO6ZKgzuf2CE7tA+yO81hpznXdH+7FfRouZdubbzl1C2/giwyO5DWpLq0dvM6ZBpwuZEYsb3o+Ae7iMWeu44dZzu0ckk665eEOok7DrsMYxg75OCiuScfzrf/1D666ceauzYTrbvS79K+vMbgOp4aITpc4Jy5AnmTOTyBFLo5Wa26rWbqOvJ0rDoSGgI6pq82O8yulr5yRHW7H/IdObBIN7kWsP+7CM5OvGOKtbzrVyO7AAPyu73zFDnJhF+7uuqTO4wmKL23+1A8ahuoO04FG7v/E5Q719stO5nBa7v/SsK6Z4iePA9+PLu+LBQ878zcOb9xDjuqiuy+GVLxOVBTHD0z/0G/7dMrujLdgbtjnJk6u0W1u4Fmj7tBQSk7slp4OyjoGDw5zuy8p+fzuIOtmbrTWVM6ExgCv2P0TDzgWNO7lPH8OtYbGbxraaS8vfjcOOXhIjykmfu6FbTqusRUprpnNtS6diDzuZTHZDtqttY5g6WWOz39jjobEzM7vWglOw5VNDsIAiu70k+DuTQZkbnFZZc6Wz4wPJHpvjog1xe7qQ/qukS4hzvTmO+6hq3ivvOsiTvVmhu6h/SJuyR6JbpXaqQ7/C2ju8X1QjpdRG67","LC/Su0snk7wMJQw7ONiJPNoIhjtcqf28OvK5OsJZybtTAi0905OdPJ8qHT0V6ZM8eJW1PNErhrw0Mi+8egKLujPY6Lz+tpu8NuWvvcCb/Dz8PiM8tADHPEVKH7qSDxM8kGRPPLIK3ruyITa8b+GgOxx5wTvu/848ET3IOj2E4zz/fra73fsUPM2Gg7svksy65DkwvDBzgzyvkNa5+zKFu/nERzvHCZc8ijHHO8CW0bxVcoQ88euFuwf6zLyrRCi9pr6gPK4nkLusIHW9cOKFOsRZCL2hChQ95GQqvaSqfbudfEC8/06IvI+stTu4erK83z40vNZEST0Y+gK9B+UKPIdcDz2Xncm8FMKovNsmlLy9kfI84PRnO/VPoDvSHIi7cNNUu+vHjj1wXwk8mnoOPRCXvbvfiIY8IZBzO8OYDzyXRrW7n7KWO44rqryyndg8zT8ivVfwib3GVg4889g8u0SCCzvmYwg9m7U4vIqAYbyhuRM9GAcWvc6DPL0P+Hc7ROoDvCgpwbulLg29uBCfOx+Is7noIT68HNwzPc11kju3hHQ8yqf/O3QpG72JvSK94O5UPQ8UnLxT25s70jIHvDOSjbwRmJU9zl0DvYUmrzteMTs9fuVrOx+Do72ZUIY9a6lfutfFxbyAoDi7y/aBvOdzejwGqaU74wv0u78trbx6JRI9nvUfvBmXBT28Ywu8Z/UhPlt+KTu0tMO951i6vTZ2GT676R69g7AIPc/0kD0qTqy9GeUdPYu5gz6p7488ByYUPTtn3L2mzeg92tzXvKzXDj2WluC8Yu4CPf//vTw689a8tzmtPdQlGr6MIAg+6wpoPfybar0605S9NEMIvF1vgj1qf0u9ZVGtvUkO97yxbSU8HceOPOXyoLzy9Ym9ye5iPDijGjtUl4u82LqkvQHfaDx7yjE9bJ81PLYFEDyiZak9XvmNvUM/Y7zmMao6KIUavS5RTLxg+L+7M7ZePcbctT3HJpW9eU4PO2iNDb3yOKI9gpYbvk5axb374ZC9","/zFmOzpMMDxehWm9XAjnOaOuXT333Cm9G+PEvXOTyLxqwwe7lFs1PQQ9VLwpuAa9WlN0vFYPQjtdQYG9iN0cuxELjLx5C0A+NhENvdbvtzx8VCa8lPAEvpkqhrujt0c9wDeEPbyGJb0OwiC8qQj8POe/fD16GsM91eQRPUcsX73tTxO+pZbWPdJ82b2rD8C9tDqOvXHVjTxsnDI84j6HPMQnjL0bL8W9WaIavZXnkjwLGgC+NBkxvbHmIT0Dizc856NGvSik573Q5pS7Bd2WPZupi72tcYY8qQLOvn3c0D3Vn227oIMkPLyn+bzPX1K9XvIevjWqTT1ueg898JkMPD0dcLw6fJG8qk+fu+HH3zzDEjy83k7VPPm+KjxmiEu8CPspPV1VZ71qBIA90ruuvPK5h7x+fJw8pIQgvys3b7yywqC8f2jMPPVlBz0TiaO8zcThu1UGsD1bDHG9rWJ9PTEnTz0NcPO7JbSaO7S4qTzHXaa8JAEvPKA2azyuDPW7rAH4OyRcDDpN9As9lvGHPGP3+jy221w75IHcu85R+Ls6XkG8jfCLu6ZO9jkgFbi5R6H2u7T7fjohe0g9TmbLPC8GiDvmF3M76BoOPZ4itbtMnXs82wqpPIAV4ryThg88doYgPBcrFDwYYc08VpQAvYMPjb4gebc97m5mu79rDTwdNX68G5tzPPbd3b5qpK28B6WnPaiaHj2QxMK8JdMPvLwTRzoPgZu8JOSoOxrkJ70F5ku8amrVO24QQjyUHAK8o1ccvX6pIbx7VIC9Ewg1vEdtNb7Flk++9JDVvOGtULvwkT085DwTvdFokDvT91I7e+KXuqddnL3Uv4A8I1MlvKbUVjwjqYC9yCddvJJ05LyLCg09BDhAPLKzGT3Yn2E9X+1ZvC3y3Dthjay6jqBvvelQTT2aXeO7CQR5PC4Rnbwfa7A746sPvF6c7bstj788l8OQvTyYJLy+Z3u+DN6RPTHgtrxlxmI9UkXsuvq3D7vRLBm9vPLGvV/lCrs+iIq7","Mba/OWte+TvL7+I8NPkLvVD1tL2pnXa7hy7KvFaAHb6DIFU7CK5NPaHygbwOuw2+qDHwPEvmUr7i7Wu+cmfHu2wNob6+0cC+QRcOvPh7FDyCfwS9C1E7veMQxDvbc6c61EQcvM6xvLyK2HO+OHscvUJJvj1P+sa9If3fvJISur6NzSi9gT51vveRGL6TVZI7FEZaumuMqTxiVAO+fSplvM0Bojy5RIe+3obYui7lxTzK7Gg7pvl7PaqCEj0B4pw7HYf4O65aer4cJCg9rniOOy1tFLwu26y8kFHAPAwFPryda8u8LkqMu6WHQ75X0iy7TldqvPuG17x5Ezo7j/ZBPKmHo7ywddS7TxkrvRODkb7afQE9JFPJurX5Pjy93ai8IQa6vaXncjvVVUO98//uvO17WbwadLu8FuAovWeUxLwgJro80aAbPQXnoLq853E71aEYvK1LiLy610o8+E3RvaXXlL0V57u727N4vBnpQ7wiMKY8KIHFvBoqNL076oe8B9tIvYBPvLyb28e88PQLvojtWr7WWYg82mn9PLry7Du2PCy+gjT4vTH7Hj0+1Ak9Fv64vdWNlruggjK8Eg6ouogtPDxnfaS7UCa0O+KkiTutnac8hJmXveKAnrw6vU88IEIoPP81EDycBLG7QDiDvquzP75Td4Y8163lO8mnxDyXfwQ+JuIsOzvVET0l25s78gD0Oy9wUrz804y9OpAbOs/WhLzUxAS6orZ7vfLBW7yI67W8v1AlPc5QWL3w3nq8E4wxvfSvFz3Ovg497BiOvPOnc70jTg2+LVP+OyNHkzxt+1U9iJ2hvdfxTrw3dCy6mgmOPYEbPDssi1K+izZ3vZ/Ve70J8MS8hlp8vY8Nd7tE9Za7NkwGPGBlE72eFmO9PC7pu7oqzzxqewA8w+XAO7xCCL3w7aM9jCDfPOIXrrtJWwk7WRfKvGngrTu2yOK8SV8jPaB02bvrBxC+SO2HuqvWGzxbXiC+FXvcvERAF764Wiw9Hg9NvdewsrtnLFY7","BsesvfjRo7zWgia973czPXuwAD7KdyK9ThzivZWACr5yYkG6yKhBPGTV9L2uKsy8E8XlvDRMGTyGas+85DtAvS83ZDwi2zm93A7QOcrzlrzTR3S7osu+PFPYsjzfsxQ8/UwovS1zOzwIOaU7xa2TPF1JCT30ggC9HeNjuptBijzCBRy+U8XwvRkuELyB4ja87Aw5vYh8qDxQW9e97mkQvd0mBb12Equ8JEk5vdNoIj0lpW2+lasKvbkJrz2qhIo5G+OTPAPVJzz0cRM944/EPTcQtTxxYUo8usT3vGLWhTzkrS++EDlzvbYP4jySxga7jdRYPJK+07xRqw27gl2SPKohaTyK87I8oJagO/1hBr6ZA7G7on3XPDAQEbmM4oM8ndIWvevBAz1MzwI9vuRIPMPSJz3YUoa8WWHTPDuhjzouInQ8mtYzvErnir61Goa+m8ltPO+U4buJcQy88x8putWnqb3JShM75kiTPIKiwrtq2NC7J0mzu2lmZ7xKqs88jdGQO7u6urvHIlm8KH8sOyMA2Dt1I6e8x1iXOxufmLsOvI+7YWTYuz3qSbrWm7e78DI/vND23ruVOy+7KzsNPAlL6r4mY++7qchvOx3lKrpAY187v0iuvFEN5712Ohw77O4OvOg7Ob2Jgf674x5SPAFWB70xxLq8BWF6PD14tzlC5Qa5quVGO8xVKD2BSKC7LdpPOzwsHjxlYC08xTvIOaHGVrxa71C9ATpovCBUg7qcvM+9eoLSuse5ELyivbS7ZR0UvNUyEL0odGo9P2HPPHp40z0K2ya+Wdp1PBl6JjwQZKu3Jb1TvWh4abusO547HWplujkJEL1m/x29e/ChPJuYMDwl2k88aQpEvCBEwrrS7Dw8Y3kkO0LwGrzpmhA7Ykt8vJOWkzzrFkO7A99fuvGHCL2Xvh+8i3kvvFujizsaqA+9PdxwPAIrDTxrYBG79REavnUasrzNOuS9ktEvvROzBrw5mJc6jOh5O9isFrzonr48bSHcvAf+AzsxnPU5","yHy9O6LRGr4ZAyq6AvwUvK7SibyKzx2+mpmuukhYIboXbI07HuIoOz7CdDw1kha7MYEIu56eezzoVAo+mpOpuH46truUusK5bbqFPHz8crtL1MA5NxS7OqP0GDsIv866B7BFvOZ8Db1X4kG74uRgO3WZb7vybpm70qk5umPWIDodD4W6b9+nuyJl8zpXz7m9rzTkurBOiLsTrIk7wXOluejQbTy3LUG61seEuqo2fT2D9gS8TCnauvNqNLzEVFK7ugunPDBJBLsZmd27qmmZOZ+6DrxdhOe7A+ATO7QSZbsu+ce87mUKu6h9gDqHpNo7K1gnvNf8IzqQnJY6tHJiv9hXYTtKuWC+xkMVPOjlpDoJvXU6iVgTu3NXfDt6ehM6Fl6ZOpQsATtqtgS7hYmYOjfLtjaJuOs7GRVevWw37brPN0W9Br6LPHX3CLu3S/G7a/CuPGXTWTxfNiM87niPvGSCxLtVk/i6sEfuPDEHSLxre6864ltQv0gfnbtbwTy6S9yEu6B/9ruLJTe840wUO9uViruJEDe7cD6XuyaSHzslg6C7TtVIOhEsyjqxz447UyTzOrfvCjrmphu7yMhLPFKsLDzJQjs7ASOevNk5LjqORJQ72dEbuzC3hL30thy8Np1Su0IN+DqOAPU7cM80O7BtfbsHYaS++MNhPSBZXjp3/x8+KgADu1Mj5j00auS9Jnslvmp2lzvkXSu+4yvkvZUJGDwpBaU79/zzvUCIh76spti7t0CTvQ3Fa77QHRC+NfQXvrdsjL0H0iK7ySFQvDNbQ76EGUW+HcbWO0ZvvbpzNia8/4J+vnZI87xHLbm8WiLCPbeb072MeqK9lvfSO8u0Fr7jGKu9yPjgvWn+KjxKmxu7rJxvuvQsOr4NzxW+CBL5OmwZrTx6JU49QQ8WPRxPoLrzc/M9ZYzNPIW0mDxPxTo8T/kUvlP7OD0MGpG9I7syPMUShboPU1S+LljCO117frxLIhO7CWovvsPdB77HYAE92ylSvYvdibw6WMq7","igRcvncuzbr+kOY8kqgivjcPHz6gCN88WzCquxzRE76+bou9zFmcPHXUVr4NJoa9LxOTPAq1HL60KRK+1APuveCcgbwBgeY81ur3PLGhqL2k7qa6V4pNO0TArLwv8L69WyBnvpTnsTwvDpc8fusFO0MzgbyF88M5z3PFvPtATrsYO7y9wfjdvf7vELy06De+AgtgvhqODL6ixxe+FN0oO2ivjbzsISa8+EIBvOwfkLxRkxo9f1PuvTHKzT3AwdE8ToOEPGKtxbxb8KU7xGKZPRcE1byiOzW+Z8CVOw86Rby8cTq+YYVEvtTL8LuvHEG9W675vRpWPTxYz+W83rCUOWtzZj0rZcK8Eo+iPLbRHr1abIu9wBv2OhhCiL3OUoc8lU7oPIvjJLxvwoO9V0dvPd4uAbzxscy8e0QNPJCj/juYqsW8v9E0Pfgns7xY67I4m0AqvfAA+70w7cK87NoWPTp9Oj34/6K8fTULPd7xWT1OTbk8gUp7vTdkMz05glm8wbADPPk41bxITM87g0ymvGT/Eju/ye67wYSsvcOeEL1QNqW8h0eovIAHyjw5y1Q9SdbZuuUYjj04kyW9v2tAvPEsOr1eDfI7o5W2PMSLLb0cBVo97zZvvPcS6Dyi3Ja7EaOOvLHUgr39JFG9mGgLvjsxljxI9wi97dcFvdEnDb3NI7A9LTlGPXV297wnJDc8bdAwvUjO1zysCI49bXmsPW03ibxwHIk8Dv+jvSkUk7wIuoA79G46u8T8hDxD9P688hgGPOfGhzyOMQU9VmFpvVg17LtBvSg9oVnJvGNdijymM7K8N10rPWXwe7tjIow8qq7DvOqMMjz4cTU7DkDzPOg+Yr1yhB++mFjwOj0dHL11hmK9pv/uvaQNuLvwuL473F6qvGYLFL1Fkqc9/4mxPAV2hj1j0di9/eW6PLheD72uzSq96IMGPEJv1j1lRJk92ORwPZ/Cyb2+s0Y8Xbyku9wOgb1NWyS9cX6HPFoeGz2a/UE8nENAvQzdCbzR8nU8","lNavOTHLU717V6e8uMxwPD4wLjq3iQg92j8HO8PdlDpoQYG8DWIKvLZBd7zRYKi7hWXzu92tqzz+eyE8ptuXuhDPxjpRRwG7vLbGO2C0Ir6Sv388uBmdO2zzL7ytIYE8GFUvvvVIQLy0TBW8sT4UvCp2nDvtv5w7jWsXuRKxTjzWbHy8aGwnO1YDorjM3Jy8nHDIOmgXiLx/ry28EcHMO8Q3TDoQjjg7Qi6Tu67Rtb3sqIU8s7nWPMN1tL0bA4W60r+NviNyALxTeZo81oAMO+d9ITxQIZK8Ri0IvuKuD7tP1YM8fDkevhVxozyngs88Q/i/vW4RybsLqYq6MnKovKe5FL2eJSu6Xe5/vQ91YDskNCq9Dolpu3LGQr7upwk7Q7n/u6kZg72RuYG628CtvEB7DL0MnA68p5VZumO5gLuqY8m8pKOjvd+oP7wlT+A8l8ZnvCA5szxXMwi7mX1tPG044LtMNZm9/vXku/+wgrsLN4c8+PXquz/PTbzUtBQ8ysk+vRqaXjw85/e8ODnfO8gqsjodwUi8IvKTPLczgjtrvbU8R9P8PKMtUj3dgn28ER1BvrnnEju5tLS7YpsqO/ETtrygZha8qiuwvNdSB7w3gCu+x0/sO4zfh72phDG9Zge7POGj5DuTsDi9+jGgvOF6UDyMwSO8ZQvYO53cCbs6hxq7OdUQOXqpGDzvPDa9VxzxPA75cjyY0ye78RoGvXwYkT3kdkC9WAaYvTgCHz2Fcyo8Rgxcu1cLkD5JYFw8pVLmPTOYITznxb69hSBvvRknpDyxeom8oVAEPSJ3PjxdOuy8o64rPCgxNL04bw89rYjRPC/N3Lul6Ya73EXROy0pazsR0Zq8ajGkvCa9AL2rFZq9Ljnku7IexzwfUbM7kMdEPGXIjLx35c480xl/vSe0lr24Vs48WAZBvbg21LzaVVs94vauPFWfB72tFFU4TGW7vTOWtLxxnUw5AhLhO0KrvD3fi9i8u5S/PHaRTzw1IJA8c88aPkCBbb5FHi6+","ltc6vMy+T73Ry5s9RKPovIzkJ72GKl08TJ/0PFGuVbpqpBC8RKXcOyWkIrx0X3e9LEDPO8woAL20X8080BCQO5bkSrr2rWA9PsIivpqLlLyO8dY9H32bvSt4Db480VM9ys3BPB6NlbxP+S494ViTPAD1Kjti0Ts+lSQkvevsMbyyNYe9yUHLPfCTfL72Fe87QyaFvFbForyS5JI7enq3vCT+/LyqmwC7T/iXvddx/ry6Vey8X8JAvKqaqztO3lC9gK/Rvc+/QL5hw8e9wWwwvAvbwz2wsdm7AhMnv9gp1T0WkV69E4wZvSnwejyHGMY81tk7vX+wNj5MbLY7JrpYPHhaAj16HTg9cjycu9AvFb2E5w47saOIvbgpjj1r9zk9+EXyvVSCGLxJY0A9BAy2vG9U8btI8LI8spuVvNLhjTz8dqg88kGFvHrm3L5VijK+PLR5PdoKyL2dWRK9NOgLOh32Kb6olVI7kxIUPIJjED3u1wQ8jQkYPbzrbLzza568N8hCu0W4Ijw89TU7ld+tPIrOCr0/PBC9AWGXPAHFt73glZE7bjgDvNU7ajzCN6e9hRMIPGLyV70rTvu9nwzivBzqC76v3N48y1mVO85vmDtiays98WQzO+P8K73dAxU9vrrwPCMZx72V6NI7reVPvYFNKD115qm93Xv5Oxbkj7vAa9O9hRnQua3wY72rc7k8EesMviKDCb1sSag9x4/vOY2FQ7krgCC9AQOFO8GXs7gekUO9JpeJvSxhhzxLeok83iCMPX2vhT2a8D+9OeZxPCGckr1ryIO+w48wO6h2Eryb9TU9hV6QvfPQBL0IT/c753LiugW8rbxZev+86U2AvCVo77o9cXq7/KsRPEsSN7y3bDM8WpOgvOZ0Wz1YESc9asywvJTnr7yYnp69y3yBu0DKur3EnD28j+fcPMZJEL0QVFI97WJ3vTRCkr1lLag9VlwDvap1ITv3tNC+jeV1PQdUtLwj4f47OauZPL9QQ7xVNM886xiFPCOzETwFPa88","S0oCvSL8arw+BhW8eJ4FPSf6rjtd3cg822jmvJsmzTxJoky8aJNiPO+F2jwSpQ89v9ppPQKVmDx+jcq8X6Z/PNO+HT1q4D48IzVBvuFYSTy39he9VKhrvXLWDb1PDB06EPIlPVWBDr3qbAk9bHCtu1Y9mTzQowc91NBmPOHitjtdMuK73tKjPG/Kozw2W8y+R4mhvNj2WrxtZso60eDMvJc1MbwIpWw82ZJwPL84YjzW1uQ8KC0LvWBCJD1MnQo9VQzLPDR5Lrzsiou81oXzOzT8OL7I08a8cW3wuzSzqDyryjK8CQMkPFU7eTs9h7q6Np+rPcB9Pz3gWVk7NpcvPMeM8jyUl8E8yyTAO49Gszyy4eS7DWQePCJ3Vr0gdw48Vlj3u9gtNj1e2rI7E2VXPdXKMjvusXE8jJyOvGXMIzvpqBm9IgzVvFn/sb0NCkM62zh0Pd5CvjxKcrY7uzATvGc6gzxvZQo7LXoCPfqcvDx9na887LICvhS4Q7sqSoc9aCEHvP8eEL10ZbK8HIdSPJyYDD2c9Xc86oiFvLAqn7yfXE88VoEqvbycOTu1lTi8hjssvCIM7Ds7g4I8l6covZZoAb4ZwpC9M8Icvb4z4Dj/Eaq8+4A3u14LE74gfX89dSEyvH1FfLyEbFm6q64tPQoqkD06F3s8oKU+OblsOLx0deM9Y1kJvIvXWjyIiLq9JwpIvBbtTT0AKZa9M716vY3GSLyqyBe7aDI2vib6Az2DrfI8EkxOvWU/Ib0x1GC8b4ejO8i1yDv6K248qL7WPMtwgzzrwH69854yPMpJ/Ty4kSY9DnSAPGwRRL2kEYG9ESRNvCynjL3ZNpQ8WFARvbkIkTsk6Dm9bAP4vIWh1rx2tlw8gVWuvPIfTby3YBC8pGJAOoA1rLyAyk+9IwyrvBJVsjyfR+M9gaFTvLJRLbu/3J29steavLPrUbqXfxu8cj24vXw0zbwk1M88Nrlduo1VbjucpHg9GHkQPLiHr7wCa1S8W0BKvH0Qorwkahg7","rtgKvE9jyrw7hWC8r7/bvJCKwr1OtpE7e9YSPW3MjjwnB4y9ImcbPFPSLLwArT09WMMHPBpKNL2628Q8ldhWvG3377y2PlU8EC+VvGXsjr7ITtQ8oLjCPYZ3HDz674g7mSjju4EWfDzyRQe7YtcKPGwHeryTKJa81ccXPecLCb1auJS9imSRvc6kmjz+zgW9XWImvFSDuL2sSVG+swwGvYAnuLwDJc69VjlFPTKarLxJagS9xsyfPNqsiD30DBw9TloGvan/aD2ye0O9GtOHO2L2FD2CfNa8SB5yPe0t8ryHbAK9/pKMPEslQTr+LZ878lhbPeymtztEwqQ60XMTPTroE73hGOC9hOsFuzTeWT20mFe9jnMTvQPkUb7gsKI9ullOPjF3nDvAr7q6I3lCPg+eLTzKKv28avFSvHBl0j0q4wG+frfbvZ4bs72BKky94DAcOzkcGD3Cd24+r6UaOwNF+bvk8qq9rhdVO132nb3umKG9gD6vPSgLmz3keH28ujD5PNhro735RaG9yOexPn7G1T5vbYM+1puEvT1epbzxczC81HizvHzjqL2Qnxo+ONeJPOUkbbr2qwu9kzglvdR7Ir7rkRo8ghBVuhzWsD2gEju+gYSWvMsmLL4heQg9vsMRvQWZB70vE8g9zesovj9zYr6PErJAvHO8vm/GUL1jNf+8HX9hPrG96T1GmS0+nrDjPh0OLz3z2xe9dF8EPgldwT1ObtW91c0tvgCPtTwFWHM+3wCMvc8UaLxDtQo9uq7VvamlEL3PMFC+mpFAPoiUJb7Aqhe9PS3LvQkKE7qdjo29qcOQvJ2KxLy17BI8jY3SPI9QcL4pxe68UX8SPNPJvb1WLOG9MeUivqt09730vBy+3GsFvv3KgTx0bwc+swOXve/JVbwvJoI89nXPujdk6L2BhMi9+UMKvIXsXD2pFCs+0HjVvcBdKT44T685L6SVPaQ9Nb3+tZM8koz9vchsdb5lEII+tAojPIeJwz20NmC+LGoBvK3qYbzyOwO+","uT0mPLImQb3/j9y6OezxvRdQMbylKZS74CLnvZboCD306ky87l9jvBhnnb180BS82rDwvArVebwnj0+8jTrCuyxXn7y6qBw8AYLUPE7uBL1ZDqk8iUMdvPJzUT3Ei6u760KIPLvXFj2qelS9kXqIvX/DmjxgyzM8jnwDPee0YLyfsdy7RX03u7f1prplA++7j/PqPPCPLT2v6sc8yBsBvC9rjTuNSVq7Hl4Hvei4JT03RRQ8J/zwPO8JyDxGXau7QaY0PTsMaTtUko08oEptObpii76Uhro8OPkmPeSwtLyAv5M7TOupPG26VTldZWY8FhjDvVA01z0urMS9tqIkPGldiTxTK1C9d90CvRVoh7wsWCq+tAdQPOKNRjy0zxQ8NUlOPM4Qlr0fnZe85mUMvIawlzyeV0+9+qy3PIThGDvKtxe9DZgyO+9BzrxF/wi9elWSPL8qI76ve8E5T6J3u/kAXj00u0K9I2ZcvSwGmzzDbRQ8NTVivTPtkTsiWuQ8Ary8PI7pRzwG3IY7vVtoPML4PbxL+Zw8QU4kvcPzpz1Olok7Ov3IvFwC+Dz0Y9K8QLkUvXnzUzwzJjI9a1i1vFwJd70yggU9SGbwvLsYkzzn/FK8Wb7gPP71qD1co/m9sRE5vAD1n7m4WKK7A6oBvesUyL1uhOQ8iasyO81CCD3cSHA9dWEbPP9hr7t8lk09xrvyOy3Cjz1eEZW9m8JWvXyi5DyMHz88BbzNPIHVKj1YHxE9/uC8u8Rghb2L0J28Y+YivGzAlDyXK1q9IioDvfuc9zoW7EO95pSMvB3yoDw+TkC8gpoHvVmnfj24XLq8vZBZPUORLL3E2Qi9v7tqvdGDmjyY1OW81SsqvXj9PL2oGb08kTzYu94jHTx8A8K8ADH+u7gKTroC2aO7Z/+hvGoXijzCpAE9anw/PZg3Ubthj6Q8n10DOh15Ez1SRXy8JPtYvGP07jsYcL28ckUkOnulr7zxsFE9n2Igu+7vpb1Tf9i70IUhvhGdmrxqHrE8","pgExPZLggTwMW9Y9GkOCO8Pw4TwSmxg96Cx5PB6FgDwKDrW8bCOSvNKirrzQX4e9IYH9u2HwIL3NcKs8v9s1vcI6Kr1deJI83AMUvABggL1csqc8ghGuPeNCPbz2fP+6DFInPDisgz2Ysfa8xYKTO9wxBD1R7z88eSjOvPQemrpqp5o8FBKFve/JM701slq9lcoAvZnxHrzqtM68d86xPDKzyDy53Tc7M0amvCkTKTv5iFE95klxvTkZx7uxP5m9riZ5uyKLBz3WwLg8qfUxPKp0sDz8LM86NlDqPdtGB723V7y8iqucvPKwMb1jVX695pDbvB/OuzxnzsG6LVMKPVQ7hLwqEou7b8ZXu23SijtRWQA75CI5vJl/m7sbbKo79oCcvCVl5r1Lyms5Sf6lO3blsrkn3eM7uC44uxvmtDrFlwa7O2fGu8S+a7tl8Ly69gOGOw+aijwJega8GWssPFdSar5F9xa8RdTOun49EDoaeim8XjM7uvFAgTqNCQY8dggoPKXaIbohSje8gcPnu2UeETrsgqW7cAgiu7RYMDoNdWQ7a0gwu7AK3LvPZd68d7EIOOFTFzuaSXQ8WQrFvdLyMTxT5AU6W4uHOm+H2LnuACK7fSssvlr2+7h+4Yy6Ah1Buo48pr7C0Ba8Ag4IPFZDWr5EqB87bVKROgB6nTtp8Y+9rIOXu5LhCr+6ONy50NaYPBOuljug3xC/xL2OOxoaozvoBoC8ubNwOxANmL5hgsO7DTZmO2ArHDtygZC7IZHAvGylEbwSFNG7i7OSu9D48Dsw8fu8CFLUu4pfSLvT+tk7XCqavGgCArw2lPE68TLTujqRKDxsaJm9WZ3BO4nAlbvaa7a7oldHuS1IK7wM5o474pxSuyjQ0TsM+/q9i1OKO4f3q7puJAC+F5kAvu5VRL4Z2nY66OOVO2b6+jr44Ka7iYfdO/JISrx8oTU7tCSVvq0bKrsn0oo8BBbiuC9MzDuO6sU7jFA4vlytH7skXek7JXGfO5SIwbo8EEK8","DgcTPo6r2LwLh+Y8GUMCPV6Y3Tq2aU29T0BDvQzLPzzDHDu9Th39PFEWf7ybgFG7fDQxPTn0RrxXIw28Jc8MOwYCR72DNo29xNEzvXWM0D1MPSa+a1r5vUM7IzvN+bQ8cf7Ivb0pGr1Pfqm9h0uRPddZ7jyfHjO7mZeFvZ1ULD3pNca8gwj5u+TZTb1uAgg9s6VNu1JEaj0j1qC8NcKBvfUZXLtDaFY9fP4nvKD1cD15oYA9UTtbvHybPL2Q+xQ8VjsevBwihTmpgLQ8lRM0vEvosj1LlCM8VdQhvYtmKb0cIQO9Our9PIKDnbl1FZC9QqGHPSEqgL2apbq2zDZhvD+pnL2v6vk83rwCvuah5Twrus06BP9JvLwWzr2sDpe8YmyZvDrz9jxB+bS8jwqwvaYtWz0gtJK9KNGGPOMHlLpcgpA9/g0UPPHHNL3jBdo8XvqnvdUxaD2UuUm9yv8RvaxbZL1qNDQ9Hlx6OwhXNrxB8Je8DHBEPZQJyrzkP6Q8keHJOxzuPL4nRMM84DeavDUJpLx67q29171SPeErLT1jcZw8AeXBu5AFkD2YnDq9DTCDPdfy6rzRL/M8SQ8MPNmdM71cai29QkKEvN2WcD2qupU9+GVfvKOi9j3iejI9jWe6vUJZ0Ty5J0o9faWQPIsr270/MXE9ISlguuzPUD2wx2S8FaYtvq6CW7wYcDW8p13wu0KrqL6jeqa8KXcMPakvdzvUVhE8SAa6vEZaFLspXCi+T8pePFfHq7xZsIQ7kZKuuvawhbpnSHE8w1MvPE8yEzyoDTS8zVWevtBdVLsvKs06q+6WOwUCsjuEmle+6yIivRVXzjueUwI8Db6/O81BXTtRAaM8fiIFuiabd72cVq47dXWqvH7CJDy5Sw68tAGgu2JC3Dt+4wK+EFPkO0mJI77qfJW8s3vbPGoYTzu28C29uFS1O9E5DrwmabG667sUvp22FT19NLS88ErDvQYcPDw+iLa6GMnruyssXDzblY489+GYPHxTGL4rw048","i9kWPP+UbL5Mk5s9nHuUOh9D7r0aahE82rAcvFiPXTtH3jy8KHCVPDdCqbtoz2w8PQJ0u1wsnTs44Nc789I7Ooy0Eb5KaWU87WrPPCiJFjoa7yI+wrEuvU3OkL2e6aa4Jnupu77qyLsVxsi6Vp+OO74ui739jJ28sTgCPXCEi74k9ks8+7dvO6HY2Txfa5c77cNhO7Z3Mzuq0iS7YIbCO+W4h7uGN387zTDhvLilhDqsUj67QcV2POuyOzwQMea9yjbnvnz3W76AEm49A/xTPO1CTb3sOFo8D5pxvTPGWr05PaO5nFlHPJL+GzqCKx494yEnO334yzsna4a7CvEcvlNR+jzPVc29EOMCPHNj2zxbVGW9ZbCpPakhG75vids8rPC3PSP3hb0ZqAw9NHdnPt1w1rwNjsy8vp5XvfKGKD4+34S9ptBzvWIN270b6Be+sX/kuKaaMj2SX3A+Wf2+u0pKVDxPf1K9oH6dvT+Mib1fMkm9N1DvPStmOD1HDcq6xGSwuwEkWL3big68MZwGPwNL9z61lLs+gLb/vWon5bsoLoG7IKaGPHXMpbxAmxM+a2u2Pd8mmrzWx8u8dzyOOzOVEr7QPeu8PHD/u5exwT3n+LK+R6o4vQRnar6mK9g92jdbuqdNN73tVSY96q8Bvi9lF757thRBTOoKv+iVir1P1n49jm4lPhjW8z3XdVA+lyDYPn07Cz1DuhI7rD7TPdTawz3cXUK8NIwRvkhHVT1draU+9ivLvaxYizysfHo9wj4rvq3lpbyHeHi+x15xPhDNt71BuaK82uyrvQbXKDyPOWG9lwpku8KJWrvaedG8jc/ZPZu+db7vhPQ5sQp4O1AYu7zkWam9Mq9zvhkLzr3qfma91d0vvT4Xv7lL6vU9lVWdvWTANj3JyRY9Qws6vVaqjL1VYKK97AWqvPGg7zzEx4E9Fa5hvvemLT5dCdC8+eShPC39yb2SgAW+RvXBvQqrXr6PnFQ+SpgVPRNxfj3MZGG+Kr5gPedNZrwjjhC+","bWzYuu2pRzxGShW8eeTruyDlgjprw8o78w4IPBTbn7v1j1K+64qNvjLv3zsYrG674VXhOaKEebunYec6UGEGumgdBbznxOU6W7sTOvmICzw1/q0559cMusD73rqSa7m70BJWvhJ+SbsMW+Q6A32su+u1BrpDXWo65eBvuuReQrvOJBY7QaVdPCJbKTxMczK7/iOOOu5mITxrIVw7b9rHut46gjrEc1E5B9ucukDOn74+/fw6N0TPuzXdxb4GhbS9/JSxOr94eDv1oWq8/vExORCofzw1bxu+xzetu1auWby1OGm7+qibvZGuTrxYXkC81PGSviITxzsh+127+EPxO/3GkbunwDm7mZmdvjiyubr3a5A8SeIIvDaqLr5Bf9A5OZFXu9MNlTwe9eW7iXUuvdzTd7pIdNQ64DQDu2pHkzkdCnE8LaygPDhQDTvXM527fFUbO/KcwTuAwT46dYmeuvIDNzsFHY07an8WvUJw7bkqeRE8vdUNO/UsgL7798A5JHwvO4FQazte5Tk7E92BusjcNbpB/487as9QvOpIhL7hrUk7Kyn/uxzIA7+ssQ++6GaSvXK5ZzvZojg5d3bXuwBAlLsAY8Q62zA8PBo8wju8LwW9bUOFOzoVMb1TNkU8Tgo8vEP9abr9a/O+CNyaOyxeOruZf6y74mOYuzBmGrw="],"bias":["cEXHvmY2cD7OZ8e+XSQvPibuHj5bJJ0+rVhRPteChT4Qt0g+V84wPpB8Tr2Ii2o9D+3LPkpqQT53mJA+MD65uzwE3j4cIIw+QimnPiBmmz5/hpU96LFNPbSIXz73eBW+GVyWPjhYhT5srYQ+a93bPmNpT743xwY+KI6cPMrHLT4+Tgw+cMGlPiIGgz4yZqc9/4KCvchBtTyqqII+kdh5PtAONb7x2hk+0AMUP/1ljj7D1Lg+dp4OvgRipD5sCLw97eBcPgYZlD7PtXG9tDeSvYaNYj6G5aQ+8QGuPpQmkD5U2Pg7sQKtPjf8aj58Qqk+xkyBPotfAcCQL/0+miw3PnSvuT7pkEU+5MCzPk7EtT1Ko8C+GNwCveLp9T6nhr89iXZ6PXdfhD7gscg+tGRYvfdM4j1wRHo8UGiuPfj+RL3c56I+JrrNvRdGyD7Y1V+85HKcvTMGzj5Q+bE+cDDGvXPK3j2WUMI+JHsNPl6NAzxW0m0+WyFxPsLjXj7ijcA+nJirPqnlmT5Tw/Q+bT+SPr0Ldz4s8AW9A36QPmHmYD6eA6k9WClYvNWjyT5OOx8+TKGsPjqbxT7yQoG+HJXSPkPxoj7ie4s+PrZ9PhuCfL7kfbM+xfBFPgjkqz4uwgE+ZWGZPo5FszwYc8k+LnUWPtneuj68Ncq8yDKyvjHawT4="]},"dense_2":{"weights":["DACsOydefb4pZi+8mCXSPDfrcb4CBZO++W+PPQIJOL6+uQg+dctpvf+j0b3RNCY+L9zFPTn4CL8S2NW+CrvovXNPJjxUqWY7TDuBvudhgL5MNru+mXQDPUHhcD1EFaI9DkLYPdwfrr7e1e09nbhBPtQs3LnWjYW+RnmBvMLQG75YSk28nevzPATdEj67yMW+b1yfverPuj33uV6+6xHLPaz8Yb+F70g+skskvgw9977TXHM9+jn+u8bIZj4RUPE9gRjJvlMCSb1/cbQ9Ixc0vWQ3QLyUKHM+uG5NvWKOGz5NjS2+WAAQvWuFuz5pLjg+l3A2Pq+2N7wvUvY9xWt+v9bnmTyrNDK/Gf0ZvjDTkz5WqFQ+g1r3PVZ4Oz5wQLy8ttx7PdSb0b07lw0+eIM0PbwJ4r0cA0c+ZNSlvy+w1zu3Bo++UfbXPADlFz1sPK89RvekvTRUSD0IRTI9VMjSv+efYj2o1po8uGBNvn1Ru7102rI9B10BvheJGr5Kytm9EMNfPSAry76x3Qq+VYFbPl6GpT38lhI+oUIwPrqQ3Tw12Y8+xcJ3PYnlLD21f+k9uGTRPFw+ND456hA+OE1oPl7QwLul4lS+82yuvTA+Hz7KSww8zEMzvnRcF754BXg+NRVgvI0lObwzlRo9sPYzPE2FnD645va+AQt2v9AF2z1i3Z++Lo09PYKwML+wBa69JRuyvu+4GT4uQva9EeC0O2XyZ71DTF09/6oAvmGZ8722wM89EQvTvRS7Cj3E9a+93TSdvjzE5L0xdmS+2Vgtvl6EF7+dsSq+xLanPXU4q77lMuG9LmpLvonFMj1Z/bS8z0R5PQ97RL01gpW+RVjgvNW6Ej59jD6+TxDDPJmlSD3TFJc9ulTqPVciKL2YsxM+1rBTPFCBQr1tMoc9Iqj/PffDXr3SdfS+D24UPjV8iD5BRTi+cyKYvTAFBr4atne+3M5HvfFMRT0ZmEg9qN8WvNrDNzmpBJu6zemtvqBaJL4STTI+22kuvSd2SD5r5Xi+","UDmQPeM0VD5XqIM+3PwkvOv8kr1KioS+mlUbPjmaC77G10u+MopCPSZpDr4FXNi9vr0xPmfqkrzMNte9V9EIv9jMpD2tH4s+Dg9mPgwdibxcDqs9JQFJPtbZezzfREk+N72vvrnffz2JUz68fXAEPmQnPD6cZCK8X0n+PDzhQj1KOPw9tBGGvv7S5j3ePxm+T7kxvqF07L7k16W97T4ZPphbdr0zRiS+xTsvuy73Cz7ihAY+O/EXvt47674uOJM9p5dtvRgdpT37Jms+eTjgvhBgy7wYZD+8yTgCPpTrUj6un1y+ZZKCvqSwMz6nmVK9u20wPmxPgT6Y0xc803/8O5fwlL7PrDE9XfQwv0LBur0Fidu9KYYHPQdTTb4ukt889Q/5vf6nPD6Em0I9jC8vvufrTz6jZ0e9voETPv3nuj3jQF29GUkOPnJ9BL1PxcA8DRsHv5hnCr/VsbW7e9urvr1XrD3dvTG+aA7cvebI7r3zqxC+03sBvszDGr6nXCm9aAeBPfYFub2gl+69FTnivQr5M72LQKy9CPHFvgJiXz2WVxA+/kc6PquH5TualEM+sxzZPM39mr4qAc29SBBkPsoUvr1EQiO+Sp4XvqZMlb7vSX096UQrPvVzET4mYJe92qO2PFraM75flDa+GKnYvdeMQT4r0vO7pJC8Ox2WFb0NkA+9SaNKPsH19D1CTxm8fBwhvrJP5rzDXoO+o3VpPY3awb4iryS8Zn5Svto9vb1X2Bq+Jb5GvnhXpL5g7CS//KBXPsG2Rj3Um6c93WWOvrKzUz7mtL49rDoLvWugTz6mk/y9y2ufvS5LWz03k6W9zD61vR9Ni72zE409gd9mPirupr5m/tG+EuUCPko9Iz4hFZe+f/nEvh/XK74L2QK91DwBvuGafr2DvR09hJv5Pf0pML6FSMi7vIGVvdEeTz4nwSO+4CkLPgZYkb1fMcq+SI5CvtPheL6wzgi+tCItPRQ6ib5wAEa+CLQ2PtL+0ruaORY94zc7PpmmI74JxxU9","lV6/vX3JK761p4e8iZ6qvUETd7+bxM6+wlNUuldQHD6gnMW+E9uTvpfztz2S1gs8bmwIv5hUBj4yTyG/JKgovzVXG75xeW8+Ec4xvjdV6L2jx+m9hh3AvflTcj2FQxQ9kyLCvbeSR799b4Y+D++KvWJCXT600j4+8dCwPdVYZrxI+3K/rY81Pu1/or3wRBc8yvJLvO9bITzxY5U+h0A9PQBImr+7/AS96FmouWlcLDw+eXu9N0Pluw3M8rvM/vs9CrArPpdkYj7Yk+o+bjcCPrHjnLzHkqw9APYuvWCV1z5eSYm/3AkePs8KOb7iXJ693Mb4vIchdzsW2Dc963ugPQ2Wv75P+049JddJvrdW472W8NA8gUwKPtZTez4J/H89n6RCPsrLpT2lqUu+K8RhPoGir7r7+QE+QVU5Pt4tfjzdZgC/zWKWvw1svz6SDiO9PJtKPWTvJT4oJGE+qHtqOby51DqdE289SXMvv46llr83+ny+9ReFv1BTlbzORQk+UCMqvSSEML6q7bu9cISEPbMfwT2zkGA82+cKPZu1zDx1kZw61q1VPZS+/D2L+pI8OPpzvaS4qj24tuC9zqDFvKDMpb1Yugo+kmONPWOCSD78EsI+YgyqvjfhTb3yf6C+Jl1QPqP46z31Q349hRbiuwNvzj6klFO/tIudv8HNXj4NXxO/rQMGPhdmQr+lcUu+nJV/vGAFVT5GRmK+QnfTPHhH9b0e4j8+FLgzPT4Nb74L5x+80I+XPc0KrD7MH6O9t42FOz3jqT3EeD2+/RsBvozGA7/coRS/vqCOPg1cDr/UKD+9fV6gPVzBgTzAJzc+N2aXPH2ABL5fJMG+zM9BvI5UEz1pixI+OHaEPlpBj75zHYK9tSRzPew6bL75w4u+QM1FPW9VPT6JGQs+WBIHPrTolbx7x/G+9bMmPlPmMT49POy9NFHAPkXUMb7PRO49dpadvrD9CT7TqsO93IOIPtrV6jw3sLo7+V2QvepJZ74aAKY+Vy8vPcQmwr3Dtnc+","exagvJ87Kz5nAQ4+PO8GPoffM76N7t++I30vvbWlB70HF5G+n3MKPkIsHT6CLTw+SGqrvbJ60L7cP5U9v+z5vo2FKj5MhzC9psvFvLoBaL5/CIe9U0SoPeOqTD7vg2S9SZiavgDePj4lia08pJTqvUf7lbthl/W8/25HPAuU7z0Bhy8+mP+SvkaDI71Qd5q9AjajvaMfOb+J5Aa8O+F8vOLoCT4LEyG+pinAvI0tZj4YgDs8wetAPpW3pb5PJeq6iC1ovnsVFr5d9Zg+0xPDvvyTNz78E9m+MOdKOxrIaTw4/wC/gcWjvt0ogj6SaKY+/KTqvZPThz40u4U9D/lwPtgiib6LLV0+TyxTv33DMb4R97e73CZkPm2dCL8bo7Q9JkKKvlWfCD4CDy4+B9j3vq++Xr3yLRw+3O+rPnjzF77kK3W+qDnLPWInbb7onA8+FRIXv6TkB79agJm9/vb5vq/qRr7VOvQ9JL9UPt3MET6NyUg+e3mSvr6Z2L7UiYg+1m8Zvp04ib0sMym9SyV1vlYTNj1mHUa7ZUrmvfUthb6pZZs8I3KFvPuorL2rmNE9Pej0PBWeAr8YKxi+RouePYnpdr4nt54+duBTPg0XLT4ELx+9qx4MPfXYIr4a6as+Sp8TvEq+5j3Ll9I9ySigvhd4AT6bWeq94RxtPni/JrjYTCA9XH0/Pgv0ez5xEH28P58MvogStb58KVI+LU0jPco2V75PfK8903sMvRCzdT7e8Wo9lkIavxXu/z1cb+K+Gu6ovZAkbT6yXOO8g6auvjcEDT5GCS8+gbqcvTGoVb3bNhk8vsh7PV9fk73uHUA8k+EePhmexbwnTmE925uqPsXHOb5GEf+9c7YSvlpXPjyQ5ho9Jbg7v95AnTwUA4u+zMyiPq1Smr4BHk28MnrEPSQIK73+P2M9h2/hvkSROj4Z24y9ZAjZPaY1vj0ec5e+4DxJvQb2xb4Xema+xo+QO2Slnzxkrwe+YrrEPaUpij2DCfc8V++1PVl5gT3Cniu9","nZEjvh2+Iz42nms+EumyvpFjhL2W+Y4+eRP8Pt/khb15j489uK08PRRfer1br4a+bCs+vUTONb+n97m9bajyvftZZr6Uphe/J7yMPmouAz51YyW8wqvRvLJMgj23bLA+K+OOPiQghr6DTLu++3N4vdq6H7/O4hu+vnBqviobGb96z5K+YuSgvjzaHL+92Ng+PUNXPibrJj7PLn6+KpD4PcuK4b0Bczm/8yJBvlMSgz1Es1c+nPAQvqElkD6gIIG+3eqQPL2bzr5yhpY+pdAYvU4BWj5o6L29E7o4Pn7dMz4Ff4C+h0UPvoAx8r4jlaq8C9vDPoNIab3idTU+ge4DP9cOq75EqKS9+sWsPajFNL6HZqa9WCI6PspJPj5w0Me9+zpyvqHO4j1KIxG+cbaUPVVBMb5XgLM+vheLPnbkTj7Lcey9iisNPqO/sD6GY64+6urxPiBGiD2V8g8+aFjGvv/EujvhDEW+EFvzPkYNh72qM3W+KybDPnmxK76SDc68pQ8pvZ2T7T2t1QY/ctvsvdcWa741urc9BgzsPfwL1b3TbTW/TBZ4vhoSXz6zKrm9jgO6vo59iL6p+O495DVevPUN3T6ht509W7vjPV0epL7piVA9xOEBv+iTtT68Wss+hlLavcMZED7dJ949DORPv58Vjb5Kbco++3hxPa4Oyz2ncQY+8moBPh+Izb0Trma+RnYpvnPURz5JxpC8xW6xvv30Lb6cE/u8HnZnvkmJfr5sHbW8a80Kvw6rVL2QgY+8VPOFvhHENb+fTKk+D4SXvROLPT28eYM+lVzgvefE5D7gs2E9Z44Fuxrj475oNZq9ATzTvhPwlr1IyYu9UNr/vmDnWL5x6si+uQfTvqGzcD2QqKY90XMzvS/sE73GjAq+fXPuvMGVSr/nL669TyjOPa9Exz34S7i9eC4tPkF4u73oQxc+o2EYv9/M/D2kvco9xrK5PoqBiD65ngI+/ClrPjOcnT3nr9286BkRvhpxqT7/boo+CqNyvEiRtz2UHlY+","1jDkPU1jiT3klWC9bDwBvtUEAr5p+7Y9tXYGPmZ9Cr44Sf++qMp9voHW6b2u56O9JGgbPHvDSj6ZYlc+pH7IPRNJNT7rEsa9RmfhPgyH+z1qUHY69T3ZPfXrfz6f4Fq+WAx1vWEmnD1c0D0+1Zk9PZDCgT2xj1k+47ADPrNJHL5qt7c7wAGPPi/08z0vX9u8XEEVvynaMz6BEDE+X5BpPmxONb/b/py+ITp2vAIRAD55rNm+tmHNPZoqEj6QayG+PL6XPq6zoL1hRvI9BsPNvtt9sz0xvoS+p/puvQ+bLz06Eb0+zN1UPOwF4TyTzgW/3zepvszHrD6sGpw895YJPXKdNb7Mp0I+F48MvudkCD4ZV0S8rWklPnQQvj5gdYy+rNorvivJdz479SS+pvsbvpfAHz4lVWW+5prqvXTElr2RFz29/LKGvjRkbL4i+k88NFqwvaY9GTpmPI89+npaPiYkd72gtiy97vKlPS8CtLvemPG9Bj54vpXa4z0jl0i+uAabPhbdKz2ycom++HQpPTK9hz1XDg++wTVIvkpAOr5P4Ms9SRqGvhp6qD1zVai8n0okvCHuHj32EB++ZuSOuWmtKj7XqLK+Yz7yPIDHALzSgiO9eZtIvorF973XKmG+UOLtvfWEYD0JqeO9/k9yvgd1cT0snv49JZNcvgHSST5C5LS+M1THuWdi1L27zK294l6vuwd49D24Y9o9thV8vX06NL7Gdd+9CwTlvJ0UaL5WAU4+xb1dvk+BKb5kdAI9zWQQvmPtzzzerDa8GuSKPKLcb7265Y09er2pvGdRL70K+yU+IYY/PZQ1STsp/QY+NhLOPQdV+LuZa2g+Y66IvcwcrL07Awm+kOHQPXwtj7556LM90jBRPCgYtr4knko+1CSbvkXHHTyq/IG8p3wzPPggMr69uAc+3d4oPu23gL1FhMy91x9gPn3C1r4WwEe+WTM6vhfRHr6CrVc+/B0DPUB9D7zwPUi7DV+wPX/crL709h29vM0dPqzWHL6pDz4+","aB7EPecpSr4g+gI+3WRIPgs0ZrwxPAM9fEWtvCLQm75D3Ts+n/+FvuQAYT4U+G6+6YnDvezzQ76jfqK+tq4CvToknr4XiKS+ySS/vUA66j1EKOY9MqyIPXeQhLwFdao+JI4EvnDWyj2Wt008TvYyviSJhr7dYOE8XkIFvsA/lb6kwli+Qo/AvgS/Dr5HI6w+crXtPdLrKT5Dtwi+CDCIvf8m1r0BsMm96gsvPdJftT4G98w9gyJiPmB7Dz1PaZK9A/UTPnp6vL7LUz++AlQ3PJy0vLvY+8y9J13fvKWPG7w+KBo+G3WvPZDyLb79kh8+Aw5fPSiclL3Zrlq9FoOYPr6J1j10I5I+5UsGvpzZQ70NfZG91IYxPomiWT3M1Bk+m8mIvoC/870zg24+yS1+PTvMVL0I2ey86jUHvqHvMb69Q7U9UCFWvP8SNj4GG1Y+gxy3PbOY0b21wGA+IOH3velFeb7YHjk8M50nPuPHkD0EHtI9+yuIPh4r5b2/Jna+BPmCvmIr87oMSpM+s/glPNyLf7o2apK+gdf1vU9CWL7a9Na+jTOFvq5knz1elSw+ak24PsfFPD2jYWK9Bvs1PrNtEj3hIws+1jlRvoID4j0j2Yo+gr29vsN0zj4R8Vw+Txp5PiX9hT7U60q+qEeTvhzPYjyG63Y+hXT6vQxy7b0aaSs+4R66vZdJlT5wTeA+bzKJvljBBT7HVBO8z+pXPpBWYz7gEiC9FmRvvWG4eL4vRz4+9bZmPvxsez4DGYU9VL6yvcKbgz5PU/Q+CTizPfV1PD7AbDe913P8PUpCnj626jc+YsoQvepvgj5WkZ+9lG1SvUhdzL2sQSq9RTZ8vgTolj2VOZ6+N6yMPV9aHT2GXgw+cmg+Pgm6Ar/RSuo9iP9lPUuODj1Y1he+y7YhPl4ANT5Czci9IB1HvcVuZj52uNI6UPDAvuZnHj0waDA+H8SKvcWTgD4m9QW9hUqzvoh8nj0cuwA+XHKVvr0MGj6YYZE+Q5yku9Lup744q6K9","bdGvPQUi5D3BDEq6myZiu8TzOT5PXAC+BjOWvTZnKj1vE0u+EIHCPe6EuL6Aj7S9mJwNPZV7nD7Cuqa+eCQrPimDVb4I8s49TUJCPrLwPz6ucm0+5EAFP+NaYD68xO88nRQLvl2GtTxqIgY+Z6s0PX5m4T2eOZS9Hl3oPWkK0jvnRwQ+3LQQPDEkzD2orrY+dhqOvnXvCb5hSu+8C5RtPY1iK7wd3Lk+rlhavlbaiz6QKw0++0LrvutrCz4t2VU+omE0vE2hOr0G3vk+ioa7PUFOK7wJlsc9lygcPgrsRz5MK7k9ulhbPb1EST02Mb28jLXQPsBmcj0ilhC+ppSjvqlvS73wdBM+LdRGPsTCkD6SSTI97ysbvuCbWT361Vk9TlsMPnuLOj257/y98KwhPkpN2TwKV0M9er1LvingC72lI649Y5mBPpNViD5Bs40+t5RNPiNSGD7oUQu+XjUqPpymSb5PWE890GyyPnAlrb2Df2q+/uGVvobU271ak189BaeSvtuOkTt62Ie9yuglvliP0r1w+gC+Gz61vgk0kT7qRG08q+wovXpnVT4/rGI+rbetPABFRb49ABa+s/r7vbCxDj7IFuW+DnQ7vjrdKb4G8Z8+RJEOvpuSVb6dcJq+fjvtPE9YvT3wCI6+ZAMSPiPVTT1Cr7M7R7OvvufbcL14SJO9QfPqPHaTmL1B6q692L9IPSVyAj1yKmq+trIFvoVblrykL/+7Bx1rvjiiSj7HUbI9RamTPsB9Rz2PwX8+ldb2uXckdzxaaZ0+4jkzPgFqMD7Qdvw+bYmQvBV2KT5GKNq+sDgqvfMo9T4eFAS+CMaKvRjxlj4WIL89lmUfPg0ZhL78hQK+a59aPgUDWj6iihC/zO+Wvq1Noz4/P1Q+IEeIvgMmhT7MYnI9NVXRPh0bPb3SZ72++IMavirwjD3ebwO+HD/2O0uDiz68aLO9waAKvSbYkTsdEsg8tEaEvSdVlrsc4oK+J5OmvW1EOD6aLOY+ZEgYvk4hFL0ju8C9","NEvVPcCKXb2XvSc+ifKRvYdGNb7awP28hL8qvktoqLys6vC7JvJIvpt2Z72rPqI9c1CAvvVM07swTwe92r4ZviQQmL5NVZa8i0+bvUpKSj0FjP095YFTuxTr7z35YQq+4TLFPVh6B77ihrS900aEPTELoT17fFy+gfRJPkPo8T1EFbi9/zkovgCANz4PD5O+88ADvoyi4r0GJ4k8stL6PYVIIz0iLhs9k+Ubvvl1Ab53o1A+xKOavrPkm776ChU+qSlhvMa8RTwJY5G9y/CIvvlUij3NH3o8WZAjPoQe5Dycjy0+OhYYPreESr7wYCo92Nu6PfhUFLwinig+2ZOoO33k5z0ek4i9FWMBvoVCx7wSLiC8sdRqvVutBD7BG+89VvMmPoZ2vLxTAJQ9qDs0PgdaJT6wmBM+8T8Avs/NTL1K0U2+JGTRPSC0MT0TNSY9QkdUPY6FWzzcMMG+lNdsPRsqfL4fULG9KRo4vWF2gL5KS+w800FnveEMij0mBo09G/UZPoIMi77Z3ky+lJC1PXkVer50g6a7y/KSPYvAuL25T6U82fhiPU0iD74cmd68k3PcPaDBar7kO0i++vpavth9DTsnJe093y9kvQO0TT4A3h8+Xoy4vey5f73XWLi9Y4fGvc2Ewb7OpsQ8y2rePAnKv7zfZwU+rDsYPEcHKL6NnFC8ap6nPelPbb15VhE+YsdGPrJovr0oxaI9/5oKvL1WXL048Gi+NysAPtoUezy0j3G+HYswvLy5Ez2MpTK8FDeuvuCpM7zczDY9b3ZPvUuYtL0AAg+9bAawPcTBjj1cFGQ9oysEPlCRWz7lVPw7dCsOPqgme75051q+NNLKO/K5BT5dnTq+Od3JPR4hkL5piQc+CEW6PXCRED59Qw8+LRtWPK6PKzwlLiE+hMrkvf/lHL4BNL6+BwKSvhbaCz7CNLu9S2dovfinKT4c6Y679rIIPknY+D04eji8sCmBPT2uOL5mDUi+3F00viFzTj3IZts9N3/Uu4i1zT23XxG+","m0G9vVsGi720Y2e9sp+JPYH6kzpjpe89w5QQvqf7yr02+Mg93MlKveMXAb6BoCM+cjJkvvMUDz4ZVYW94HnGPS5+Q77HFFW9PgDdvA54BL6uHDE+tlZnvHcdv76ZG2g9qZxnvi0rij0j8p68u3hFvgo89bzs7AG+0UYevRxvVj618N48FDuMvsXjfb4c+Is+vEsyvsomAD4ybxc9JrQSPL8L1zv74zA9zOD/PXEcJb1k/Xu9xfdXvk4PW74zfTA+fG6hPR4J/L1a1hO+FwMavu4yDL7z+AS+HEYkPtaP672kVA09AZSWvl3K0zyNW+M7B4KIPR17qbyiozG98W48PrvRgDw1K6s8SX8vPaMB3b1iiog8k0KWvaCtOz3yn6m+7TOePOL1gD36tpU+KV+Mvn6ac7wZptY8JssTvQMHIcDhlo07PJuJO/COEj7+lAa8skVrvc3JkL4CyIK/DrAMvXjkT7zaAwg+7JurPdFLPzyzSYA9CWTWvVhTib9PqeK8/4OuvIu8mz28EIQ+/cC6vvx9BcD0e6O/69DGPYAeHT070h08vNvGvYAKAjulS0m+RwGTPRnl/ryrsVk9lNRhPvCscjybAjq8Qnn1PIF6LcAzsl49f0Kzuh4QPD7bKaq85P4Lu6XyuDoy8wG+VF+TPaMDqz1PCAC/0zTKPfh3uzw/7dM7u7XHvgaxrTyJrO+/V8gdvnKtg761XCA9QJUSwHZ+o7x/Caw9KLMlPekvCj5/Qta/scwuPj2OEbyl3ZK/8qkLvFyI/jvcaBc+1Ei5v2A4jD7JEBi+iGMWPvCXNrtr4289qJH0PJ9uOb2QU/y+b/EzPsjKTT0mN768jRBgPVLgzrz7TbQ9f0FnPVI3t7xcwAg9TscCPqs30zy13CS+Iy1OvD2a3rzsTZI6UIQmPdKQNDsxeN48AjIUPpb5kLxlpf08Ie6GvTg5rDxgLzG8HhCvPdqdPT201N07pUx7PBB8Dj1But++Fp/lOySvRr9dPQq8q7fGPIcwED1ZAOA9","MASjPlgaa7+Foq++WiSrvoiFo773nhG/mdwJvitkpL6EJxY+nQCtPf+ERz7TcKy7OTuCPrl29z0bTJA9G9+VPVQv3z4r3jy9D6wUPaUYLb9q3Vi95ekMvkBey73ij7U+ag57vo+0nz3YVME+wc6zvTKvD70o6Rc+nhu1OVjRFb66WC28Pe1PPS5HsT3UmY0+fZiUO77iJL/5tlu+yPgyPfQFbTvkDHG8Yoz7u0G7WD582zY8TuKUPpRHGj54Swg/0Rw8v4YNhz2tBzk/bKgUPddv676Bzg2/Dlkuv8f7BD4pChA/+PTjPd5aYD4a0Sa+h91lvaIAyDzC6QA+h1imPM/zvb0b3KM9yi3lPfnJnb0RUQS+pFMCOxNSgr70qem74druPLUAl789OWY8gSjBPa74A7/bk4k8a4fQPQTjUL4/4709h5VdvW+6Vb1U3kW9eoTSvvWcbj16Hny9JQ8Nu4kv0D17O46/DopKvQjaiD5KkI0+tN0CPlmDOj4wjuc9nCQovwCAAb1b2ao9yF6dPqVg0j0CqKI+u8hRvtmSOz4naMe99TIjPkFYRD6lZqQ+hHCMvmYYij7h1wI9qCQJPpT5U74YLce+GjfmPg2siLzLToS+NpKtvDJ/0btvXzy/tLCsPsA1yz7gHtI+HgGePUXjNb5qayU+GsIUPro5gL1FaBm+Mv+bvcgNkb7+/y660jSWvscqQj/LoOg+bhWUPneJZ79obHi/jX7SPhYVq734eyw/d8Llvb3ufL5XIHc9YW/rPtMNur6rmOy+omZnvlURHb4CPQi+ZcA5P6/HvT0jUVm/Lt/jvheT2T0wBQ4/gZLaPhNTLT/Rhyi/EiWYPsnMxr56OS2/D7XHvekcmj5ef2e+8zirvgNzs77py8K9uKy7vneliLs8qnQ/NqIfvwY/Lz/Gz0K/tYpbvzWQL79p/ie+GcNPvnIwET+bwRQ+1U4dP0wvWL/zHRW+41gKP2d+CT9OCS+/+ChsPjWQtr7KQAq/Tn7mOmWvEz/e0Ya9","RxJPv59XIz90JWO/ipCWPkhQCL6+sT6/Yz1Wv9NA1r5Qvb4+gslnvmFGHb9eSzy/7zSxvrML+z7ZqNS+CXG8Po5DXD9EDNS8EUHGvhJjXD4IeRg/kH2SPKsMdj9zMgg+qzcOvslsmT32uMi+6izPvlkGXj8DW5k+rQyGvw3LRz9ADbA+PkPDvTEk0j6p8rS+F/eDvvuJoT43Cwc/cp1kvw9w7L0Durs+gn9ov2Xhbr/Yq0S/2INlPhdPn74d1oE/Wh4IP1v6Kz8zTy8/i2tDP7kLOr+cxuw+zVaVvsiCfbwRVc++AZSzvRWQRr+KJbW9W1NOvS25Lzys9Ew+Xk4vP9wccT4t0Ck+MFmAPSR5Sr0fkEW++YAIvXpzpT744pc94IpUOwFaWb5N+vA9dqz1vpJNbDwxvMg7P46ivb7QCb7Ewfk9dyRrPZ8L/j2Y9Ae/e03wvzJZTD1JEae+E605PVbBZjyKIpu/utb6vto4O78ezwU++FjDvklCWD31TbG/i36tv4xDgD2fZRI+30sIv5UbAzxdJJm+aOkRvtHvEsBkHi892VjHv/uSlz2OmeK7kdiEv9BMgT3msMu9wAMTPk4BPjwtEjk8UMtmPp+bwj2RQTe+ClaJPBm+377FfjK6o9mwvlnsh7/1KBa9Jh/CvxhLoD549cM8ua6kPDX+k74w2aE9nuPivrwoVD21hAy9TFnZPgQiq768l6o8B0s4PTwfpb39FRc+bVPXOyvzQL5+07+7inX3POpaWzwc+zK+oaU/vtMGwr33YBy/gWC+PWbFeT4uh3K94PHguunXHz6LQd6/+AoVPTgP2Dsd9Yi+2g8RPeES3z1qS2w9HjGsPYOrCD5FjcU+8GREv0D7kL6EP6w9q23qPV+i+D2iBUo+9w+HvzRaxL+NJ8Q9IbePPTHSxD1rxFO/E/gLwNqSqD2kSou/AWARPgJECb6SCqc8t0NUPNeT2Dz/o9q9oibPvqR09z1DGc097Ku4v25Cab0UV0m9GaFCPizonz28YKe9"],"bias":["Xwi8vlNdOr1DIGW92Z4hPomihb30iFY+ZqFHvWZ/GT5s03i9KOJjPW55dL4QVhM+Ufc+vLxlhz7JL8y90qj5vXH/tj5upP+97ivbPbU9RT4dWYe99dWrvgcXO77u0gu80sybvrYAbb0/wXE+sba1vEWvor5Oorm9vM8bvaDmML3SbsG9hXGNvLE2iz4faJY+aecSvoEMFD6oBJo+apQavelYub1myEq+E1/iPknT1r3awy6+sSYBvpm8Y76+7mQ+h3Q6vuF+gb20v4I9it/nO3cwHL4WJm4+L/bGPS3rDL6HnJA9Pj8KPn6DGb5ikuG9V1sOvVY6/L4rAQS+v6eZPUpVIj7pay09zjINvirOBb6bWK2+D+fGPSoO0b1wBRo9Ofb2PdFRXj68dKc+p5GmvNRw/DoHRHG+7iKWvPnQtL7tpSO+0KQEPYhTyz4dYNU8qh84vhZE2rybg6A+d6USviXuTL7F/30+BeLxPV210z1+cBU+jvWlvXGGKj5y1hO+qBh9PjAvsr00/Fw+1N6BPm5PRT7xwMW9bprKPTih/T0Z1B29pAZSPTAwhz5ZoHg+KetivSRQmT5dZeg8UYaEPpy7FL0rdBc9shxYvL7oor53q4s+JBxbvr+sTz7YbSe9PG9TvB7sW77A54A9XzFZPeul37xSDQM+7pmOvmNShz4="]},"dense_3":{"weights":["pcVLvUOU6b2mgJ68vp2AvUrVKT5qW6m9sQJ6vurGkb4x7vg9BN8nPu2mtjxVVNA9eiowvDfeQDvgLYG+eFK3vRnkm71Xfoi+89uovH81FTz8I9i9AmJavTGBa70T8c6+vvjBOxGrtzwc/5S+R7AMPl386b0ooUc9OMssvmhLYLyXcZG9Q9bdvVkD97zwJYW84QGVvcGjg72qc7S99vG3PXTrrr4oHkA+681QvV1HSb194Vu+D+nRvdAEqj1U9aK956zMPNoq5b3nmMG9kNWOPqAA4D06tv68ADtBvjGH6b3NvwK91fuWvsdDED4aY6a9R9MovMW4Er7WtMg9NLOtPPDoXr54og49dcicvNZSYr0iLhS+MkE8vu1nyj3Ojis+/G+xvfvKlj0SjI+9mwaFvdxllLxfJA29XwDoveY9iD05ZJq9njqZPTXRN752mwu+fILSvdguk706xSa+9k7mPYi9O75NXxA9wY5+vf32oT2Fxr+9gi6TPiYFFr3EoR0+5kNyu03QCz5gylc9lY3Xu0Ahlj1dV0+8Q8jLPdS/Cr1r0cg9NZRAPVwMoD2/mOK+g4sRvsM25z3ft0e9AUgdvoCxor1qqwY+FvTZPULDLT5Ixw4+lGTFvDszhT7ywlc93rcqPcvjXL5P4A49wNSRvSJXgr3cgEM+OU0PvZxAQb5gNUk98aSdvAnmAj0gvcY92qcZvuVdob3J9aW93wkdPr3tqj3/2pm89pyXvrpPcL4Lbb27GDqNuwgbsj0XDqe9FE3LPZbJbb5O/IO9JfexOscD+L0/mLG9ps/CPLkxuDy+Sbq9itPKO6VAnT3c1NM6xqeTvHGw2zyYg2Y9nxFKvX3Jkr6auwq+f9s3vsekej1Q0pM9fx5nPbHiF74cPyy+p0ihvqgntr1vmCO9nfDyvrHAZbyOxWS+3S0GvymzHT4rqxo+C1vfvblfvb46X+69FVCMvKsfyD0FzJA8kAoHPuSfvD2Y4/u6Vn7SPZbQH70v9Qg+PgCWvH4Uj72zqQk+","TcOOvbZrdL4d04c8QmTaPF3+QT3gLZ29cXw5vev58D2K9IE8EQaOPktUN70PM0Q+KGaBvGchlb2Lc28+EyGVPGnbPr3OCBk8GMuNPQMZbL10F4m9/0lxPsA7rr79WCU+kyoEvcmKJL72/2O+Qq01PG5vSj2ek1E9KlTGPUMNCz7pvZ+9n4EivroxOTt5b6m9OMoUPsc9NT4zwwc+VrZavYeI8r2S1p89m6URvtRfmL2o/hs8b5nSvFzZhb76Eu+8Ww5nvYgprb7A9e29AhwuPlyklL1DkbW9RoCjPZzqCj29hNW7OcwIPXN/u7s+eLI969OhvdgzvL1VhRi9jBWkvYa6vrwMyUc8XPIHPmC3EL5zDwo+ol0DvtS5TzwlX7A99Xz1vOjCpT3Jt/i9NK/su9sFDb1WBgO++9wPvscYOjxdIpa8aw6cvaGEkD170Ai+/JQWvTvaBL4ZVN68r2C/vLSDab1Ae2+9iT0ZPRx5OL6vBY89SHF8PU+6UTzDTvg9/n1evYsOIrswFlG9zmkUPYZNXDypiZi8a6bavRM0rjw98pk8ek0CvaJKoz3vtqe92B6yPOYjlr3MtFw9qlP7PMvm2D3j8Ju9AoOyvV29Kb7Kcd490gdnPIVKsbvsRia6UVvuvF8te706zjK8N6J+vaLvUb0uu2K9jhdcPMH9o7w5vWu91mONPPe457zmMDe9K9JAPX5OYr2Vgx0974DZvRh4LjwS5q889yjevWG4crxARnk90UyBOb/GtTtEW4u8t+yfOzwYGD7UK849rD1UvZ/3FL0fAZg7JDUZPP5+nT0XZxk9jPLLvCoXnz20NQu9SF5qve2h7D0h5Rs9gHtFPeL+jz0uFC29VoqEPWmd+T1zD9S9M4ervCq58r3Zw+E9xERFPRNZYD2lSyY9MhssvI7AYL3sUIC7IzFWvWzhHj5F3dS9hLDyvS/svrsUQ6K9QL1QvhdGuj1/Vec8/Wc6vWTiDz4Zoe49q8dCPWetG74us0m8yAqMPdThm7wdpRe9","zgnvvIip7D2wItw8W3Z2PWriyTvTGNA9ATKBPFhyPzzAoIm9j2Z5PMY6kLy8bCO+8htvvWwJmrzHmqG8rONqPlgcDryvGL+9tMg4vUOG7735pDO9IsvmvA1u07zeKYI98trrPAYLW72AQYa9iROJvQkYwT0kNj69ePPgPccL+T3swAY93PhUPVKH8TzIqCc9rORZPbewwrwO25q9rTSEPCJBuD27Pdm9I0qXPRxnAT5Zhl883JmOPfzMpj0sJAW945JGPc+2mz0OTzE9jjnnPR/gPr3RZmY9Zz6ovN5hKjwbahM9XSnJPZEAbr2mGrg9TGQWvUkcdb1FQgK9XrMBPki8CTzQOlc+3cUCvVmdWD3+RrI6mo8qvRNrH70DQ+i8pHE/vhw5Pzqsoqk9NN1bPbO08bsk5Ii7eeZrvumhwr07JlG8dz+/vUeq9r0Cfk28oDkavZ8DFz5/kIw9m1e7vY2Kdr0YaRi9SNWUPSlul73b5zw8W1dzvbGzr7291qc9y56kPZtH2r2frEy9uc7ePVtHyD3EvxM9VlCBvdXNYD0PIYc9J1JuPeghjT2Tjgk8dqkRPlKEPrqaugY9AorkPQugkr2bKTI9sUzRveuuij1s5pq91GguPWTRmL3Ayum8DXlwvSwnUr34jBo9oYOvPbCmhL2VDPm8I4LivODqwzrwAXi9XLi8vsJ/4b1hUpe+MkLHu17mlr28OjG+FES9O5n6gr07X4k9/cBzvY1luTxRsEq+ycAhvyWGML8YV5g8txIjvsPVcj4tXfE9N9Zfv/I7Wr37TWS+2PmnPQ30Ab/JHlo9PaCIvMGWLL33hYw9DESKvD+UWD6SpzO67wiWPU4GhD171hU9Xm3XvopbxD3zddm9aXAAvngyHz3uZDm9eQErvj99VD08kru9TjuFvmFQ1r0fNSA+V93OvbVndj1YpZU+hsYLvtHPOL5Mzx8+8VkwPYhJV76Mdtu+lMRgPT6b5rzW5uc9hmwivmxeuT2qLvM7rRFZvVS5b77RJFW+","V3yNvZKZNr6w2g09tS1TvdDXgL3W9rW9U0YePrnDkr4ZWRK8CZyAvHKfEL7emfA8mUUIvvMUOb7Agt29eaervVdm5b1gGsE9zBTQviv8Nb7Fvxy+BPa5ve3Sj77RGsA9BkFJvkYBWr5S22G+7+33Pe/ugbwk6DQ+8jvwPbd5tz0BYGe9Bg2PvrNH2D0ab7w9xCObvLm9hr3xdem9H3mpvEzCoj1yI8M9XcCnvGCpTb5Nzx8+wjbMvU6QEL7nN/I86zk3vgHT5L3VLla8kUs4PGg7Nb9sti++Hz3VvgL6CDsxzgq++fEzPTikAzsjHnq93IdwvoUMxj5y1yu95b39vhuL3T1Hyyc9FyXzPE1d1r0my969fkN5Oz0uSL1aRTo9E6GePQujqr0N7tG9kwvZvOLrfL48CQ+9t3cavh5U4j0LJuk8fMA4Pd89Wj3ZhFG+3IA8PQbrzb2MlIK8yAPevbl2Mr4+kYm+51QTvv7mm7539Lm9TdS1vZshTD0Alcg8C/qsuw+/mb0kDVS9p0jlPSU2Ej5dPIK+AKPVvsHxJDze7gu+3KXmvINf3D280c29LXuAPmrfGb4P4aW+w+hDPlIahj3Sb9K7HG8FvxQwVbwJSQQ+Sn/oPXFqUL4VxjO+jE1wPrj3Dr54rCu8fgZfvO23vz0prCU+AsCBPVunp71ygAm++KzqvZdIOL2LyTq8dcKfvT9WOb4KrZa9ej34PI7gjL6nysU9QIp7vOd7pr3sjWO+eoGJvGDds703L/k9Nxp9vfZfDr4vAga9q9OOvRERz761Bfe9gs3rvkEUUT7Z+Hu9OR3RvgRJ8T1M48w8N4DkPEi9t70cdia+bsawPV0e0b5LPHy+TxjYPekqHT1zbgC/43P+PDahAT4J+aa9FD5Tve7zMT1Yb2C+KD9iPQs+T7302Oo8WIOLvTmtwDytlK69WumbvLeRer6eIoo9KiJrPk9B8b3TZbY830paPFT7hb1vjIQ88LuGvjebS71/m8q9XQySvmEyED1XEF29","3I9GvpWACTtoZKc9Pdj7PT8/L7wnHRo+VZoJvnxan76WN8G9sVq5u78UCTzf3pm9J8n+PaRuP71F+co8iz8CPJ9rsT0Idgw9plHpvKLRVL7IT0y8mmHDPVi5Fz0vJMa9wf+JPTV/Sr6uD0O9T1A2Pnbz+zxkvgO+fdBhPuVqIr7UPo67Wd9MO8EUwTwYKWm8cRhPvT0eSjxvlJI9rV6LPVbxtL1tuiI9lz2rO3OTbb2hSEe9g7tmOwjrFr5HvxQ+2tMJvpQSQD0/Wsm9nChAvCnnd74sQhQ813O5PAWiYL3rPqw7UGN7PA5+hT0pjgc+VtiKvYV9Mj0VuAE9TWCUPahxtT2gRzW9N2EIvo3Zwj3XSbG8Pp7IvQ0Nez4nQAQ+oxRKvTTjh7x+Jzu8y8U2vStY9L1mZ2W9YwL+vRrHlr3d8LS9VxGRu8J8KTsBuXq6uxWuPY35Jj3SrKk7FFQRvrBLJTznxq69A65CvBV8sDzu0sg8iIkevqDzOj2RtMY8oeqTPXs/jD0c1jY943xIPenFlb36TlY+pmbPPLUwHj0a5Uu9e+72PFM1Ijy7mYu+ycrlvYHwWT1zSq29u8NYviZzrzxz8t+9J8ENvtA2X71PBSA+QFwhPo3YUb4/k0O810dVPawIoj22FgE9XQtwvVu8kz2U6yY+fvOlPRHwr7ztRIO83VdmPfPMxjwW6rK9s0SSPdwAIb4QwhM8NNV/vWXMRL5Q4AU+5DjyvcH9770jURq9v3v9PIcNvL1L3/c9sdz0vUcCRL4owge9yKAyvq5tCL1BbCc6AbJ9vVTva7yaxyo7ufCzvdGp272ozgU+Z++IPXh4tj3xyzO8V7UEvhoeKT4Enrg8gqRzO85vUr6OXDi9/SyHPT9VKj4OjQo8kWoIvrHKOr5UqjY+fl0DuosrDL75PW49tFXSvKMR+Dsjw+M9XHjIPTCd27wdppM9cjTEvUEjjr2QEho+KEX0PWEgTz4eyd86B7zsveVoWzuY/L28vBVQPQqtyL17Aqq9","Qd7GvSd+Tb3cWy+9HlBNvO3S8rtdbhe+FduFPTsz8jyMaTe+TK8TvmXwfj29S3+8326LvDEN2zze2AW9QPm5vV1CBz64SNm71e2gvD9oyT0wgxY+aIQuvYnWYD5/KY29DGVwPLr8GT5fgHq9MOo5POjq3D2LWje+Fed1vaeK4r2l5os9rh+CvjOSJ742ane9I0f1vUXXRj0JQMC9QAIyPcxMhb0h9Ak9yivRPMtKD77CORs9LPvsPWZioj0bZ/S98OFhvQD2lz3kwEg+FPwzvfHfx72lI+w9S94HPCIZCj53DnK9age3Pa/20j371pk8uG2YvbjOaLxj9Dm9+D3SvfoWgb3qkt49DLj2vQorBj3ZbBW+80pHvqh+cD6RL2W962MYPUSBkDwhO/o8RQwUPWbebz4kLOk83wZxveb8wzzx7tm9A7zRvfctsb2TlDo+FzY8vUS6Cj5cFZi8EcmHOorblL23pB27hGQFPbfOcT1DXV8+wtFrvfEEpz1MRsA8SSWnvA8ERb2dOpI98gJEPToWEL7lzI490S8Ivktek70GfgO+XxYLvq09WT7Fm5w+DYxRPARQbrxscXs9ynlsPcIHa7z2xCE9CsAevW4kYz4C4D08DYdDPtk2GD1kIVM8w/n+PYCrBj6qkhG8gOELPpfg1r1LR+k8wto2vj7tH76hBVM9es5iPld2NT0Gdg2+JiuKPVdsMb5Y3HE++sHFvc4YiL1+cUg+5omsPWjDiT2OGks90A+dvL7R7b3aJ4k9PzgpPS/sjbw3PUI+F+2KPQRGqj1zGMI9rjcgPQmIIz0Nx7u9yCwbvruNfT0rZ8C6M1vaPGSPcj1Jeai9fwEJPV6YSz2l5Ni9o1LFvQidbL2JNSS+KzDUPG3igb3+q+W9zXWCPVMoyj4UAt694kS1vE6vbb0kbKW9iSeDvYpnWL6HMcA9ub17vq+gxL2qpQU+sNVAvZbOET7eSaA+5sMrPpIdPrwFwco9fnI/viA31Twvc1w8bYAmPmdnSTzfj7u9","H14/PQRonTzwY1Q+EyFNPU1bjT31L/i99z/LvCRYMj5R6nS+UNclvhC8ML7+4/c9YRWRPc99Jj4T2vY8v+wpPMEKCz7Vvmq8w9EZPuQme70dgIe88xjQvJ9ikz5WiyY+TQaYPHDz8b1fLTw+VXVyvhtcQT4sjQE+hihyPbmUMDxE3O89rPl6u39JnrwFUds997J0PeZOez7eNSC9iNZCPvqQI70pRb88fTJcvd/sMr7M7g0+BJ9ZPVXYyT1fO4Q9KTyMPcVCtz3mWg49b9T7O1fPyzyxOjo9x5/oPUCQrz2L1wE+EaEKvSaiyT2FvzO9iYacPfunD73XRM08kyoTvsUGgT2+dsk9r/v6PKtS+ryufG++cFWAPbAw27yqboq+hNqwvdzYVL1e1UA94iZMvb53h70xnAQ91S2RPlwncz5bJe49oU0IPcMnjr1abVO9aWZEvWyPML4yqka9TKRmvGoFpbwoakY+W9mePccY97243VM+AhulvBR4GL48TmA9vG+3vbdbB75NA38+j0Y2vaxmhryCxZq8P3yCOSipmDzBA6u8HeT4vIF3Ub42zIs9qB+pPQyXAj11h7W96JsRPWgR/T0l3dI9OGpaveZNKj1yNfS9iA8cPhNSTz0HSRu90wBtPqF3LD4NqN09Ks9WPWuBery9yca8BtzkPCqU4LzwMH09csBTvT6ncLzTUQO9O5WJPff/bL2qEDq+bGe0vfWOkT0L9Ny9IeEHPiw72jpi6BA9PKquPDT97z27CyO9Wb6vu7jeSbwltSI+lyXFvXxZWbzTtly8kPk1PTciQj137ge8KENGvTkKFjzOGyE+bNMLPOJXOT4X2w4+QhGyvJlHnb0XnYG+/72ROpZFTz2YVZC8ssFEvRBjCD3ZcPq8WUu1PJvRLr2/oxs8CQceviaBhj0lOL+9GmrOvVfPcb3TXMQ9HQEbvv9uvD3YvhE8MSDrvX6kt7xiMaG9iznevesUXL2POec86iXCvBHTEL6jtcw80bgMPtuJoj0ow/u9","YnlRvdwom7y0JL67QxCavLXGnTyrw/M82Q+dvCQC0D30v3c9pt1gPepyEr7wl8M997i6PPVpiTy75Qq+S6zjPVVXsTwkaLI9gD7evQHt/j2agLS9nF/7vDbqMr3lmn+9HICkPUe7HT2uvZm9ACkIPBHyNz0kLBk6fUPDufkXiT3sPKe933XDvafKED2y7pw6jR6nPbxoMz4GIva9A2SnPV5YKz25BZA7noeHPY5g+L3Uk9O9J8HmvFk/Tr0iM5I9d+ITvX0CbT1JGVg8yydhvqf64j2zoiY+IXRFPEdawLx7kfk9mIDJOqkwbL1N2IS8QAu1vRJcjj2TykY98tiDPKKZtD11y5G98h75PBYotj2ltHO8H2GavfSGRj1Se4I8F3GsveXZvj2534A84dgevqGHBj7TlTg91MBYvYClqDykN6O9CpQzPo+tzD3O9ck8+qQnPgABB77W4jk+ejYXvXs1Bz6tbx496UTIPZ1OmT1HAX+++yB0PamODb7l4b69xr4ZPYOyH75nKkQ9DTxGvZL3oTxwEZA+C1YBvRp2CT733w29DNUNvYQrirzjPAs961ZpPZrgOT20YAC+kezTPeW0XD6P/469JeLuvZn39rwa1a89cdKDvbnvjbxZvjG9cdl8vC14dbytT3a9XoskvdXDjL2NmF29AE/HPZlM2b0cYYE9Mry1vCMs37s8M8w8fmSNPGIsQjxsauI3YsQ6vY2Our0l9kA+H7U7ul7ptL1Ei7k8MOgdPu5eSz5MPpu93e6DPTk43j1UE6U9V1SLvfTUOT3rXUo9+PElvQXVE76BUfO81doBPhiEED79UNE9y2n5Okxuaj134SI9M1TCvWJ4yD1WJYe91mOQvcoRAL1txhA9CChfPDSbxD01ivq7p7ymPaDSvDyBqZ87U9WGPuyg8r3bYTo9bp86vQp4FLzaLtW83NvjvVcYGj5gNHQ+dGUEPhV9/j2nr007RKynvMnXxTwCp3I95wyePejjLL3pyBY9gfAJPeky9rte6xs9","/PZGvmtwlDygJYM94Ee4PWlokr30OVW+snILPve7KT5JdZO9dR+bPbuhN73MjJW+dEhTPclHsD1YuoI+KHVTup86x7wpOYc9bnFtPTy5KL7JACs9R1I7PUirqTy0XlE+1hcNviJtDz71aCK7ryUxvfRYVT5kJjq9XJL0vU4XLj7hbGm9dzWVvQhTHT44Y4e92grIPd1d97tnd9A9HzhyvVv3Nj5axhC+bFA2PU/vWz7E07s9+IGkPIoQzj1trI07oyjsvdPJMj6PGUM9cJ4WvtMwbb5Le2S9IhEtPuufgT1WZtw8Kh3mvSNERz2kzpe+W9IuvYvqAb7Ne6K9/eNHPpYZtL13eRS+/GOJvZ39yzxw2589hqLLPN4sdL2PQYC+O7CQvXHdmbvcOIY9+JHGvdyGBzya/oS9U1cSvuVwqT0B6Ta+2TqgvRxUPD2sFwE+ggXDvdO3wr0SVpw9N/a7PCZLWT1NyTE++/yZPL4RmL0xRn+95NKXvZZaXbsVywK+hjegvaS/mb7Ytk6+Lu+Xu7caJL2GkAs8laS4vOlbMr72pty71SFwvZh07Dt1caw9M3CBPdlKab3otV6+4DLMvGXW1L0YkYU9wcmFvpWvAr7UsY88xLoUPd/Dbb7WyVI9hF4jPXkzED2Ilgk9bmSZvIxQgbzrKFe8lBvDPNxfmT25ax6+Zm9uvpfkRr2LXTe8SDR0PXPihTxUWfQ6EIT1vPzPgzzRwRC+mR2ZvLFWJL5MHZE9Ke4WvemSj7yBZoK9PwvBPW6Exj17LAC9C0kWvhNTir0oowU95X0FPMMGy7hcfdW8VdNVvhWaAz7YWnE9LR+nPZZ92jzJ5XM9GLrFu6v0Er0gs6y9WruXPQ/NXb7jzLm9U0q9PUchDT19UTM9aR/dPPD25L0X9V6+n/DsvZhsgL7yjkQ+Y2EwPpFzizxsPeO5vjGgO9WCBT4Haoq9ss8jvH9Atr2AmP08Es81Pdsihry2PyE9F4Z2vgKPMrxjctc8jgVvPV0nq71WFts8","XTbUPD/zYD1JbD2+zGZDvECMJ74f2Rc+B6CdvXwFXD6rsVy9D1DnvM78ljzk2Eu9jqHEPb+4gz3VRwY+7twLvUQlFb6ZDAK+hJc2viuIjb1LYQM+1gubPdffOz6sjAI8ICf0u0nezztgdpG95gbEPOhRvb3Is9C9V+cDPlSq0b2sfVg9pFPPPLB2Ib3/g/U7ZOdqONTrvb1Ttxo9cgijvR86770IMka+qL/cvPP+DD6bD1M8HrcdPTtau7y0PnG70YkaPWw8+z3nkY++Hp3Nvc/+ZD6Fr6q9bE0evdLoQj0AViQ8eD3xPeAmTr2wRGC+rfM+vTmD5rzBVyK+lrOAPXV2yD2L8Is9UWTPPV1ElDxnjS++e3VqPtGrtrw6Mc+93jLZveviET3BIok+gE5Hvq1Pjj08JRg9aCUEvrrfDT5xhzY8gWWBPRI1DT2P+Wo+/3Z+vJNKGT16tAm++xIYvRKz8LqP/Zy8MyrAOV2Z073cM7m9euxZPK3ZXDvUWNm9s/hWPJVVx70GaAc+he0/vhmS6b0zz3y9alHCvSLUpby5Wo29kymevdgYKTxqWXw9HOo5vcDr1z2+KPS9ZcGVPcwizDwU8Ja9PPMGPiuD6Tz9vbU9Wcy+vYEMaz3d1pm89pI/vXB85z0CA+29N5UjvuldzDz8QNm9QFAFvmN65rzs1zY90Ct1PGvalj5Exck9tlN9PVWF6LxVYCI+YfiOvbBw1zx3voO9lzGjvboUnDwamg29DE6zPYb7sb02+gI+RP/TvRFPbLz8CYo9lYNlvWjglz0KwtI8iZwBvTVbj73KEoG+1tedPeoInb1lY+g8qnE9vDJXL71VvOs8KiV9vbjUKj5UY/w9NY4cvr5Xsbyyop29yuNvPfGd7D2PRSU9nfq7PffnX71FnKK9L0JDPtJ9ibvLj3s96noYvtUH/Lytkyu+skquPJwMIT6ABNC8l3hnvSrYIL6bcsw9ScAzvXhYtT2dBzo9BmYrvbdNsT3sEZc9RtfLvHVPED6GWk29","MrofvZXV8j285ZS9udsTPuEhPLyp+xA+AgbBOnTThj3XqAg+U23JvU8hoD1ZaOA96AwEPWoO/TxFHwG9npGhvX2WazwWSqK92FFcPI7b9L1ppEw8MOjevMoDvzwYFxg8EKLJvNJtMb7Lep4+RemmvROUQr5KbkS9BDKNPAlLmLxN7fi87AhPvVAEij2fdFc+quSyPfmJar7PkQC8pLJXvUsE0LvllK69sk8OvbXSpj2TovC9rqoSvQVaM7xRlZy9D1/3PZtkkLuBT7e9pBxdPXposDyBd0m8/Ta/vbWnML1qiIa9wcAsvvPWFD7pZ7E9Qf2TvAmnij0/SFe9DAOrPZGKmj0tWJo90gPfPTt9iLxLhsG9MulcPvyXvjzzmF6+wk6CvFUGwjxxOYc98nwZPnDUHD2k/4Y9WfChPdIOUbsbZMO9pT+JPU/inT4WYey8pCAXvq5Orb23k5e8EXWbvNL2jj2gbai9VZAlvZjrfT3qGCs8tSkYvWvq7rxDNno9EcXEvelSdT1ytcg9LldBPRtlVj2UHYs9MEcHus/Roz04pw8+M9hqvQJBRz29Les9vI6suw69K7wMFGc9sFawvQN6Br6Ig4E9gPM0vBL0ATzHxe89F461PXGPwrzP1bk8Jp9nvYGCCL7ikze9W0rkOzRIHr3MGxM+4k6hPAtfjj2wdlc9LETpPWUdjD1wSDG+fbnlvEvknL3ykxa9A8CBPSoHAr59jaa9SD6bvVHuFr3HmGQ9rAJQPRHrDz0fBKW9hMH+PRUMZb3xQEg9wUQnPkVfVz2eyg89V6EIPZjKV72RihY9ik3+vQmKtLxAGFg+oFiFvgOdLr31qhS+ofDJveWv+LtFHCk9zPnMPKJjojxaNKi9bN66vIKKA7y9XA68U+RlvUkVSD3ophs9t36rvUL79rsErKY9ZeIfvrMJaj16cI46G4RavQGtoj0tryg933GPvLOjlD3LLgO9bFdzPcEZ6D3+Kac9oKjVvRvF1D3++ak9kQJkvpvWpT1bOws9","p3QavnQvYT4tulI8FuMfvSpJjT3LSIa8cAt3Pg/lIT5UzYw8mWaQvek48b0Z/hQ77rWfveUUmD1OFkS9TpbiPdWqgT3WQfk8V3BDPMPXHbx2Bv+8MwXDvWn9Ab3bGDc9DG/MPDwrgbzOx+G8UvWHvAs9y73yAgK966gUvKkeiT1trxs9vrf3PV90Hj60Bt28/aVmPWj2Lj7rmgM8eKC1vQcNrLxyh4Y8NbINvYtSkz1fXOM8ImrXvezm7z39OY+9a2WJPWNx3bxBPME7+h0Pvabosj3Qewy+uPkMPpqRVTz70KO9Un46PVlFo71HyRq+oirHvI4VUL3atpy9RA/WPGzfNzx+mbc9FqzLPEuyW71LN2M8y9dyvkKiFj07u6g72tSjveqcuj2/Jkk9U7gkvg60Vb3ekBo9tBI8PPkLo7x8wBQ+c4sZvvzlCT36v3Y7ei6jvV3uvr0dIka9ZFIKPkW9vz2qMNw9FGuHPE4U+T2A96k+hvcsvQWyoztAbPi8UJnAvWrch7z084G+KYQ6Pkta3b35ccY9RPLzPFkwtz0pX2C9mfbLPZyd7j1Z0ni91jOXvU6B0z2H1Cc+kIqIO7tJTb56AxC+90ESvcxJNLwDN4e7A67CvSgM9D2iSwy8hflgveHOxj5TGNS8DSvJvQGOorwXmlU9/ZD+vYgn8T1B87A88vQ8voiX3bsehjG7BMlJu1OLrjwBIiy9D8LHvaE8LT7i98e9vKQwPbbjHL0e8xS+XnoTPt487r1NVcy8rB76PY4fiD0UUk8+NIZUvRXevj0wQIc90Rh3ui9UL74Kxdk9dIqXPee2sT0ypIM9DfIpPviQDz1DrBM+kMLoPUYE1z38v+E9nl+XvbrAhDuTfEO+cnEJvcU6sbwmU+08lTKXvM5vTr2+vQQ+MbMKvgJ1Wr2QC3G78JALvgKqGr09SJo92k7wPH83Iry7dJk91QLsvZ3vv707cAU+npJDvKd8uL0571o9QKICvEWT7LxObtE9bZuVPah4Kr214IG9","k1kOvTGrhL30dr29IHfDPUdmVj6I5ok9f/1pPXDwJzzrLHQ96s0jvX5FLb135qc9K5tVPcEqBD4xRrM98xVCvbXQWDwWKjE9k3q+vKBoWz2a5jQ+16MLPvMIVL1TZju8IP+BvYkghj0cX0M9qZ48PPXFqrwPzRC+b55/vRGoN71VZDc9QTcdPmQooL0xfgA+xMrNPV4MjL08AYM9qTl3Pe/7HD7l6Eq8GxxOPQd40Lzy0Su9Ng03vkx/Oj7pXma95gWPvW3Rmjwvg1y+lY3GPY8R0r3Jdno8T1KwvexK4z3FcCk9p4P7vPLQK7tFs3k+AdjAPC/j1zyGWr89vZ9EPSSjZD0qOBW8sbXhvVX2Y713hMw9uq/TPJClxD3/Dye+vUTMvfso6D0/xr68FhYOvSAE5j3njTU9DFLFPHAB0T0vSrI8hY9TvZVJZruL9Ca9ng6EPZL7AL6OSSw+Xhm+PHCWFj6Bohg9FZ8rPTRIBLvHmic+Qg6WvATwTr1aoj2+qrwzvbFlt71BuE890XbrvCW2bj3Wkgw+5iNmvc4YiDx+DBk92RFdvdUKuT1s6907kn/wvZZ6IT6zFz29/aMYvVg1C7xxPmG9yicbPUMijD1dz/k92C+SPbTN2z37hmA9B2pbPMJV+jedGBM9zn0FPbX6jD3VWEE+Rv2mPGqYPT1ixii9vnDKvExP8Lwl/SQ9sbgYvp/crj2qkl89af0XPbmLxz2mkEU9HtzevTHU3D34gxM+Prx+PbB2IL146129UdDAO9lGTz66CW48YDfvPcpMdz2HEbM8344Fu5YJKL3ZuN89sKhQPA5qA72TAK88uh5Lvkwhyz3VKJy9EmDEPVvc4bs82OA7TdfCPULATrzhVYs7plaLvXmTrj26Tcs8JzIyPjfzpDuqlUc8ir4EPolrJz4BOAe9WSAYvnRdFD00V/C8PbuwPXHGmDwduow9NgUPvf9Jij3Gur88QjcHPkRVDL4nz0O+lKmKODfzG71NQQc+5FnAvc/sjb0y+8Q9","Z5YIvR89Nj5L8p08sTmiO87j2z3yCNs9HjZQPZf9ED4nr/M90IgUvIz/nz0rUWU85ZacvOWWLzwUdx8+Gs8MPsB7Tr1Pu5G7vCdPPhkTuz3jn2A90Q7KveI2dT0IOGC9w+ttPS4cQD6mfgS+t1THvNmkQT5tf449URULPnoOxz2Dj4c9spnoPRudAT7b8yE9AoDmPAyuVLyzYo49Z1WOPGX94TzTRK29pwe8PcgHA76eobQ9zlhHvRGO1L2eBgw+BSqvPJTIM75nEuC9AHjtPbM4X71dagY9CHLaPAzVTT3tQSE8k52cPCvTGT6E3BW9X9hxO4L8njwr/kY9ubYCvWrMGr2Lyas9N20dPqYJKTtmR4E9PlrSvXY/kj0DbAy+DPdoPBMq8z23Dy68jn+0ux+qcrs9vmU9c6UJvt5fVz1L3ls+FU4FvtVZAz5myS69i6TMPDAOOz1LD2O84lALvpdxez2wULu9xFLtvMhABTyeMng9N6m2O+keub0ZOJs9h+zkvWK2vL0d+669t5WrPeCQnL0PDZ69nJ40vmlaLzpVF4+9yBQ8PfV7OzzqWxE8fKwpu4H9obxkKgk998qTPCkEBbyI4Vs94UFlPi9Wg720xD+9qFvZvKauo70pDiu7EZKru/8mAr7V8ZQ8USyfvIiRTD09Ep09hNaGPIwonL3KK3w9LLmlvVfW6T2QH3e9llCFPb9qBz4ndJS9syVOPigBCz09yBY8TmZtvvDxjD39rQo+m2MZPrntIb33o307yYvPvIDQUDwCGIk+diXZPBPhaj00KGQ9qXeBOTZK7LyI0wU+9+QXvRzsvbzW86C94HOlPU3uAb0Ndek9LhojOCyyCD6/X/A9tn4nvuJE0r2a71E+RUBEuiG3ebxwfzG8zu2TPV3gqD2LD7M9i8qcPhiWRb1fcBS9ahEXPBYsiDxZrxo9XmGwvV3BKz4pskE++v0IvQcZwbtvdUC9CgdTvVC4dT3LBHQ9mktGvKwI07wTfK69m6j7PVs8pz3/zSi9","pP2XvCDGVb3BqFE9JMI3vfuCpzzVe2A8WHbZOqZazjz6kDi9WEdoPEsXtj109v69V+AUvu67+TtH9Nk7POcEPXF8qD05qAa+wB8HPoQRoTw9EeM9Usy5PSeeo7wAvqY91QsqPDBfT756ezu9ds79vONGBz5k8h49c8muPXGV971gTAU+IYwUPnAJs73Y/CO9yqONu9vvtDwArCo9YjOIvVJ8Nbzf1Vo9gV5pvOxs/7v8cng+zu++uysbrjx/qRM+F/H6vdAslj2oC3m9Y84tvJiqpLwt94g9durRvQl/wjxg0Sk+datYPe1EGD3f4xm9WnmqPXZ2CT6lpDW+JRcIvQNDojyAK6o9uvvSPRsEOj0xGp89Y3h4vWov3rxhJ/W7XCfpvLFgorygE2s9QJugvZu4ob3sOL29SUiFPgb8ij3Ny8E6gtUYPt4LWj5hZBs+eEWJPc9mezxIvNg8ydpEPUT+dj2mJ7o9RtwRvr8XN7075iw9sz5KPSOJQj1FAn89hoIdvU7m3D127rY96jNpPtL02T3ElOE8RDMsPLC/rDsYFF49eMzXPcl/uDzbDSy+FgenPIKhS72/MzA7q2k7vf0fML5fcXq+gMJEPrWTtb2kwXE9HsUTPXMsoTlcZyU8aYfBvVE24T2TEe+8YyYjPhg2ULsIm+08SQgvvMupuzx3cM09uLHpvXYMID52KYU+6J53vo5oZb0OmdQ95Q6avSP24z1bhWM9RwB8PlyrJT03IUy+yVkhPYh71j0boci7uQFMvpAF2zy7lB08lC1dvMrxCT7sIKY6ae0ovsAN/jxhFoK9rXIEvgX5Z774t5U9wsoJPsxODz59K+I9i4UVPud2UztmdOu92Hn/vaBIHj349m28UNd4vdil+716nuS9Hq8+PjnuDTzE3Iq9m80Gviqpqj1Har090YtMPfwL9bsLB2q9xoE7vslKOz51/7K9fArwPSV+3T0dRSe9A4N1Pmp0Ej7RPpo80EOFPEahxz2lAsm8l6T6Pc0qcr3FMje8","bEKYvZsmPr0vLR49AAt+Pb5n4T3YoHE9Aw68Pk5CAr29Fvw9S9DVPCd3bDyamY+9+1HzPbSbGT6AfC8804UgvQooqT0iElC8iiZoPcxeqL2UpEa9HCJDPlCLC76v6Us8MEhFPfAaKT7w+EI9OxIPPYSi9D23Aw+9BDTovQpgyD0Efjg9jfp3PN3PAL4R02m9nNcaPbZgGj6KEz0+FA+GPVsapj22zo48QYAPPkgvmz2vFB2+IgIYPv7IMb0trWE+4KMXvtEnMT0clWc+JH6mvWwBjr1B/rQ9VP+JPgnZWD2DoUc9AX7PPBBjID63Eni8HqnlPTcvXT5H2/89tDjXPWIyMb761gk+RXZXPStBFz7VXj49MgCNvR3Umz3P24m9XpxSPl7io72FDvC9M/KtvfkO4z1hjvu9+RuZPVpyAz7LBLu97ckNPl0H8r14iAe+rhyZPYMjjDt7AFe+yLtLu7/NhrvMaye+RuU0Pa7V3D39Tba9vuelPWH7+jx8A5E98TKSPGL0Q761poE9SFV7Pm0ITT29zIU+63aIPaZzLr29IcS990vjPX9BiL0tzB6+rR7ZvDV36bxxINu9QipsPcs3dr3m+IQ9ZKX/vIoAT7pw4ry6JA2NPaUCOD5+5bw9DbMKPhycNj7s0g29lGAhPt92AT5c3aI9hCNOvaGhgj58JDg+OIeIvIHWlD59LjQ9IarBvLSGJb6g8Tg9jbuPu1ElgDxv6cU9o2D3vc/3RD0vUHC9TZ5YvuJxjT20uw+9ltW/PC6QdD3jy1g83WEHvpWMpz3Z2h4+4ec1PofbeL04UVk9iKmaPmDaOT47CRY+UFL7vYKBvbxNHGg9pihDvpsQwjv5blw9PmIbvvTU2bs98Wq+QKSyPV9anTybK7Y8rjDIvTswET4AiRm+g1faPYcNH71J7MQ8rNc5veeXPr3mVrY8Do6Hvd3UM70b6g0+0mQzPT5d170qDZW8ILjEPSVim71MHZq8NJnyPEKM/bwb7mM+Np5+vIOEjL06sTG8","9dDVPAFT27w/kQy+g+8tvlW/Bz7Xujo7AeYAvRUa9j1U26W9U90lvW5F371Npgo+FRGNvPcABDuoW5w807I3OiddN734oXi8f4X6PGX4nb3HM3W9r1DlPb8e4b1Nce09VvGRPJ4fuD27SII9hqdePOsqgD0sbjE7YsbEPGjsJ765Nhm8d8wLvf5wpr2g/Z49r0Z7vaE5kbzdM9g9El0GPnXmMD2XIgQ8Ow1pve/e9Ttj8R+9NXi0vZFm/72e2Yy9t0rmPFA+hrwwohG+FUnqvMZBMr6CjpQ9EMVBPVMiKb2kg4y8BS0uvENrv72ihhk+U4TjvdeC0rqr2EM8FBwkPoOrarzyMB09fa/gO97nKT16SSI9VsgZvpzLgj1lw4y+YSjLvb4IozvVaMi9G+8APXB3wzv1cYw9BYb3vSfParyWVy89meuZvbu6Dr6hjwk+Q+9OuhThPb0BW/y8jKs8PVQXob2AsVa9341JPCxVGD1tAea890OKvE6V6rxDaHW95IQRPfhmvD0N2D69tKCGvf5firur9oW+eZaHPB70sr3oNKQ843cFvn1CHb2yxni+RCufvUaghL20oM090yHVu+HIvr1KDAO9f6msvRUV570mktC9S52FvdvUcz4KJUw9o4jovXYgqruL0mu9qGviuxhCYz3AMJo9ON5EPHecjrwmdPc8Vt/dPYn7Rz319YM9bP8dPdv8YrtZAOC9HVSmO7XYnb3Z2Ei9wJc+vMYoNT6i3gQ+R/f8PFBhM70OoeG9TchaPUdc7r1wx1o92isZPPS+NDxwEjA9PBuJvCsAr70LPzK9UIDivXCi1r0oXSe+kvK0PYiq/ryVC4o6m3vuPcR+YDzchBa+xvAZPUe5DD4L4DW92jaRPvhaajvFdBe9U1+nPUPfFrwldgc+Qrz0vOYWKj3DOK29vs2bvrdWdjvenPi7nYknvg2xKj143Le9zEEVvT496DtXU/096EyjvdbC2b03WUs9KVIFvK/bKz2hVD885x2APfaVYL24yta9","E7xtO0HOpb3c7Eq8bpm7vC/XBD1kCB+93AIkvJM5Ej7lF8I9eIWSPaqe4b1ovPs8y6HBvY3XqLkujiw90Y1DvXsrqz2lbaa9mxw4PeUJu7oFb6g7SIwFvhhyL76Bb/a8jXSTusFp1722dfo9yjOwveMigb2GHdU8erQMPXSNsz01afQ8w3IwvQWxib38vti819pHvhJXKjx0TrG6897RvWBeHz7rx1Q9/cnNvdl3MD652HA5aZkXPV9E+zuuI4e7gH4tPS4XMT0MMCA+dO0CPmpGEb4ULTe91YQpPXsFgz3Zr7m9f1ZVPWY3B74AsGy9bQGGPYsXqb0zIUo8ADkJvWTykj2c2bW97JjmPdl0IL6aegY++9IgPLHftr3EYWo+YJ4svsMALD0Dy+G88saOPVZV/r15u7m9f8LHvPQwsr1rI6S9mmSwvV0nSj27upm+SVs8Pai+8z3sphY+4RdlvIIruD3cEh0+8m2evcw+2LxvPK48j/qNPeLMS76Q05o9VhBHPQ2Rlz3fAI08uuMvvoWl8jxNmaG8sIxLvW4Z9z19g5Q9Gx4CvhmYnLyaKKE9qeOePG9G+z2kKxu9hQPdOiKxFD7DrR69w1rFvdhXN75OT2m+CQULPu+hnT0GZCA95ku5vTd3Bb58GIM9PLftPF9kAD5P6OK9erc6vtqWvT27kzc+3pEfPCPZXL246Ak+138vPoYaZz4M3p4+rF0fPq3j9701Jg6+7N4DPfwqC73mEhm9UIUJvoIGJz1bWkO9pBnGvLTtNz3h8hK+LbJ8PAA8gTxIfdY85Od+vTSp2j3svhS9sy4Cu+FGsTyanQo+VRcIPdhvlz1FN4s9d48EPKqqUz5RFlO80eCRvXLtwj168zo91oCiuwGJvb0Yupc7fZjrPJ72Hb4MH+Q9GtKXvg9/bTv92I+99CdZvDDS1L0f5g8+WPjRvFjJCbwUXI4+IJB6vdnjTL79XC++3/kfvmF9XD2bodC8qHvBvaSJoL3COCs9dlWEPNVNhD2uh888","SuNpPGAo870+txC9bRUKPjgeAT1OBRK+GEOxvRk5c70i/pg86MTYu4h0oT1BkB29kAWVPZ5TJ76csdI9hrA8voUMDb3yURs+ZhE6vHLXvz2F0U49vL6Fu21sN7wSYCy+8CFNPU1IOr7LxzW9ery8vc+6ZL0PHjS+peOqPbQoqT2fCmY95LIUvuDggb0tZLI9ReUtvIgIGD1Ds1+8eFyZvTCEPb0+Vye9mfGRPNabbLo+2Ao+3YG4vV3HNL14hC49vIO6PHHpOD7Jwrg7nFiOvecRD72AMgy+AV2UPWtZRT1UNRA9xEtDvVBMTb3tac080V6zPYlmej1Ts1i98n16veKq872J5/u7FdKXPWEcorxRF909iRGqPc5K0b3RJEI9sMYyvgxIgL1Mk8a9K6yFvGG/yTxUZdq99tUtvK3g1LzEXYk8XNmCPkmLqb32y9Q9nh9bvRgwaD5JWRK+WYqXPfR2pbsLu2m8JGxuPZ7AK74JUOK8jd9hPTZR4Dxm/Ow9y/eiOhRr9r3b3xg9j5tHPhXuNz370sk8+w3AvPOg3z0XJTU+ZQ4oPldpuj1DsIe+cIupPH9hZT1Y1+o88OPHvWVKHr0WL/K9LGoUPSv6kD5ZJO+9b1AevqQ6kz3VrhW9uMcFvvRCwb2aq+69HbdFvSsWfj3zgea8YMj6vXn0yDr1Uoq9iCpEvKIjWzwogdO9zV0EvWbA773j5xq+U1EvPBMzlz2lQJK9cz1LPWg4o72FlS295vM3PM6Gbz2rtCM9fDMqva6x1bwoITk+GXBrvGBeKLzKky08I2EoPRXuNT7YAVc9L0AmPqQNlL0pca2773OlvRGoQL2SGzA99eszPTUpXTv/8Si+rTR4PQlnNz2r05K9S6cOvp5MCb7UGwS9dkpcPanRj71bAeK9DYa1vpD9nDzgT4U9mjJ+PD9D+Lwcp4S9FR7Gvj/5XD1TYwO+N3ycPNa1L7xPa5U9RZ/VvbBZy72RqnA9GRr/PUSna70bLWE9fw9DPHNU971SYmO9","qn5LvbEzF758+2S9S18JveiwQb48zdW9BEB9PFcn2b3Fx4Y9vGAAvkjyvDtFM7o7+QJgvamml72cIhM7doo0vKR6u7zTYrE9MMmfvfjmKr2TSBS+E5kNvcBaOL7tGNo8pxWvvY7PO70Y2xS9RlzyPBKgm70RLis+wBeevVgqab3U0v29Z8EXPqurPz013348cig7PsUk3rvT4Sg9C9cyOwnADr4SMSe+dke5O9ERtTx+kPM9nYDrOo74Kr40vhA9j96TPU4kiz2rDTI+A4JbPi31VD2ofpQ9WlhFvtn1fTywne874OsOPjTZBj4lV1g7TnmJvZMLTT4kQK49WW6LPcX3Pr3FPRw9qcb+PKE9kD3I1cG9g7AtPDSwTD1Zl1q7QLzCPWiJhr1xhMG9aDqzvVJL9T10h469MRfTveUgJr5IQKm8dTtLPRHFiT1jxuU8vVPJPcsMDr3AsyS+b7gqvZpNx72iyYW8/ZmbvcFiFT40YC+8ZR8CvhvcYrxGUXK74QHUPEgDlr3J3Xy8u8EcPnRMfT1TCiS+CW9ePdFFAL1bIVy9X6IePl/D4D2JRC6+7ZuwPcyRrjxtmaA9vK0jPWt2JD3tqhI+7n6vPVEoyb0Wg5G7VQjOPegt5T0m6TW9cazLPC2mgr30fgy+nhLEPRpV4bw0q6I94yOovSD1nz27UJ08zR/GvaimdjvWQYM93FD5PN0hQr1NekK9QJ/pvTtN6b0Tdmg9cZvKvTkEgL3o2ns9CHauPIFslr6O2pA9Lh+GPADFvDwaJUu97nDnvKGclT0mKXu8xReGPr5p9j3Oq0q90Pm+vftuxj1FxjY7nkrOvYbTlb0VPmY7cVLJvX3es70WhwY9ILuOvc+Qiby2BfE8fxe1vSnJqD2Jv4s6zr3AvAq5Dj0vzoa9H7hNvu+AQj6K6uY79KrYPFEGT71glKU9PRMKvETFY765pBA+djMUvAdPQr4OVFO9yBFdPUKa4TxHaaG9ucHCvVo3AL5kLFI8oQUHPSBYBD3bVDS9","sCmJPuTJgz6mWoE+w5WFvnnOur6AA6S+iMtLvs5ZMT5x0nU+QOaiPisJ/T3UBx0+TzckPRstBD0evl89NzRrvSM2OL7lBa09xl3lPXEFYD7qa5m9G8GYvROQMTwO6o093uJSvR7sgruk8KO9blArPuH0ND2zn349dlvxPHmn5D2JwM+7VNcQvmvV+z2hwsA+cZ8tPh1weTu9XEw+RdUqvTkywTwny00+0q1aPQpLDz6ek3u+xBlavithJD7EMG4+d8xmvU4IIL4yWOu+VU9KPtrplb6TR5e8k9K+PWQ2PT4M9++7AThYvgQimL7AWfQ94i+kvavbYz7ZVGA9PVcKvvkLDb4MqoS9qhfKPkztS71QAXG+G4Fbuz6k4r2I2Y++bmyrPuKzAT6GNTy+2ON3PbnZ4D2BsLk+ni+KPqPOHj4N/8494m3svZTHw74fNhg+r6OCvewuyz6FPBC+MFrdvZ4lMD1ttIK+PkE8vv6wHT7B8Sa+TSZcveL+Wb3g94g9n9sFvYWUJD6YbYm+GXGTPfRy7j4gjYS+QS9uvXe7jzuue1M9bkbIvZHC6bxEZRi8aGYPvseaGr0Wv289L32APj80db5ycaa+LJs1PsVz7D4fHrY737RNvbGanTy9jE0+ArkkvaSz4zzjlIs+vW6CvcNXrr2Ng6i9yjVtPV+cx73ztqY9Y5CfvS8aZzy5lkQ+TxRgvjmPdD54u+q9hQiDPceQcD4tjP498hzovi9Dqr4UCKI9pbCHPR0GUD4n2aw9zMhkPeMpgD6UI26+cIvwPWIbm70pvx+9knyTPmiICz5ud5A9LHnHvHEglj5rAvS9UtbQvdVVizzSB1e9VzrXPDMR/DxBdNS90GuJuz+UUz69Lg2/LH89PslerjxdLRA7JeNSvTwwuj4PYy6+SJ7xPTQheD1biE+82N3SvjjjOD32sZ49untMPD/gWj3cXEA+GFx0PWK+Dz2mdgi8qIlzvgmcx77u6QC/QQ5TPndRKD2gi0I+J7e0PanPQz7aW5y+","wNyJPm7qOT578Pg9csycPrL3OT6I+IO+xXpLPswqnT1yNOu7wIrJvZZRHz54660+lgdvOx/OH74CfYY+dYfnvOMIhr4c2xE/Q0gfP/KYkrwOqya+c7vyvTVwMr5lZ769qXGIPsYb9D2Dmg++TjjxPt+DuT3eo5E9oL1iPXm1JT2hIBw+YxSRPJh4DD6YThy+ohZCPhlyQzx5JSO9KH+CuhBjCj8wta89kM9ZvocRkr34nne+Ep7GPXstUr0km9E9GX7Ru96qjr4SwJe+8go2vnD2+z5cqNS95YaBvpBlAr6v+dw+VOK7PftTEj2yB4c+nGWMPZEzmzxcKFc+Q7e1vBKvpr1lcyM+0riLPWPMoTx7S1u9mkrPPFJcjD04aHs8iaKQPRK0jr2gwy29dhg7vlzdBT4bLCi9oh98vJp9qLx7cLe9hurbvSVSfLzfCJQ+9JmAvZC/gj1rih297jqPvewtwr0R9BS9sDigO0biEb17BAo+ipsvPpXliT1yIwy8WMaLPRzY0DzHcp881J3UvU4+mrsSSXA8AQroPTC58T0zMIE8W9iHvEvotj3MQMU7frUKvs10MD3bhmM+M2I1Ptj0BT09Yna8mFrbPEpdOr6ewre9Kb2XPOCFJbx+VPW8hfPUPAAaRz4vHTk9Sbumvft51r2Bi0Y9wze+vRwf+L0cm3+9JU9uvAElGL5wMJu91GiuvTjZVLsfRpQ91HvxvT/TBr1vvqq91zkDPS0FgD1fm209bh4PvkR4Tr4vfvA9+zNBvDYm6D0igAg+TS2VPQuwhD2Od+69HhoAvr1iTb3NrBk+vfKLPS3Esjw8Qye8Zm+DPX58pbxLDMO9tL6IPRkKij2jlSa+G1RWPZkcZjymYYC8KqjWu8Z+CT2xizg9Y0AQvBF/xj1UWim9zb4APgm2kz32cr49cCqUPbyMIr2fxkk9MvxzPc/d773thDQ+MPFQvJ8qkD10nYk8mt+fvdwe/jxSDAU+ysstPXfFGDzRIY+9ol2dPdxbrD13O8c9","IxK/PQxSFb3ZJp09LIicPcokJj5TdzE9UzY/vPBDCr0Rvjy+9zziPeJPnrz0Y/c9NzgjPjG/jb2Aqge+kck3PgFVm71rNhW9VjKnPQlqiTy9Brg8D78KPW+NBT506hu7225uvHrR4z1N6Oy8RA6ZPfqOq74e1Rw+UyVXvZ3mk72HSt29pngfvi0szL3kjtW8RFESPQ4GIb4W2ZO9HLZJPWNpbz2ZamQ9zbAyPuAC0r3lKXQ+00unvQh/zD0atzW9V8+XPXn8ZT3+5bs9JePKvF+FXT2YG/c9cmn1vFd2Ib44oNe9A2wQvv+/Rr3tiIE9rPs6vmtJpj2YHyQ8KhMiPjSgnb0A7lI9lHzZvdMWvT3z5Da8mlybvcVPuT07V+c9Jr8MvrWECr7/UGC9ZW2/PYRf6DvbUtq95B0AvhuVXr2vRpG8NUcLux4i8r09dOe8yKH0vX/7Db0cBmA9TTkZvtiIMD2c5Ig+Ny7LPIDBe7wCiLQ9ezSePcIccL3jvTk+rJzsPc67BT4jsAW+DZA1PdV0Hz7pYiQ9XoPSvLAT1r1Fyqm9xxaBvJCTQL00RZ89lzj7vBPFuTxMJvG9J0ycPAPqh72DaFs8cLxBvumBP70z9Tu+VhKfPX04Yj2GYgY9sLxcPTJCgL2mmno8QhysPA+QDr0ckT08zJsOPR1ECL3Ek8m9xfxxPQkvRj70rxG9LCoHPq3q7Ltf8zA+x8BaPl/pO74TzTC+cVozPSPkwz1V5oc9tw3ovcRfEj30DdG97WYXPTgAkz0Q+0Y+r0EBvZPmub2aKeg95BjpPcyjUL2FAMg9gbGrPYX+br6WGBA+7j4sPZsjyL3ERDO9TyQjO3fQb72Bgsw9qWJbvYBKAb6mVdk8BBRSPBb8Gz6W5cM66lsePsjy9r3h9ge9jWCZPY0XED0QoD+9u04BPguEi72ucqW9yS0kvdqdWT7Gvi+8WabOPG3rgT2pc/e91iePPSKuS7wlZ0G9LPIGPkZ3Rr1MpEi9HxaQPSsLCD7OmLc9","KQzcvdqB0j3CiyG+oJ9OPn51Aj77rls7hWHqPoH6yD028/y8oY8fvuiU2L3ENje9tAfFvV9xMjx4Hcy9q9KrvSVF6j0oo1y9VFHxvE4m/rwvGtm9wfrcvecP3z06ec89VJwMvZjuTz0djqu9aWPBPQP/ET4hBHy9uDx2vW+4nr2WSiQ+DZrYPV0qwDzMDog9V6eCvQpMar7+wHO9EbcOvVgvYL38eEu+4Ll/PGq+Sz4mWxg6nSYYPKCNdr0L5Gc+CcOFPv6s1LwT89c9ZFt/PHjk0T01Gb66yEQ+PhNzOrxn/Hk9i3g2vtCrJ74XnDw9F7axPTfIZT0V8oe9Mpt3vcb4tDziEPY9FkQvvm9IZ75AOL49tjzXO8AJTL2+pFy+m4WwvsHWn70uZ6W9vIUYPszRQz5ZEUO+9D+uvbchTb3TB/c9PScJvoU3zTt7Gv694LQPPMPhfL2ghvY9BFIfvodbi77ftLs9FYPEvVCq2jy4WPq8N6uGPI6E0D1B22Y9iEPdPAQdwj35/jY9xYuNPbS1Nr3d9/M93FMmvtt6Ez56ays93KZaPWChlz10YYW9JydSPtS70DypbRI+lJSdvb12jbyBDpo7w+VePRIZXj1XBJk8izSLPeK69D2V5CA9hN8rvv3/Db4pcc09l4TLvAwcTru8KY49z6fyu0lMLD0Ep2a+x+WKvoZr0r3BSaS9voWpvQO1JL0fa+K99R+IPbuzp7xteYS9T/XjvVed1LxoOLO93EOoPExZJ77zg/09t7gBvPXHIzwCHOQ918aMvdrcBD4XLrO9zHPgPfRGgLyleyy+6/D7vV3KD7313aa7/7cIvMWWgL3tXN87qDM8PtlGA744IBS+PWHOvQUrV769Bhc+4ZuZvYfOjjwrsZm6v/etvc+jLL3+5Rk+j/zqvIRWl721Dde80qSDO8GgQr2tj6g92vAwOozxkjy+AD6+Z35Ovb8PgL0viik+2h5dPmHY6z0hLZq9B+k5O9mEpb0J1AG+H8cNPI4mAr5hT409","3DNJvVfZwLxOqIQ9uNO0PT1Ear12sjE+JZ5Avg30Cb38Y689Q1eaPU72rjyF/CY9prWOvHxQbr6zpvk8Xq6CvWJBub0f7Yy9JDgdvWD7Xr7uev0941y4vd8Onb1D89O96YMnvnbVJT6HxPA96dsCvtOOYj6k11o8bHERPcyGBz2APt+8plbVPA2FuL1toyO+9VHtvFcm7T0WUxa+R3vgvchEwL0DJYW974E4PpvgtLuCjXe73866vVp5GD4OCqS9h98zvef407w0OIw+6J69vVDFJ71YUNk97o1xPUk+sr3G0MI9oyq6ve9iwD0bowA81FbeverNUT4V7N49LmnUuhYisjzyW7i8uxURvuWP6b3Kmoa943wcvaS6Ij6Qdoi9HIlKvjvMCD7zLqe86S4hvCaT+j37m689rGAZvkYFrbxaG609Jf04vUPxB742cCc9pkLUvIUc2L1z+O69T9+LPSAJHD5qc5E99sTUu3S8hb1YVcQ98Ia5va4IDD0NdM69MRU6vmHs1T00eb68p99Au/oorrw7xPa9SPTRvQhZrT2SbN88znv0PZfwv7wY5748eSGNvQgvpz2Icbg8eZpAPeucMD2+YXi9R5ooO5s4g71b7xG+81LmvYpkLb30iKy77V+sPBJz/T3PF189uw0/PeFSx7pYrUg+P6buPPlE571NfVO8wyAYvW3AWDuNf9Y9QyuFvfH8tz22Rg099YBivR3IFb5LOhm+ahi+PfCM6zukWRQ+4jQIPXw12Tuv1Em9w4KcO8mTuT0ioye9UTf1PEJ1xD3dMqm9HwHEvXUggz0Lkwy7Ece3vT5Wrzwzb1+84ihfvjEhoz0VrPe8Yu1XvdqFsrzNL4c9zCQnPu+vGD1B7C29WAGsvclmhT3PXgi8Py8nO12RVz0jZOC7YTbNPdUNDj5oJo09X/VdPqMtiTy68187hvrbvIPjB7ym5Pk92VyaPZBKZ7z9ej+9yLsDPRc7j71YaHS9IXjBvUi/0zy4hLc8HT7uPeSGgzxIt6W9","T3XsPJGGxT3SRR49ClilvdR+PDyt6BO4S1aDPJ6crbxveJY92YciPMhvAT1Gf/C9dXI1vc45Gr6VJD6+qCNZvatZ5rxndSa8Cyq+vZHgiLxOOhY+TyD8uwl5bb1ju1W7Ozu0vZIRBD6SqyC8WP/aO9/Gd72xtqo99QLGPCYxMj5dieC8u3AuPmM4g73F6ZI9CCi4vQqZNL2cqxS8XMmyPQfxwL3B2R2+eR6aPbWHob20F/898DhUvVqBHz0tDl69CGzePLyvQr1C6i+93gE1vsNS+71hfqE72fY3PYqyGj1raKG9z762u8+ukDujChY9dB/VvSa0AjxO7Tq9uqKnPSsZ0T3Ym5m95nFAvhv2H75Ncwm+fmfIvJfU170eiaW+cq/TPawebT1rnRO+n+eUPTypwzwuQg+8P4K6PRdRALsR4gE9athoPvIECz6SNSo+ZL8NPDkimz2kD849kzvlvdxkojzhjuG8pFK1vbGtJT11fKG9mFFDvQC2Ib2bY4G+Nbm9vfDcUL2o9li9UrElPGqW/j05L0e9qQC0PDAyZb3WS6+9RsikPeuWjT2z2kq9cN+CPQkRYL5v9Su9eEMlvUc0Uzy6MhY+P9rOvalCprxNKKo9OFhcPYSIlD2apPy9MTK/PJk4zz2+L0Y9uR5YPYHUKb1/lUq9bJdpvdszDD0W0vm8Bq30PZu6hL1O1+g9IqfGvMrh1D2HdIc+ZsmWvbeUuj1KAqi8ukWZPX5BYLyR3au8nOK4vUEAlryGqmm9/ymtPZgOdz0tySC+WO2NvdC6gr1yMjq8t/i6PWyhVDyEJyw+MId8vajxaD1S1RY9aBgWvnLT4zyelEA9mOpNPccabz3x7Ro9KyWZvQmgmD05ZoS9FcsNPSqTeT2ghsO9ggpTO2jPPTvpDZi9N8jfOqvgeLx0MAc+/8MNvVIBI77+QMy95wyiPKU91TxnyTA+CmHMPcjE8z2YzYK9MekEPAWXwzwnxtA9BUk/vA4Vgbyohsq9MKgMvSxjOr1Xr0W9","qBfJvK8b3Dwge5Q84zeHPP/ayzq/a3U9LiV/vSEKPr2Jy4Q8BLsoPbIuBr1k9S8958YmPrnuSr0grcW8hlc/OshaNr3tsLS8OPggPeD80Lzbr7I8x4GiPfM+7z33ZlY8koSbPQslCz4oKh88KIcxPD4c7r2n1f89exJRPaa+oD0ZdXG9NasBvPMEVL1FIA8+dPJvvYeqFz2sZYG9U4RRvQ6Fsj2JPrM9cvKfvWMSOTy9JdM9lMU0vp+vSj6h6xy8+RyyPROLqbyeY5I8HORBvfn/oTs5jNc95wzWvedxkz1WdJM7EOBLPdk8/b2aEYY7bfF5PcAlgbx5U9I9KDvUvS/gub3KCWC9RImRvXOFizztJf28ReOVPTkVgj1F6Ws9JnSIvbTXXbv/cMG8uGQwPcx+i7yF3KQ8pWlVvVXTP72goxU8BTySvVElxb0DTpg7mnCfu9zm573wzco9qKSWu6dP4Dy/lc27/0tavDhWNj2KFr69J9mePY6dWT0z7QW9LklqPZnlNj6zeDK8Er4lvUoZT73iE6Y88lM0vPr3ID1/8KC9G+FDvXn2/jxsbwi8/aw4u8DgCTs2G0w9fys0Oy+Ppr2oXi8+k4UnPJJwkz1rEJK91JxCvdXG1DxmxLI9qJxqvCFuzTyA99m9hqmVPfuByDw2qbk7SZW4PLvCYL0jYa09r013vk2Gnb2GZU8+T5LYPAU/o73GrAc97zAkvg9G+L0FS5u+xs3rPeIzRr76itA8DmeKPVkp/z31ae89xCIevqwSU75GG9i97vUBPw+Uqj1cG46+MhXBvcZgdj4Gka09S4QvPnswkL35Zee9FoZ6veu4Tj08UO+9ziUtPegLhL2hAyM+ImF+vU0XNL5RMIS+diYqPRBoCr1yKZw+oSc2vopUaT0EkMy9MjsQvZ9zDr5Dfik9B6+aPei32r0REUo9f1hwvQS1nL4a7oM+Yv4QPv1/Kr569m+9ieh1OwmgkL0VM+Y7VP1gPWFfCr7Fg1I9BkQOvaHShT5QKhQ+","cLwEvLjiXLzj3g49NP0hvcIHnjyLdJI85ujrvkxQOT7neh0+343pu5JZZz0QlvK70pPQvgDxYr19sjU9zXCQvS/ypz0sp2C+CzFdPvJldL3h6L099VLwvdyMPb7D/VQ9N7MFvkYFK7xhOlQ8+OBxOx6HGL74LpS9p1avPcXorj3RlDO+vnF8PXRMhz6Ued09QdPfvX+n871b8o2+5OR5PsVtBj61dHa+BteSvh6ILj48vze+xgfavbGWG74ppyq+mDyJvWWu0D1wIdS8zI2IvvSnGD9/4zW+9+6/PmSKBb2KwRg+vNeivR4WxT0VMdW928tqPoz8Cr/vv5m7NueQPW5xWz3p8ag9t/uOusyrwzsDAay9jUyCvXT4pr7QSng8Bfv6vOiksbzVrWi9ol9yPuhUMr4VLQc9DtgAPX9Sl70nN3Y8va79vVUaubwGqLG8m7NpPICGr7wE+1q+3TiiPSYUEb6N5SS9IUl2PfGwiL4QwAQ+zpN6vmG7Zr0HEjc+C3U5Ppluhj1RLB+9gCAyvqjptz2RINW82+++Pb4Y6r1KHMS8qxkfvQdD3b1+EmI+fWj4vvCoVL1Od4W+u3ACvd3hlD0/49A9pHUSvnS45j31sDE9aZqRPRWbtL1VqTO+i9THPWkU1L3We1A+mpT9upbYHr6DVRw9vwb2vfXRcz5470E9iTZfPiJsD74Ugxy+RAv9PfxHk70XhwG+OTLuPJDGR7vMtom8Bp2Xvd37GL470H+7ZuvIvS2egb5hDiI997RVPTHbLD4ab3G9OEd1Pgk0Ij1sd4W9avthPhWqMTxkOIC9S46jPfCsJzyM/s6925wDPhkVBD5G1g6+ypQxv8uJ4zxBIxS8EKzPPEnGbb4lvVK9MczNPfh1wz2H4C69KDfSPTiXPz1BL/U9qyxKvmg2Kr69LEa+P19lPRsw0LvPqE8982XCPKibgzqkG4O8WTLlvQNDm77lBee9rHbjvT7eEb4U6/S89rNpPa5HfL2t38+8iD4NvkQ0SrxMMAC9","nO+HvVomoLxngbo9JEMqvuVmiD5ZrCu9yIqTvuGLYz44d4+9VLaiPWgf6z1dnoa9LWugvdRRp76cq4q+aOeTvcwTDj2LJZi8w2rXvQtEEz4usYg9lbYuu5UAnjv15q6+I23pvS/aOr6t/0+9ZaWbvTUwd771zAI+7IWGO3PQeD1+auc81R+qOhyW3L3rZuG84RnyvbMGuT3jyIK9sN/uvVOfvjxSqUm+dXtavJ5/mL4ggSa+KFXwPf5n1z1G9sM9X1gnPQNvE74zZwE+DJNyvnHO7DzYsrw76kU6viyVdD38Sts9ovwCvlCAmj2+ogY+Y+hvPRAU1D10o1++6iWNviAF0zz44iS+FguNPQJlor30hXg9T6+bvfitY76WCqS9W6yju7ID1b30DBG+B0CFPZ2j9z1l8IE9+c5cPUF0VbmadDi+s6w2PYhWuz4w/4i9/zGKvaU2uj2Rxf48naGFvVbqxD3B8Mu9wOfjvPvFGz6Otgm+8PgbPk/et71OLXw9HbKHvXwf0T0ZBXu98pYFPVdgMD5rKYG9xSpPvmWdN74J1SS+ZbmFvfppjL2wGFU+zFy6vPnJurwBSZK+o37oPbV24z0T/qm9IEtbuiIKOL6VGAK/cswivly+Wb4mI0G9zJ0oPHIZM70WtGM7DNL1O5xWfr6L5wQ8yI4kvBViHb6blNy8FZFaPSHJL72Wtjs+2Y/TvYulqj2sF6S9kD0SvQY7ZD6+CSW9gm9CvlU/NL52V5M+3sj5uyCJfD1hIgk+yU+TvW7VP70gvgE+uBJ7vTPIEL7XdHa9uzU1PYvfSDzoB/O8IHAcPvFvpT0bfy++0y4gvhmSG75zNwu+szMqO8oykL2PT3m8Wn2lvb/xNL4IK3y9W1SWvvNAor7a2Vy8kD2lPIZrPj0U4ce9uF+RPhFPWr2xRUm+WzGEPd8Nj73FUeC9CfbGO4/4BL4LdhO9mIWmPRusFT7Oug4+lyKpvQipDr70Pqm+djvtvRR+hD24WPG9joZpvg4cgD307+q9","aVavvQGSU76NgwY+aYDlvFzh6T2LkIi580mvPH9ucjyIF6y91lYkPTdJgbyQxj0+/B+FvQ+bYbwlSom+LpE8vMaotbqtKB89z8ZcvUe/qr2H8zG9hYuBvISIVL4TxTC7pjQLvi5nf77hOFy9ESRbPUwjsD37U7M94xQbvu4AiT4Cpoa+KrRWvFcn+71KI909nNauvekuoDwYlLG81hDFvOYYHD6ldcS9yJM6vVcggz6yENm9WiPUPKwUgL3xFJI9ee3mPUXop7wn+IU9J4SOvXoo0b1YAg8+07HcvcDS9z3Jo909F3fdPCc5hD2jajQ9arYCvh3Q4L1wnL49xAMCvls98zyJOj0962D7vK/XEL3VPfI91qU6vb+NCz7hq4a9dX4ZPbdOOT5CqsA9DtGMPtIeGb1XIOU9N6KZPah7m7xsarM9FAmlPEIt3z2s0vC7N/mSu2ThOLxjUyy8S5IjveyDTj1reqQ9wdALPcPtdD4t1AO+vtPOPN02KL2PBpy9y+cZPrUvHD1EMw69evD9vKxcqD19Roq8Ti4Zvovobjy0rja8BnwOPp8FGzxJq+29NPenPUolVjyOs5e9KJu7PMn0Fr1cliq+k+gdvkaimT0oMJC96sBMvirhpr1gsdI9TBt9vcNjEr0I0zs+pg/VPX8wybwcYOy95jMjPqw95L3QemG7mbfDPOPRTD5xJ209p9WWPO97Jj1tBCW9W1/zvfVcnz1uuSq9zNU9Poxgdj3eJR69tOoBvK/z/D0N5ro9c/PCvcZERz3sQRa+ltIFPGdCHLzPprs92xw2vTxI3T1QkOm9/HNFPsPOa73ZA6E9MMzRPTsZPD1+O7M8RmgPveW6hTuIXR890C1xvoPAwjzM0W8+8mLZvR4yCj5RvZA9CywXPszPpD32MUa8SyeCvhdx87y5Hbi7gGJUPbxFaj5uUTq8teU+PEpZ3T0npI296Jx8PmDGQj5DeG+97kDXPH71GD2hNAA+iuiiPVAfTT0ckQg9Wex1vROSGD397jQ9","afntvOa0aLvSpLS7XdxmPOt1gDw+2yC97Bg7vVE0xj1qaXA8O5n5O2AKMj6Cj9y8V82BPoBMJj2QWm4+8cINPS22LD6bevU8eqGMPSeiF7sdMgg9cHKPPHuKzr2o6pc9xe77vSfwfD2tIKy9ZvbjvL2OdD4UbGQ9JfJuvPZGOT7PBTE9eIjHPRmrlDx8OHa9n+iNPTcZm73oklU+sMaAvQSyhb2As2I+MJuPPQHiXrrxnzs716maOtHlID4kjD88J8uMPARlNz7ULyi728sGvs6LAz123wg97BoYvGfCpT2xDqE9SjJevbSCtjtE2jm+8upVPLshzD1Ry6U9eDvsvNvUjTxE1cG8iQ5WPilyXj1hnTo9YGWrvOjMJT3h04k9gzSsvagmTj3NXwQ+xabpPQxNcr0hXCo+z1+VPoDpP718gHu9aRYtPgksjD7ZMIA9WbzOPZjLg701Wh0+j1scvtv+sD3FRhm9Tx3yPZ1aFT0tA+I9hyErPrZzST0ZdCc+GsS0PYtd5Tvf4fY9yKmlPfWKtT1A7xm9p5ZrPdFJRr1IdnI+rCpovcgKwj31fEQ+mL0cPjx5Jr1gK049L5MlPF5czj1syjg9JmsPPUdJoj0m8cm8QA2bPI/Hkj0KVuo93YApPlTNL7x9ye29lso1OvamXz57OgK81rC7PIU9kL0eqHY8tB2Cuyl18Lx4+dM9ovD1PTOZ0TzMlBS+o12ivEgzaD6tQ5C8b0aCva6Fdr04Udq99ZHsvGLq4D1pTzA7y8XAvJAOW74/y3g9gjxYPnNyu72Y8f697Zb+vW47Ej3rdwa+qqO6vI+0jL2mwjg9VdnYvXQKBz1fBnG9mOO7vS94Vj3g/aE8SdtSPYoneLtLHyU+xH7MPBKJ/73hLFS+T/1hva3aTjwej0O9z28HvvM3nzwBdhg+abpLvpgf/T2AbWU7WdwZvti0Ij7SODG+lbMdvqgf5T0sO8I8kz+0vRdtQz3sVSg8jMXKvBbNBL1dt9A9ZOjtPOBH57xhF8k9","TiGhPTZ7nD32yre9JAYqvv5Gtj07fhq8dCMevk4/CD6zrsq8u68rvSoePz2+3pi9+eeEPR02oz2tZQa9uiPYvRwitr1Fc7Y9O0rYPSQbFTvRB+A8IuEevdO8lT3NlcM7j3W0vZFEzbuQsyQ9TJdWvdYztToi0Qc+p8FMvbB43DyDwoy9ZDY8vtxdCr491Ky8k/0iPhUXiT11ji+9ocdzPZ9ezjlDqKY7HPyIPAv7ST3Goui976pVvLF0jzwx9xe+pTZlPet8iTx57TA+ixG9PNe9mD0vXJu92DTEPWLioT1txje9GsOrPVu7KL4tMe88jUSyuwkjuT0oOb27j6T8PF7NGD2Vyt+8ZrXJvJBXAL2C1Vy8wC2XvLOBHb5wZx89kA8IPtCUyb2j0J89TXw9vg1hYb7mPne9mILau7w/573TGnA9/T5bvtfunb0i+DQ+6JqavQPJJ7y55Jg8wpIiveEuoD3L97I89NhpPSllgjzDmRc9M0IYvH5Kyj3osz29sBvyvfciebudluk7moQPPr41cz6NePa8HhHkPOqABL5cgZW9vEO2vIaEGb1Qnry+wHNdvt9NUL2p8ZG9aXoFPPLFGr7Y9Yw8ZA83PSq1Er3zKkE8bW4ePsqLkL3ehYS9GwxMPQCCuryt0kM84oZWPLt4FT6uwns9rlmMPeytg76izQw+2thKvYJNFj2iskc9M5yzPXzvcD0rAJQ9jyxkveQWxL0MaPI9qWLBvKqzU72DlLA92o0jviehGb4bgd68JVY0PlBEBj4zCEu7VedZvTzFET0ZsTy+XZXGPaNIN7zMwcW8WVtfPThxXL2Yvn080wvgvaHLSD3bm0m+r6M1vdSJu724+Dy+aDQlvJDttr05hgw+QIN/ve4c2T15XcI9CDDZvTDBIj0ryZU9FZMDvoJpzr3z9YG93pkQPjRVeLwLtao84SpXvfQ1hTxsPps9kS/0vd+rob3FBPQ5xuIyvWhwob3dBTO9M4/kPXFZ+jwb1EK9qn5uvTLRrz0f+H+8","hfuAPLn3Jz6uK4+9/g+SPXtDvL3kuRo9Iu+cPGrlIz442/W8ojiJvczcqT3edxa8YqqoPcM0NT7d8CY97w/6ONrojL3Xnq6811h9vXVkqj5tEKE9zycjvufmAz0k+Io+4jVGPjbwhj09Kdg90IcoPnAmlTutYX2+bOhZvavpkD14rR69P+cfvBM50b0g2CK+9jzqPWkfoT1iiyQ+UQYwvNmPJDxrEPi7KCYhPYXIAz4JOq69egWMvfu6vb1bveg8VCj+vfWbFL5ymYi993KwvWANv708VQU+9DGcvLYDqb1fWA69yUTYvdiwnr3CMaI90c/OPZCfNT0/37w9pqDOOpBLxr3GJM09DpYRPg2v+zzfZh2+rMHCPewfPT29Rxk9fG+DuixxkD2JxyM+PbvjPX1emL2T5Jq9PuBevVcoUj2AWzG8AJCvvYwuTD67ZJk9aIK/vFzBtr3VISY+UCyVu6Py4D3UaVU+a952PRdbfT090Iy9SeVwPL5wC711msS9dGDxPetQaL12f589Nw7cPTawtb16xYo9mnuhOwQW6jvkfp89st4ivbBd4r1z/Ts+k5VDvXksUr1XjDu9+PyoPZ/1ED1AASG9/FnRPRtrCj5X4Pk9zCwKPDqc/LwkxSU90grDvKY3pj0YnIy8SX5IPabOdzy2hz46fg4tPQovdz15kfU7xQm8vc4tpTx6R1s+AfWpPWbCKr4zPGc9VGksPYuj6z1Ocgs9Ica5PcC11L3z9/E9dsG3PUrfi719LVW+zAAzPnRDhbtAB9E9zgZRvOw7sD1J4xU8h07IuysabD1AyDA+Z6mUPJO7SD6SU4+9oO2JPJI3ID2kePa7yX05vkGNk72JngM+BR2tPMWn/D2ouBs+cgQUPgR26z3inDO9Eko9vedYj7zYRy+93K88Ps8w2TuKlyI9uVP1PA8dkL1h+s89WT+oPQw5Lj3EN6S831cXvWOZoz4TSio+3TpjPnWqZr3mYyW+VYG3vWHU4j1CWYq9PCK0Pc0UCT2KIwM9","SMs/PRRTbLwN5d888fQSvJnrUD3na4c+CBwdvV/UDL4sYO+87NjQPRFqrrwd79Y9FGq7PbMHND7jHZ48VfXwu9n/DL4i1Y89AlNQPsx54z26YBw+9z5mPR/QnD1tuSc+cNwsPs2L071gmN29wz65ve2cDj5dWBO9SxYNPvoC0L3lnco9bl3+vFNYjrw5E9o8Cub1PXIcXb0fEfg8vSOIPdJcNb1KSRq7Y33iPWnHcT7PB7m8vemFPavgmj1fbVI97UcjPQkd+7ooNSG+d2v/vcz12j23fQC+lkMavQ+hNz5gZni9B0S2PMPrzDw+xgI9UGS8PO5NozpHtVS9ya/0PP/7Ur2zELs9nTZVvmYxaz7mDkE+PORePTUijb2OUOA81YvZPScFiL4dm1S9oW+Rvf6xXT6AKVg8AaWKvfAp77tmyou9lxaIPQIh3rwqlBC+uqOhvP6AgL1tcks9Nk+mvXVXzrr7vny91IFRvnvKwz3bcbe9oNCNviIyLD1WIsk7EeShvSdQJbzRTze9f/siPiIXSrzJnv+7rXWDPQ/DYT2YOyq+kNG7PC0AEjzTV1C+vfK6PbKdwb2BuMq9NecOvmFG/708EVU9d3BhPlR8UTzxov49Wi7IvaeKBr7z9Nu9YPOcPc/P1L00lM89qJ4yvmtyCz1HAOs9Noo5vgam9D3n0II9vqvtPY3PQr0WFtS8zgRtPUT3PT3U/DO+3AWIPs866r3ZFGs+QT7+vOWCLT2qZiC97nBrvjUFsT1DTZe9//QWvV3oc71iT1U+hy4YPb7t1D14ebi9njv3vQt61D1lG3O9lLusvQZ8GL0ODUO9urvqvTQy5zscc4g8P/ngvHJGKL0Dx2q9YYI+Pl8pwj1+azS9cm2aPeYh0DwazOY9pdLCPckTwz052Jy9StEFPmvWhr0l8g48sHFkPROTkzzftiK+6LKkPHbilb2QEcC9V/aFPdRf6rzugOo9km7KPZw8Jj2zZo+9wLrVPUonFD3gdhY+MlOtvA5zw73GTj69","WYM6PVG6qj3isgs+L6AFvnAgD77dmwC9i+5nvZQZmz22P9s9ff6CvEZoCL71UwI9A4osPZpU9j2kzt09n6auPStScL10m049glUAvT7fBL43ctM9y39vPMelIz5nph0+297vPRqDE7zkKvm9r6yRvR03RT3H0BM++4q1PR6yRz4y75U90YJRuyzkeD1fWGw+kJZZPm/aLb7csy29jBo/PbjIPD2rFhy+a5QjvpoUNT6kKLS8cN24vN0w/L3edTa9H5bPPAYzFT6KxZW9JeIYPAoirr3YyFi+srXyOz7dfr0GU829M8cOPmATFL7LTwW+oS0dPYTZnb078Ys97fVHvuo9+rwL2Ra+Z7M0PV4GEz2oDSS+jzidvfI4BT2w2JY9kIaCPcUxwT0VF8S8RctgPO5ChLx1eZA9mhCePqPNMj26sX289OMLvrrmAL7tWms9tivVvdSn+L1yoBq+Nz3fPX64/T2juNO9BWFSvfOIczy+Tqg9TCo3voCPIT1vUqC+8oSaPdpf3L1g3FK98YmtvUYVAb4S6Q+9VohCPZrngLxZzli7jW06vtuSqryJfxm9KroIvm6EHjx0rB0+0puyvMULtD3PxCE9lh0+PMzyOD154cO9jpV/vfB93jwdfYU4J3SPPba3Aj3la4k9KmvdvMpcyrwEOIU9EtwpPp5rAj1haxa+U+KGunoSl73rFwa+eIuGvkXpND2SE6E8BGNAPirU2L0GVso73pGQPW9iS76Pd6y9xxQlPuwjoL11ohQ9TkSivSwDl7u5hFm8pzoHPT+Pmr2t2vm8Gj7lPPuipD1Vqb092Es+PSHbcb04JB28vyRdvTVFQL3kvhg8QcTUveJ9j73ag5e9U3tTvDsKE77g5/A9rVarvBb6UD2TKQu+NeGSvQrB3LxkhB++Wgj4PRPxwb1wqNe9ZggpPWJVrL2hbX49lMpzPerKrr3UfuW9WLqBu80BQ7zpdEY9xj0cvpEFST0Lxi29gMUmvYSYhb1ThtG8cAMQPfNJTr3rbcg9","kcwFvUWpED2+nhU8IrvgvejTOL0Xup28de9UvlEKtD0UGcW9EuWGvZLcGTxMWJS8PRkPPi0NCz7YdRC+BTVfvFW1w71C5DS+t8QSPVWVXT3yJC89qkM0va2K/L3g7iI8HE4SPv1TLz6d7sO8c/SyvWHqcr117048Py11PbL/ODoJvWc9QG02Pa5/Cj2pPL69KMhXvQgraT2XBpM9RC5APPHJn73XAVI9avmiPTN4xr2YNDM98GOXvIhMw73zSfY9P8Wqu3WQwru+Sl2+BZybviqQTD2cIfw9dbMFPaigNb3sD128+qSnPXss2TxnKPM90BchvsPrM7zN3/O9Xxjmu1zonL33FtK9N0gVvaIDzr1g/TO94SC7vO3vpz3JYhg77UPPPdT4B7w2f7e9LUScPBJa6T6rKxs9uD4uvYLlPz2pdv08Jl1xPe4XGLy2PD0+Suq9vW8397yiTEY+sX0PvRNH5L2p7wu9x8eePdF6Sb1H9YM+dHQDvlrTML3ERWQ95QmUPdzHGD1obPO9yPeAvn593L04OMQ9H9MHvhybUzvAG4Q8KuPzPfSzIj0cX/g9Z9ktPqdB771q6PM8Vs5mvTnACD65/0y8j/rTvOyenDyMVJe9GpvcvBRRrTxa/W89kmGFvXnhHj2/cV+9tyTzPLURF76whag8sbgJPpyV4bzEcLw9AjI+vvNUhj23U1S9BsNZPRF/mr3P/mo7QAohPbsrz7vHodC8m55SvhAUBj4QbIA9D808PsA70Dtqw4Q8N01vPc7QPb1vuHg9aFBvvedZJ720daA9grX6vZ3hkb3eCLg9TQNKPibgZ7zN51q8u1gWPlj7YL0bFQ28ZH9WPZO/PD7t14K95GD+PdQjRb0E/NK8k6wwvYFC5TvQPpQ9U3mKu6zC2jtVEWO9RrsDPp989rywf9m98YHhPObmSz0+HHE9T5SCvRDoEj2s6KM9TOu3vSVHFz56co89eFSCvevFL73K9YA79YAhPge19z2FAhY9r597vEyW4b2p47E7","ypFFPdL2yzyOS827BREmPWL5e73J0fI91wIlvecmhD2xbdi9cLnMvbpoIz3sO6q95sjxvN8wrDz469q9ZzMXvCWnJL1wsQu9Di3jPPXNKb7+rbK9yAhmvR+8XD0mVYq957FsPVd21bymHRg9ZraMPBd3FT5X1wa+FjynPHPWiD6jxoc9jWTxu7UHKj45ZJy6hGjRPQgpDryQ5Wc8AwQOvpPwmDyLFp+9IZ4NPflHoj0UNh68yqf6PFZdMT6IKTe9XpB/vUfyJb006nE9qGSvPTKtYj7nnq49eQk3PBEOOD15lQI9dwdDPej2QT3ebD69GgKSPDWxWjy9Ygi9qilrvMGKMT3/tNy8E1zUva4v4T0/tNU99TvuvViEpL1yBM68jxMnPDa/FD0Bnke9kldjPaYdVDxzG/68DFZKO75QCb4fLB0+poGmPVtrMz39Jyy9yUzOvKKLCLw9sSY9QJyOvGxmzz1cw2+9fRJ1vSu2Iz09L4E9c7Uwvf8slL2rcq+8HT+8PFyxID30Thc99OskPaUQCDyGx9C8D1aFPAlV1zrlZ7i8uReRu3Kk1D2fEoK+zaLaPP+lDj0s6TU8I1SQvMyepz3GG1a97eBXvCZoh70we2c9LUYSvV5tHz4ZbTM8j0N7vLl0l71ENzC+cPFFPXfBwz25YUc9ZkW4vCz38z1mm/89+y46vaAp1zy9Ejo8ckY2PUJnVz3gvu69v5mBvGs9NL1OrK09D8c7PjZmz7u9FW++l7YxvTlCHj2xXIo9FG/uvGDHO724Te29+AsHPMGjWDuCJXq9bNC2PWhOz7xn0Ly8cpyxPaf/l71MXgw9lGXOvNZWhj11jeu92NiKPdmrFb23Y1i+oIQqvQT1ND50Tyq9x7IovdmygL3AYoA9MATZvfERsjwJfPW7hrBlvct3O77/snq8ON6IPjvfh71WDhm+eWMjvj1L1jwtLCS9y5ekPFS2+jvkX7M9lXVAPV6kJ75qhdI8B9oPvYevKjwAUIM8BnN3vZAagD1SjX+9","F7UqvuUbg701/6C9+rmNvMvHiL0i2Wc+73rpvVy7zj02ANU9SAatvcvJjjx2uiC+RlBwPWXd1bxxHaW8sb10vC6wt727y0E9Rlhjvihbpb0+hy69CeA+PqrU3TyJrvS8mPm3OxmBsz1Jr0i8vCsCvnPpPL2pZmm9ypFdPQ4NGz27rNo9tn5Kvdo3qb0jkCc9E5Q/voPYFzwk+Fq96ofiO6Hg+Ts2Wrk98qQAvaPjrDsT/xS9tDKpPIrcJr7F7RU+EyOcuz31+LwABLU9gZAtvbhcSz11tD09tdWhPUfp9L2H6Yw7bmeaPK6ilTyxnRi9TK2zPJQljr22q0u90LPQvPJXgz3G/nS9X1yovo6lYD7pYBc+3qOUvYrGBz0tow4+jzCUvTcOED2mYUi++QiFvP0flb0kmzy+USfHPQjYkj3Rki09gLKhPOe6zTxIP1y9fgAzPc9LHb2n9Qm9f199vFvXBbxif4u9G79KPRTzFLxbHDQ6PxCBPdwVOrnur227qoSLPPi9Eb2r7vi9BtXQO/e6jb0aiqU80bqEvWWggTzwYx+9vaZ1vDGa3zxT2sW8bIthPW09l736lCo+RZqcPHj5UT0ItJu9vh3yvYWnvb3O6Cu9t9VMPJVGDj0g3TI8ILFMveZGAr7fuIe9cvJbPkC/kD3zH5g97tEAPtIwhjsdKpK8Ok97ve+qb7280jU90Ejxu8hKU71rMYQ+derBPOwcGz5VAo+9ZqnZPXdu9zpeudg9lFtqvQLsyTw5Al68eoHtPYq8dD0WavQ8gEDDPbK9Ib1SNaA8gVVTOmGXgL0okSc9ZjEAvmeTJLwrRDK9UyOlPADxvD2VZJI73NvoOlr7tb241X09FdB4vXdOpzuYxk08EtgvPk7gdT283yc++pOuvRn/gz3zZDc9IiUKvM1Gzj31p9U8BS9KvU+0D77lTes9hpTJuhv/Vr2DB788c3JLvX5MAD6BsIO9i2ZlveHr3bw+7Lk75jdAvdBsury6PxO9MgwUPks86b0+cBS+","RBunPPb5pDzZfBA+VXhMPVeV8L3uXP+8dIeKvOs4wb2yrsi8zRs7PmvnqL2kZBK9aEDCvOcuvr2f7uS7HFAmOKmRr73yBqE8HeGOPXYtrT3zl8S8MhfAvFJs8bxA0Io834MdPepvhj1iHJQ9oXFePd4tQL2hFGc7SSYbPZv10r3TSLE919kjvTpIM73/JuE9aXyXOwV6Bb4FoKK9oecDvmjyJT3E0jA+shtuOs+b/z3I1wW9PbhqPWidgToLlzO9FD7tvRFiK72imqA8hlRgPdFh4T0UFVO9cHMkPbB+I779O908+aCBPfqDBb6nGC89N4bRPLd6WL2CszU9Y+4HPpXwwz2bcBA+RyPMvWz0IjzGPcQ8H+0+vuBfkjxVLwG+ZOtWPZArxz0SHqa9HyLUvRZtS72+byu7spBuvXCIn72sqL297aBYPMe11b0VL568K/+GvUKWvzvkmq09Ma0svsqGOL1W/By+0ZULvrjhwrzERIg9LFWBPY1Ej7xwY6A9lSgpPSCkVT6s4OE86oiBvbYNC76GDy69V+a3PRppsD219h29ldGbPMOatL1Lds+99PHNPUhvlr02XeS8sgIZPaTfkrvuhEM9lGHtPYMuub1EgsW9c5/uPMBcCr5tc88759kvPTuAPTzZ0rm9yjEvPeQcWb2to5G8kTSDvb5gfbuykaw860YovWBhSD2hdpw9jUbIvcUiGb0KXoM9LshEPlFNST13iRK+ntnKvMscOz6oS8E9tmnQPTJWZrz4V+I8ZT9APlz7Kj6rpye9xhNRvvgERT5fHie8NDYfPUt9qDstXxQ+BYGbPaeGID2Ppbo9PLLEPVvP1r3k4Xi8b6acvDKmszxcSqO8R7YZPgBblj2Xoec8vMM/PVEisD29QYI9R07iPDLlbT19ML68mhwGvu+EwL3B0CA+95eYPDTE5LzsHSm9HqL7Pb0xmj5reSa+T5eAPfZnR75WIX68z0YAPoupG76BQCS9koq0PYawyLyGLiC9+1cDPn+VZz3caQG+","rh5lvbFdJT1LuFA+qCVlvAX0trxieHw+F1glvrnIGj4RF/K7w6aovdq+lD4qzIS8JzxrPsY6lj1+hnY8aja7PaWhPb0W/uE9OCLmPa0iTD1nc1y9+agQPuEmMz07ViK9cB7UvSkTZzxERy89LriKvQoW8T3sIsw9qXicPQ3bgT0z33c95VUaPrwPez0o75g70DdJPtxcqj3zyME95cqMPZEiZjwoQje+XJeIPeCMgD3tX1u9FkGWPUhsET6fe3c+YjvTvdIVz7xvHPy8KQ76PeYsIz6u0xw+sjByPXam3r1xOLW9m5MlPKbKUz5cnd09lsIOPq9uvj27VaK8eiXlPCGGMTqyYlM8VfMNvV85j7zFK1K9rtMnPRIaRjwoabi8YbLZvfW0pT1BYQW9qwVAvdpWo7wBY2u9F+sUPu2SUj4Qy5s9UKVMPuJMyrzGtku8OeAdvC8PPD7muKc8nHIsvg+KJT4642Y+IhrhPBbrQ708dbW8DvvkPfDfEb2wmZ+9EbivPEsBjj4w7jq+e6DQPe1MNT6yvb28MAf2PYSjZbz5Ps+8WCqsPILzr72TRZK8IjtxvcEq+boxKFK9RDCXPPPOPr6gmxg8rsoSPsC0+70LALm9pQkCPe3Cyj3YkAI+mIbLvQ0HgT6Po948pzsOvUkxKL11ACu9BlbpPWP9jT0Byuc8KnKKvEQvOr4t8Ek9KUp1Pa5a3r0JrpK9gicPvpMavD0/Wbc9pvg2vZMnxj3P6aq7zmlQPeppJz7n9lk9aJ1svQZSML248wk+xgRNPTAEMTvM5Qa9tl+XPGOVKT75vuk9/LtaPsFcCr7d/x0+rHP7vXrI+r3MeBE+1dKBt7NcIz6GK+C9T0DEPJt1Uj1201Y+1P+APeoLCzyXuBs9svkcPouxqb27gzo+XNHTPj8bAbzVqrk7NSEWPg2nJTo9eT+96zm/vSjZgT6jWAi+TATuPXc2Yz6ezME73jtYPgaT/L1uVcQ8KucYvYSdwD166Ia9LUERPg02zjxyDcE9","fzK8PjJvwz2Xnoe9Rd2KO33bA7zQG4Q83bg0PSlfar5VKao+/3SNvaynlD6NEk8+piX8vAtN5j0sfSu+OdE9vYWJLT7J9FG+LfSiPbxUmj7UX6y9cKrbPfpVR70v9ey81HySPDqghT0PpxC9lymWvTx+xzwolAS9qKLTvUDuRD0sOqC9G65HPBHXSb3RQ629a3WXvQ3wWL2lhBG+tzfAvSFHUr6IEga+H9ujvaYTfL6Q/S8+KDWUPQttrb3D1+e9zWrBPdzNSz3HLT6+CBNmPpqs1L2gJ8+9GDMPvUg0Pj5O0X893i6hvJBVvr2dUlu97U6MPZz1Hr4zroI+xR2nPWenHD5H3YI9J30qPnk96T1z3Ki9XRMHvsk04rteAHM9AbarPikWDb4go5i7hQkTvVM4KL2Qs3s+PrFgPn5nQb0ej+A8yJCIPRed8jzVXca9iTy8PD1/az12MM689hbtveVtS7wam+w90mNdvTK4uTu2l5i9QHUPPSAeQbxXLRM+b+jUvdX3dz63Yg6+SjoNvkmDET7vuEI+jNhPvR8lGLz3VaK8oTsZvGAxI73+oUE+pNqkPItUzzvj4BK8iUKMPs1WFz7+eee8erEcPpC+Hj5qKoc7jtTxPS2PqT5x3wG+JBMlvdw2iz2f1Z494ZjFvczBUz0ocie+aDXePS/imb1HNFe9rSHJvXLnmL3d/g8++h3DvQysaD45LXG8BuOxPGA1jT5sj+A9xaQDvv3P8D1VkHo+OpK6vY/AJj532Ia9/LIlvS+udb1Dc8k9ab6cuQ+e2z0//mA84LyJPV2Bnb2YrK89cx56vvy74L36ph4+6nMtvawVKb6tcYm9R9levZQp6z15uMO9048yvfdEHj1s8so9aRp3vhtqLT2IMjO+8w2ZvsEXVj7X2pY9FttIvsF2KL0GJbu8f7qgvuPsYDwvce+8/zQsPfPUxb0m9Le8JVRPvEtrtzyxEj87c0eYvJ8ezLz22Ck+ZTkkPjWwJD7jX/i8qLUUvssUab2hBTi9","dBdbPkiCO77vskk+7YJiPURT+bwwF4W+WbxEPaOU7T0ew0A9vZQGPTgPOj5SV00+VxEcvnKv17wRm1S+We1RPqMg1b3MU54+qb1DPu9OCT5JKoI9V8MRPk2Hr73mWSE9EobuPcK84bxPn2W9N1BvPpqgvb23OHI9H7rDvW6Sqj3Bfik9Y+03vhGGA74Iq+s9318KvdAXGD1wYaK9GUqGu+9LCbwP8Ts+r12nPZSknb45Zv+86aI1PBoUVT38mLa8geY5PDvcnr1loA0+t+kVuhroCD3Vb8Q9N8q5vF930TxQaY88fXpGPWelEz76yHa9S4qLPvycGL3X4zm+lAp0vQ5eFL4QtSy+YiEkvsDQ9T2z6GA9uZqLvhcNGj5dE+a99gRlPFu4JT61AaG8eJXCPOuv6j1GeG89uI6+PMvHHj336A8+0um0PdPvjr0lpj4+k9oOvcLzb742UDy+RraGvTpZ7T3hqc486bWkPIndG76UDQ4+4nK6PbWsHLy5py6++++6PS88OTyDmAC9v5XQPW0pxj2wcZW8wZ3JPcZh1rwpT+E9Cu7IvfqJpTwq+/c9gn6JPWL9tD1j+Ci73dKrvMJodrxScWq7Lfq4PY3mwT3IKQm9ts9evWgJWL3j1Ui9FdKTPOHtFT51ySq9mPbrvvFPXTwRmNg8WU0HvvLbBz6hGg098aoDva4V9b3DHa89j2toPaU7br2NGRa/ytsTvbTOAz50igW9B5j+vdPPAj3shw6+F7RBPBpyUbwKA8A9spF8PWqLPj3yoMA9GNsJvn8X9z0y2kW9NKxWvXJn5TtaFbk9Or6dPJOPiz3VihI91detu6s+vLxBH+S942MtvcsFOb12Mtu95FMiPbWgGr0gkQ29rLfYvdV8XT0tnzq9WZaFvRE5mz15cxk8+SR1vRoFrL15GaA9jpTFvPnNO772BwI+xrVAPf/STr0AInc9CC2uPGkkSjxA0iA+NwQGPV5lsL0Mec69w4/pvcDDfjwZeE89aqOHvpEuPrzgWtS5","eYsRPnXtqT1nqh89nK8tvl9Ghr6t79m9jtUevazYM72kvhc9wL6EPWi3GL2CPoS7ioGYvpiYer0QUKG8bHLlvUM+W7tDA3S+iEYQPRTA7z2znYu9eN6YO6i43j3Lh3i9NZHPvaRSFrwroCq8OmYCvhv0FT5NbZ08qr3ivUChpr26f2k97D1WvFjK5D0VEBS+hVdYuwPEnD0VmFo9fepQPUzLWjtLzti85Df4Ox55NT5E7R6+JWGcPVc9ErzfzcG94xUJPvmB4D2kXN08PKBUPa9jRL3zYaC8yr6CvX/Zk71pGQe+2uxEvqAQkzwscxc97d7HvSXYdb5T6Bu9lT3YPuWLqTwG8gY+fZ7SvTOpGj02RhC+5Hn0vX/IQj3/Ixo+GdfYOKAdtT0OOgo+rBeDOgFLk70Ca7u9Xum6Pf9/7D2YTQ07noF4vYKMGj7rJ9s7B/9XvfCrXb1OeKs9do43vmBIIz4cyRo8nr0gPhWSs7yIHAa8KHzcPdhcubzRM3e7z5asPcymhz4nNa+9vI6uPcDyH74aTRQ+kuCDu0vpLD0sls29TZwFvr+v2r0Etwu+VwJQvpu/Db01vhu94BKqPWUU7zvLn1E9bBxtPgsqYrz9zO29e6NsvevFKz3RguO99DCGvePCjL2ZDQ89uqzmPbRLbrxK/ym8104NPd1VyT132Da9gxOEvWGF9D3U8oM+Sj3YPocrH7zR8o0+YStRvjAXZr4kEZA+NeAJvqprl70gZkU+9VHEvfJVHD099A0+y3C3vQPbXb0Bccw9GIUdvjz+vz1oOGW9M84HvgLcl72MLYS9/a+wPRG6670TVnG+kezGPbb8vL1vOYs9/uXDvUV+9jyCW+M9uLQBvc6vND4HlAK+Y1gmPRn3gD1wNDg9GOH6Pd+liD2OX3c9lh4YPskGorzw+RW+zH7HPTmfIj02hBE8iI4tPWa0Bb6CBwu+1Gh7Pmsp4rsbGmG9bfvgPIVczT30R909yp3Xu3txvr55ar+9j9vxO3ZbWb4Wcji+","HCvjvd8OFz6g9EU9YqdJvD/VSb2f0AO+yCuTvkabDr2trBa+lWAbPh11NT7RJFY9FgQHvlJVijvADQa+LcENvgk3FT1nMLA9841Evolkkb0INAQ+w0DbOhBsnby6li+92EGHPph0jb2GLFM9M91iPGeYu723TaC8O6n/vdyfKr3HJna942aIPZqChD3Lpkm9QfQ1vDz07zxau8m9N1NjPlLNrb2BJ9A+zHk0Po7V3L3S2RU84R4oPc6TGL11x3s8MBt6PdfCAb4Q/gM+PbYzvp/4FL60PFI9mJwJvi12LT70QP68/+aBvuKa6b2yXME9dFF3vN5GvL5UDx29K+ARvJ6D072JkTg8Ko56PlkayL2X1Le9nGYVPjIw271FqMM9HxA7PhYCEr2lips9dk+wvbmgpb4uGm49oTmSPDsc/72GlW0+jaFRvmeWhLzHhA898g98PQsvlD3lKeI+HhWhPSZzmj1CAGC8OtH5PbqKUL5HVpu9b+VLPjbVh71T2eW9kMVAvRVK47xAVcY9Sksdvqd+Ab7aS6u+r3qLvXxiXD538ju9EHtmvY2WZ7u4Mdw9e/i5vhgRAz2orJg9VyDHvSW2Yr3uZLS+T7CPPL6IErwLaYY9yVY5vmnsOruzSJU9r/dXvvckBr7miJ+9RU1yveh+zrzCvIW9pps0PneDjT6y14q7JX1BPp/eOr6bGq+7gYpDviNSKD2uLDk9e5uHvgAmpT5Ceg+95ABXPtX/mDsKeno9w7zDvXc5k75rYLY97IRCvWpahb3ok9Q9U2qpvCEOFDyzJEM+PvvvPTtQjTzdKL49I1DMuzY1Er4TRZ6+5J44Pim6kzt+8s699OyAvvmpB70oMlY9/uVJPnVlYLuCadK9qS+9vTQFv7zpEqO9lCEnPt6Jbj05RxE7aYVFvvACdb4YQqq9hkIjvlMdHD5aHq096JCYPbIXN72Ph5g8uyBTviak370+Aa69xCC3vdVikD6RYds90U+dvUC+O74jfaq92k3mvFJyWD5cQCq9","YxFlvspgNruhHEY9kNv0PUUbZrya5YS9BySXvUer3T3v4aI9vQvQPJiiwL3fyNM8eqSLusI63D3DEMI8i8NlPV2QTb4Exli8ctWZPIQCLD5SNp29BDwFPrcjGb2ID6U9Ov4xvdLKaLxsq/Y9yJ2uPV6pEb3jHEO9/y6DPDsqaz3iAJw8fR6oPU1hPr1spOM9q2SVPWfoqrx6zSk9HoybPUtuTT3EsGe9lRb2vZXxTrsmMy88YwAzve+/kr2eszY9bcBXvWFe6Lw6zuC9ZeIDvpKcgD1f6uo82JOwPZxu0b387Y69px2SPC7Cxz2t5AA+2P0hvf0Gj73uvou9O4OZO0qaCL0QDL48p47ZvMWyLL6soyQ8qbQEvSyXaj7arRS+sWY1O5IRrLuNFn49tS8UvUcPlDsPOBK+lVsMvanSab0vtJ69jRZHPMPKyT176CY+kw0qvUm4ED5p/cm8qod8vZ4sGj312Lm9llKwvWf6JT0uudS9Xa/jvHau1bxFCt08SgIRvt8srb089/29gILEvaQngz1ShCm+iWIsu9IZgLx5ada7nBL7PSIxlj0e2aw9PfY4vvtga71J8Po8tFKmPZbiwz0h8Wq9sUhVvvXddryMlum92ym3PaVLE75Trea985sDO9fP7T0++5y9GaU0veAOKL0u7VU+5yzqvWjnA724q2o9Yw/PPcFUOD18X6a8igWnPZOzyT3kzqM7uV+VPSks3D0C7w+9X7ysPTm2Jz7nnS8+EH+YvSqPCT4m59E9M72UvTC6O72tgmS7oES4PJ6Vkz3Vx0m9zCuDvVPCpjw4Rq09sA4JvUdcF778QSK8zg1lvkZ7z71pZgK9+hyqPR9oEDzZ9/Y9so6NPWqjAj5e2x09q25SvjxOnDwj+xO9J6WSPH3VtTxOI9c9uUZMvqdLST6TQ2a9SfJ2PUkbwTyFPL49srhAPrtfDT26gnG7uF7YvE2oFb5uMQe+LhhrvYVG0D0dFCu+cc8ovffmxz2okAq9PrrwvS4VNr1lQee9","b0aMO284+j3CGlK+gMnEPKnBVTxo7cO9LGwpvfSHGT0cVQS+Vpg6vUgjXj2djW67Q+lIPdnKOL4O6bc9e8HWvMd2nzsUFhm+ZjGGvVor4r11ucg8Z270vAxBhb2/AoO+3moJPURNVz7mNSo+DvO8Pct/D76kTce9A13gPVa4OT5q2sw9T2oAvmbquzzqnty99yv7PERaIjyjxgG9ePrwPcM64T3w/DQ902TZPHyU1b1wyoo7oMguPU9Gor29pNa8IVazPH1x+zwSEDe+6+EkOxG3zj0d8dk96H6NvUVi6Tpv/aE9J0IfPA0SUz2i8aa8edCoPJJVVbx+b3e9Iro3vRqspj27Rya8kWTPvZpdKb5F6ni+h/3qO1UCPD6lyA49D+LXvWAIBD7w/q28I9QTPcd6RT24Q3+9ZfufPcz+pr01E/a9wUa7vNb9Tr1mpN49Tpe0vfDpAT4EK+A9NdYhvZzuHb3bupK9wbo8vUXdPz2LYB0+foFXPprDSr3k5du9Mi3fO75ECL4HkIK9jTNlvTSD4z1q97o9v/8QvvK+Fz0oXZi9QUm5PIdnLzzin+68QfxcPMjfBL6dUiS+sOSdPTcdhj3sb+a8en2NvWj4Sb7MWI69iMePvbjYCr16pI082GKpuwmT3jxjG+e9RC+1PnZVDL21U7G9oU+EvotpSz032Au+BOCeO5k9jjw+eAu+vEeBuzK8YL1gt5O9h1teO5U0Sr3+eFs9ovC3vLlpwD2DA6+9/mjbvaaEXL6cYvK91fo5Pd/KXLyzF7u+OM6BvTbwTT2sz4g95DMNPmqOmzxjEdi971L9PW6pSb3sNFe9LgeCu9XIB74YxCW9AX/DPYpMErrwgQM9bW/jPQRWCL3mYhy9jHukvcWm770KyMU8PLybvW/OAj7K6YK9PRe/vFYXTD20kWa7TNRDO3i4tj1ugYw96A1OPCrRM75YZEQ+LP/OvHUN8b2VKGG+rT0tPZ/U0b3IiyQ+arHJPVfVVL7WyJo8fyegPv8fDr6WY6a8","aK+ovcVXC757zII8D9csvi9frz3YkxK+qJybvJoeHr1jtdW9iTe5vSM71zy7LWi9g+v0PZnBfT0gvsG9a3yhPfdeJ74LoH+8u3qbvYzcIr3dHYC9+jn2PJ2cpLwC+Tg8VZvDPb49Lz5TVpQ7NaTUvFeFsD0MhwW9aJZOvbYASD4W+6E9HTfDvcBp0DzRT+o8yEGWvQD3T720g5S9DU+HOtr5hz28irW8TSYBPcsx771LyGC9nWw0PTtOAj2MV6m9uoqpvFPa6b113Pe99BD3PIZchrw8WuE8pPVWviA6CD0wc5g8OqAWvv4v3T3GUPI9/gAnvb7eY7uonmm98Vchvvv9tjx364k8zbi3PC9UOb3rqXo9sohWvCYFgb1Xsri9Di+sPQCAg7wzjc699RaSPZZi+71qE4W97DafvYCA/TyNONs8lVzavanjJrwHohG+ETezPXgl0zuEnH6+NUb/vQQuXb6XmfS9BUoUvR4zoT0g8RC8w6mivQBvsr27Kdk9/i/kvWcfRL4Msya+MUsKvv6buDzKE707XdMPvZM4Ib1jHay9TayZvXAZHLuktLc9kMaBPYHdp7ziAZU8XduPvNdD3bzJu2o96g+iPV3dL77/KQm9mCCwvGXRpz1Wgmw9sUOsvfpGFj3UDJA8Ws/0vQrw7b38b8S8TNBKvEcVxbxGlss9NORSPZjFGb72TLE9O/e8PfJcJD29iFQ+YZgJvLgUTb3bxNI7lyh0PCWWAD7blDU85C7XPIUmEr2+ls088979O89mGz2HWLk8bv7gvfCakbwWMN09vzxyO3nUcbzQfWm+s7S0PaJmmjzaOSo8eMs2PZnJUbzSjPs9yPcuvn1Nbbx7v5s9k605Pb95BT5cULg9sr63vTBgUr2Lhag9cj6jPdV4zD2LMVO9/+MNveGkUb1k9e29yA1wvZlaCL2oh3U9R3sTPb/qyD37pA09xVMbvlruHr4X7209V902PvpInr2Zg769/bXjvT621T3jXTy99v7LvVZmQT4SF909","3BWPPdE9Ej4Bw38+7/2wPPz0nD1XDyk9zeHyPV88j72SU5m93HJUO2kDET5vph87fd4LvY8407yxoZW9S00NvXw7pbxj2lA9RYVlvpQjsLwWSw0+K5b2uhGs0T1B38g9or6FPfK1vb0svI082ayBPedfh7uoqqa9De+4Pc29+7x8uAY+CKCXPboaSz38bsW9EAoJPldexL2tqdE91x26PIUMuLx7FMw99Ju0PJ7TWr6qnHk+rI0lvmWVCL0yV6g93Q/nvTD1uD0lk7O9MDMsPqBw4Dy8jQU+drrju+5NxL3SX7c6gA4xPSG7j70mR4g9sVEsvmHD0z0VLsC8coajPQxkKzxfVbm80O30PaL2Hb7eBU292bdgvW2tjL0OP4e97F0QPvrhBT3FdDc+0n4KPtRiCj5JyEu9RgupPfo3xj20kJk7AAJIPh2vB75PnIg8yuqIPbEn+j0BXIW8wPUkvbcbYD1LDhu+BC4QvhgEUz07Moc+7AnhPCC3Jz1OJiG+Lq0quzCdgb0/qYg80C8evTrTW7322tA9tOGCu7Gp7byrlmE8Wur+vGSemz3XUQQ+wz4gPvaxjD2kKxm9UywVPGINjD0fORg8M+MSPaV61j0k1bi9s5Y7PTlTzrz7Ba89D2HyPJAUir17XDy+55P5vYudBD6LAd4939HpPbM2Nb2inSA+Gb+/PTW34zxV3YM9FEE+vQ0dZj1wLOY9pXj1PT9m3j1WJQy9U1SGPbHjIT0TgGw9atMfvuPvHz4e+5Y9kcaFPeKAEj5l9Qg+JPVHPcIYBT1StM09MGKePVFKaDwBqU++Fy3ru+wolj3kqls+LitvPbsgD72Osow9FQAlPcheED4h09I9/Ay5vd7NMb52DIw9ql4iPb7J6rzSMJC9dFVuvZiaYL2zGIe94GpZPUsQOT4ZeaW9HwqNO20Qir0LV688WXfbPcbZBj4M9nY+gPzkPeDBAT4WZOs8HptqPPSS9D2U2xi9sajCuy93Sj3SUrs9I+bku3YQWr2HUG09","zv6sPiQbrrx9ClE+ChuFvTnbrT7JO3S+DXjkPEFRgT47am0+LCerPFt4iD6LJ4M+n/qCPQ7leD2XHRo8Tp7RvRW5wr2XD809uzWWO8tuOz5KU8K8yRJEPmE1PL6vuhE992cQPEmjfb3mdLW9hq/oO0eUbjyR8Y49v3iNPYX5sz27L487jS/TvVYOeDtX5ms9PbKbPGiZq72bUUs9ZyTjPOThZr5vIwM+qAzYPXCiuTx+VPW9be2yPMd1ZDy1XOK8MW6/vHJ7dr7d1KW+YH6PPpv6CT46rcS9u58BPZCXNj54O709ZuWSverCkz0vbEw+DAaLvTHWGz57ek0+/f8Svqg3Ur0TODG9ifKVPsPpKTy3lPK9vgm2POqv7T15U3g+hr5UPTev/LpEzvU9AiRLvEgeDT6ZD4I+edLLPSiE1T26CxS+ncMSvUkJOr4ad8i95Vc8vgvUGT7xzsW9xVgwO/ZzEr4WBC6+TpTDPa1PwT2rlN4723aBPY2YGz2hG2O9XIOFvXvUKT7JaCc+KyJfPXhOpT7XGPO9R5OMvEg8ej39WaQ9Am49vTmoEbz14im+5zMYvihO1Ly4ZzU+l25YPn1i0j0Ap4c99rR4Ptmh/j5W4KM96BC8vSuFrryC2ZM9IoSSvE1SYr5RYkg+TFecPFdngr04XFg+aqAVvpLDgz377EE8AJbbPUncMz4yOL+8Y434vebYdz6yHky9jpTfPeQCR77ijJg9Cv6IPbjBxz2DIX08bkqsvWibjD2KvbM9thehPb/XaT5XLM+8qTz8PWUR3L107Ss9S0uePCWrTj0KXcW9LiPrvc0dA74bQcs65lc7PU2c/z1CTIG7rT8NPogkKD6SVwC+jLvaPEAUf77N4bw9Q3dOPhdkp7tTz6e9kMgjvrOUhD70SJW9amC5PW2XKb55qg28OHcHvxkogj0rHXk9zG8BvVwfRz30O0O8wlKNPAC/IT5+tNY9HbooPnbzNj5T9Hm+h4W9PR3cDT5M1mi7ycSgPdyw2TxotNe9","DrBGPbsjWj34NpM+fK/+PS71uD1AcWS+M3okvenhIb4xpXa9awWcPPvIpr0mSoU+nFSlu156cjsn4gc++vEzvgGQQLyWH28+WXmtPnsacb01eBA9BLXCPmyebL7SRBc+bzk2PjQnzT3U1aw9WbgcPkm7Cz79XIq9BzbMvQWkGr7NjMy8QlC9vfAMij3J9r87EUZBvTuMgr2wSZY83XHOvbsHbj4uZgg7hqYdvvAuWb+9wU+9cdkkPf+rwr1tjFU8AABLPSjsb76q58K9PJxjveudHj6oOnC+pkEsvazDib6X8gw+XSmWvnNrmj0riSy9c4x8PbDtajtB3yy+/kONvHGfBz76N/49QAY6Pm1bSr47VXU8jg+0Pe6KKj05amo+kV6FPiAUOb3swZc+YfbyPAFNsz1KciO8GypbvaG6ab1pkqU9/IDiPcFomD0mwK49cFi3PWB7sD3jSZS9nJOAPWBis7xEz5g9YUmOPYoDKL4hTTE+iyoNvcqAIb3TdR+9mPiRvRqp6bw71/o9tXZXvc7tEb0dYwY9ekcpvOcVpDtqGqq9fcD8vCNd6bzXJ1683SPFPbgvrz2Gow89O2AHvZ29Hr6Y2yo6tHF9vRK8x72FBRu8W5+fvVZ5oLxTUns9AOs4PPWWsb2eMHg9Nkg3PHbXp72EXd279RCiPrWHFz2IGRU9lycQvc0aoT2KKoW7Xd4UvMjiuj0gNYK9pXuiPScmwz1GSE89Ir5aPe72Nr3joo29WPBPPYlXFz4YwZe8enVUuxCg4TzKAQu+G7scvD/zsr3Vb1Y+AYFXvVFb4T1cCwM8KZlBvtjCAz1HJQe90Th9vZfBlLtVGYE9E2AyvJXaMD5xGFQ+t8s2u/+IPj5HP18+sI7nvIoWMD4Dmzw9nU34OmaSvb16rMI9gpTGvXdi3D2PBAO+lAuuPWVkN776Azq8euU3vKsjmj7esJs+vBE/PZhZnr0Q+7Y9218fPefmFT09sq87MfOePDka8L2nrY09XJYVPmNtWz3hiyI9","pHfJPRJoJb52kGM8QZfMPd+YozzW0Ok7h8+AOzlRdbxMMvI9gJKpuhsQYjyeL5O9lp/LPNkHrD1EcV0+WnoMPcvJw71HsyY+FwoxO/fPuj0zpnK9xpWvPfQY4T3X/gA9kybCvfAObT3sOXo8BCWFPYBGDT4NIbk8gr3qPN6o9b3KBMC8bZArPkcc37wZFZg9n8i6PVAAOLxwS0u9CAwEvSyPMz3IB8A9t01+vYgTWj20crK7WI3MPZZDrrwdY6M7bAm/ve3CF76es2885w8QPp8fybtG1ki7Gej5PYkaBD0LtWU8EUd7PXO607w3OeY9l6CRPW5irLwN38i9dTcOvgUDuT1VCkw9YnouPupjGj73wDY+EDLlPc3ZvD2i4r29zoy2vNOfgDtSxpO7/zY9PhW3Kz0yNjW93BEoPvVv9ryzSqy8wKBCPiC5Oz4AOrk90YfSOoeQCD4CFKW95+flPD43sj2i6Z8997qzvfhFDT4zTnY9Zy+QPWW8CD41JNM9PtZjvWSsAD3BOZk9fA7RPdbsbz2YG3Y9vpctPKvvEbzg3Ec9/jCvPabm6r2TK6A9RhqHvahXNb3WvYE7Xiw2PSQ2qj15qMO9Tdu1Pr0+NT1/6uc8g1TAvKe2tLwOqYg8T3mru5ubWbwu0ko9eA1NPi8O5Ls7JYC98uCFvc/OjD05vtu5P17PvYM74j0kSzk9WRHqvJOHlT1Ik4k64sl3PCgOUjtqKCS+jEWkPM8dq70eSLA93IDTPZpT2T3kUrm7Al6AvXKpoz2eoJ69DrYlvkeqWDy5h8E9G0t2PSI+AL3u39m9R7jiPeVnCb1RcUc+rdunO6iai7ul4+69UG1WPpWNPr33uW28ylcWPqHnLL5brNE9IIi6PVVqGT4A8nK73r4CvqI9zj3U3x++ky9JPv9/Fb1bzJK9ZajjvfCNDD4brOC9kdFePevTNT0xLxi+Q31xPATkzj1Uq5M8zFUCvvAa4L2yuAo+OSWtvbOOvT3xIo+8w4asPSU9az2WbmK9","H239vb2iJTtq1gW+xpmFvSt6oj18Dmo90DQDvtXWsj3E1B++KkofvmfkjT1xp209VY80ve5aBr4LrL69N32funlBnz0BHQ++ziSFPmH0Mr1/1PK9aVrSvbj/mr09VKu8Y3UEPqCAnj35DCQ9daFAvqwA/LvGZbY86/bvPBBeA728WBC+ofWcvipjBb10oEY9tPwRvkMPYrxgGUq9BhYUvUoTcD1L5ry+o1kCvVMeKL75Ho49vcW5vQFXgj1U9Vy+KqgAvo4EwT24e8O8CBYLvJ3vET7ZYyE9+bKCPvTiZL3G+YS9LXVGPZ8C1b2Dej481guNvRfZXb2d0B2+D6QmPgoau7ybXAe+zA08vm5E1DyRS+89RtxDvt0ilT3h6Ca+RtpSvrvkw7y8RHs7Wq8bvlfjmLxIPmK+Ju6TvRj+Vj1y6FG+WyyavEkonr1PKh++p95ovQRjoD1pVwy+a5xavdngrz2TBwI8QDAtvjCnQT4aYJu8+pqdPEwmsjzB/dI8NvXOvM5iLr6WmSS+L1oWPtwolL0LKXC9wWp8vQNNi75vw+u8HHwkvibC4z1haec9AJxpvZkkkrwXlu69Jo8VPXm3Zr04K8Q9Ql04PTROkj0C3cY9hkOUvTHpsD2c/x4+dtIJPb2n6jxoFKk8rybzvJd6oT1lJHe9969GPK73S74DAmu+Qgf/PXIMOT0XEs+7xcKtvSBTWD0kIs466VShPSxtPD67ypo9HcD/vXCh0r2MNrM8GJXxvKcEvTwKv5G9iDuSPXUtrz3HTCK+LTTmPaU63j2DP8a9IybIPDLGfj2VfQs9Bg8cOyNWlL2JlSy+okx4vOVyrT2IY+Q9ZoEIPgi8XT3RxbK9x8urvUc3vrzbaSQ8BJAtPZTTUjxwoa884/1TvaiODb7UWm69SXsqvPqXgzyyjOQ9HhMivej/pTzc9pa9MrmtPfq/Gz0TCgg9u28xPtzfpbw5lGs9jjQVPiPgIb0JQki9EHjIPWZxkr1y+p+8XGfIPWeqcb3hMAO9","dQkrPc4++z1Jof88rivBPXLgHT3FAEa+j8CZPY4A+z1HVCa+e0ruOhzsgz30LKe9IgB/Pbyn3D2oyg++Mv5HveDedz1MKgi++477vWDvtTy/zci5CokovSbqjryZPQ4+j7hnu4kzFz6wG129zZWfPNyFlL1jhZI8aL5HPlPJ9r3lE8G9BKDkvAOvPLytEBY948yxPDTxCL0bxmQ8eOfDPcK7RTwgwRM9n+GevMwfG73CjnO9T7i9PVSODj7z5LG9G7rMPFDi3DzQerI99u6JPcxBgTwHqfa8bP3nPGg7Pb5dJp281jsUvu05XL0f2vm+41OPvYpumT2M6Z09Oy3mPHwaib1Q0pQ9k1ehvaYpgj1CAm89tUq5OiYzY79cmYC9qPKZPGjZRb0dNSq+q9JqvQxLgT2ErCC981mMvbIsw7zZxU89esX9PRAftj7IVzS94SRmPXjAfT41rf68Ic7uPfHSjDwun9k9bdK7PSRKsLwKI5A7m3ALPltou7zyaT+89t4HPRwazr3TYw0+YK8svHS46rta1A+9s6jZvdJ5ET3j/IC78a7bvXC3Qz1ac3E+0E2bvOOcyj2SIgG+p7zyvZ++ID71Nhm+kKFdPaM4SL5K78O9szCWvMIF5Lw+FLU9KThMPXzhtr5klOC9XTImvCDjLr6bUT6/eBEBvjlK07sWvKk9O/lNPjGzKjrS27w9kh2Uvr0U2r1y7bA9E+mgPQUm3zws+Uy9dHXtvebKHb46+Oa+UWjzuwMIXD2ovue8SaO+PbhcaT6S9iO+QV7SvW63mD0Sci4+Bb+BPUeHgb2Yx4U9ErAJPkS5nL2iCN880k4AvGGnc70RTUK9O6kAvlGyWzwdmYm9paVCvCOX6T2gmlw9YJbdvDWC47zpom48Ag26vYB+Lz1kQOq+xwEcPSFUEb80OF6774WovhfQ6LvM4qe919aJvo/TAj4vpEc+KAOxvMJnpr2A+/a9+JSdvXO/JT6ZVSg9Kr85vebblrw7HnS9cwvdviyT+7y6bx8+","2qoIvQ96VL6N5CW+34N1PU6xvb2MA8u9XYu7upeb472rPho9RmqJPZDeDL3Gd509Ef3SPaBjL707om68/mOIPGT2xb3hZlG7Ax0qPl/SI70CPpO9JBolPXuUMT3W3UY+YZIyvQe4iD4p2Xs8DKVxvXMIAL6CzAq7itC3PXephb5UKUw+VrFRPmOuTDxN5P09qVRWvWIXljvKEYy9rEs9vYC8AT6D4Ym7wp3SPKJmaT3mpMq+PCYNvlmBrDtaynk8evwaviEfXr6myj2+C/8EPhj8IrzkDnK+sMGKu+19UbwSAOK9gmxjvSwhnj2GjRg92O/DPMeGeD08qVQ8AMFkvaBFnb3lKAI893qwvaLxqj0PQOS+e6o/vu5pBT4usLm+5oW1PENydz38tsU9cdVevteorb1C4TI9mWrvvVVcjjxviCi+f6gSvcyG8r0WxBK+nxMSPZ0fG73B6Zs+UiqUPeLryD0BPlw+pHiWvl563D4C4cS8bXojPmkx5T1RuEw9+HdfvMCO0z0O2rK9EF8+Ps53wL144vG7LDN+PTmpbL4iFga+IqzPvXgMQL2DFpK9cc8nPtCVNT6iKBU+dWrkvSRCOD3lWYa+7BGFvbRLaj507wq+RulLvi+oFL57UQ2+dywsvuDJF704064+wHKOPSvBMj1f6Rg+X80KPvLk5r34GwE+YcdpPTl8CT2IHb08KB/EvkKJND2nk9M8P5cGPabkcD4zdAA+FjfRPQekO70DFRi8n0cFvfE45bucSxC9J1wAvoAGCT6SsZ48AVYDvvKQhD2fakQ+xB6RvvQOqT0SO5O+4z6LPV0+z70a/ye9oCl3vfgv5DwNXXY9nt8ZPQ7A071PmiA+MuAMvU/rY71MxZo9x48vvvvlSb2F5W68LAnyvcLWND47H6U9yHKVvqWndzyIkT89tPooPYaqOryZGMi9N1CdvQg+gT1VTFW9/9AcvT/7PT01bIm+U0IUPlwPOD0Hm389tcj+u60drz2LxCK9oWQ7vZGUL75t5gu+","XYeDvrEFMj4DKr481oeZPeLcNT77jC08UC2wPZ9pDj3zBWs+7us8u3duHj60jZM9rwOPPZvBDj5qmTm+JZV0vpovWj6wQMW8IZxWPTKoYr0y/0Y9EVm6vPXez72+eHM+YqMtvI1/i73lOSM+YkpRPeRzB72xEL89w6tPPOaq/T1+8iW9x114Pn7y/L2lsZO8qKwCvaY2wz3a6zE8YZsKPfn+3b11v4g8Dc5WPcOYur2MmQe+8MCsvUtFGr1I28+8lQTuvToCZD5ptKc8pFucvTrnLT0XHsk8RsslPY8yVD3F3LE+UaKVPuX4sj0SQKa7612GPfpyybxRZnY9QYKIuwPqVj6Ew7i+kPbWPfDu6j0tvZ+6rQynvoZDz722HJG+V0DDvYB78rwLmlk6VtAEvlFKOjwD3Bo+/eksPcCg2rxIIra9SaygPBWZCD4pd+k9iioRvT77QL6bSuK9CMhwvHcDrb1XKTs+z2i2vc9w271/bmi8OwSevmculb1UEp6+5oyoPQCDEb84PwY9+NPnPalZ+zwzwwG9Og/BvB4EgT1h6Ss9PiYtPdaIZD7WZvS7gh77PUmvPb5z0Gg+MBTZPUnNoryblgS+Gs5vvkVcDr5lq/s9KstgPBQlxr0dsvu9+YkpPk7flj5WKRE+BKjiPeSzS71aGPM8APelPX2ENr1Obhg+b7yVvWpGur604R4+FJ8WOymvnr4Nsmy+QtffvPtjKT7JtQE9gGiSvfT0fjodzy4+GquePcP8ob2To8g8n/yQvWeKSj6UDxK+c6yavjnuv712Kzu+B/OEu/VU0r3CbZa9nQh2PkWMybw21s89S28xPhMqzb01CDY+9mYHvdqMXzyNpQq+rpfvvaPM/z2f2r+9adXkvI3E4z2yDwS+QuMBPYFauLzQ4iq9AcPZvW3GBT45pkk9/nbBvYaPCr1Fz589FdGNO9bhK75queQ9IaxBvXHY5723Dbw8unpWPlGSHz6efbS9EB0wPn4NDr/DFZg9AiGNvezKNT6MMM47","j1JSvoublT0ULw++cbK/Pfk4nT2MLg++G6t7v5TezD0kiDs+J36lPYU2rD3meAy9neWovlShIb2upSa9jUoQvujOALyZ5Sm+AednPmu3GjyiR6M9QYO/vZ0wK768QWC9JWALvuxkEz53o/c9BYGDvPTbQjx+NVs+mQ2vPH0V5Lssjyw+GP+6vTktVj0Xy6G955z+vZlxm76aIZ4924vpvIe95z0fMw09PqyFvRVLYb4K/iY+PF6ovYYQ9733zQK912wNPrRUCj5GFgI+rrEmv+jPRT5FtbY9UamePS+Cwj1Gohu9BPwtvj5e8b1ZPJC8GObPPXxc874Ce3k8hxmcPQPFsz2XPB8+/TupuxQe5735fjy++1pmvegovD164F875AkjvZj2aD1ocjk7bdOePWiLA797Qqe7/xfMPd0Xcj2e90k9CU2nvJd3p73Emig93wCBPUnmcjxeR4U+Gp8SPc8lE74t0YO9V1KSvam/QL5mao8+m5cAPUfO6r1DGx07+VnkPJvj1Ty7Uq89fhW3vhCxzL2UlkA9reXJPcSYkT03+sw8d7AWvhXlSb6KTU0+A6F1vuE5Fr1C5rO93KU5vHsSS72ftgO+ODBDvS7JYL3yK2c9dwjdvSMeMD5K64s9/PqgPWIBfz66Gle+i4RCPAJ9xb3dkVa+wsv9PSYySj6yhaK9+sJxvplH5LyiBbO8nENgu7z70771a+481YrxvT03A7626JQ93Ga6PFjlwb14OpM8cbWDvC/2Wz2NEhE+dQaRPcBJhr3Qdk6+9K6sPYrR67yWbWC90ayGPYgssrxDZEy+pGfdvvlVnr7E5T2+ZtqLPmjHl73fPcO8ddDCvtMu9b2VcZe9JzOePHF/szl2Xu+9vg6CPReHbbzBvWo9JdeBvXeRizwxza69CvszPAxgp70hK/m9qXSLvZq8iL0DcaQ9/5kqPICmAz5DRj4+E+cJPnMxyb1DmuQ8gv6zPYj+hD3M/I8+e8ouvd8rAD2lBzM9iLf8vXoOAT3eLFE9","/gC8vVufhj2ulQA+jEcSPY3zWb3Igt08ctGGvhrLWz0nvsy84eq2vVxAHr43Spa9ZdTsPR36hL1b9YU9wP1Bu4BXLL7JNJM9Wm87u9ny1z2GT0i9zqQEPjkCjD2OG5U9MkKFPITVAb4vM4m8qSXpvKc+Pr6/5bk9r+SUPBZfPj37J5+7SqBWPYOi0TwVU1W9Z4/5vTATID19g+k835AyvJUI+b2vErU9uD3DPefxrT3vbHS6dy6xPebXWL2LZ2I9Fa41Ovea5DwD7tA96wsDvlNomD3VVOy8NI8MvSd037ydaZM9UAlZPWI3Qz5tCyQ+uYnhPYrVcL2mbUs9ighRPeDUjzx37e08ymdWvvGL5zxIXKm9iju/Pectvj6DnOK7Cp8fvHLsU7yMUws+xd+fvQ553L2xdOG8LWqdvbkO0z1jada9vDN8vZa2kLzqF008i9O8PLqZWj0Y7US8kqm+vc8zHL2sVWc9BRlHO0KaG72xOdu8340Zvmh6PL3EwKk9hZaXPU1Vbb0j2q+9MLG1vLCeDr3/tFI+D/J3vBMh873UyyO9JM1bvQFPTj31vYQ9fF2cvT2FI70wLYg8sjAxvdr2Yb0K1Rk9EvIdPW48lj0ibfG8xPDKPQ/AP77rhX68shG+vB9p0r3+aIq9E96AvJwH5b26WTo+zXfCPbn0JzxE+/O8HAD1vXbfv73a7Fi+zu8evT8KEL4Mo747aq4ovb7W8j2CK7Q9Cf8HPTEleTzWN4O9eOo3viWx2jkI9yW+/6ZAveuHhT1y0lU+l+ECPUIOgj3Npxa9XvKzPVrmTT3SYWI9WNhsPkYCnL3W/uW9MIAAPjhizLsuCIg8gTF7PWy1Lb1c6509YH44PaTklL3X1du90ZJEPWILuL0or3A9Qmf5uyZzmL1YeqS9L2+avYY3azyjcAw9d5RAviANhL0h6H29KwpFPtekG7z4lSG8kfktuxy3fT2VB8K9926gPCe/9L0qja69BNKWPPExMLyj2Si9oJTRup/AcL198rw9","GsBQvkxpDj4p7IO9vvKWPXuupbyFilc+2rjVvblLPLxWBTY9YlusveuxAr6UTey9WzSBPe3dMT4RYZW+ZJUFPS2o7r2paI6+kpq6vUs8gj3BZxu+ZZP2vVduiD2U6hi+/uaNvXmBT7w8sPE9phbrvZRoTDvjaKA9wGiBPcnDC70ZKb+9rad6PcJR+T0lY+U9B7OuPTJ55bwCesC92hS0vYwLwb2JmN69oXdsPaSLHbss0fi9Tb2NPQ3CajyGg4c6cPAbvGRQXLvo3Za9X41evkwCSb6q0Yu8TtBFvXKyDL56Hwg90eJVPH7dCrvj+1S++371vRFAqz0lRj486XhZvZAoFL29xUO9PAaNPkHENL6QI749zutGPAKtUb0LT/I9UJR6PnspVj0hc4s9o9HtPb7M1LzLxAi73L06PcIpJTy2ytG8s+lKvP8DOz0qR8M8wwynPUaJtD0VnpS84emlPV/rtjxmM4i87JlXvXkqq7qg+0i9G7+KPRupsT3IFdq9EQ9lPVertLyYwqQ999jUvPf+rLy4UJW9qOlfu1WNpj1TJTu9zSDQPaFexb3ia4S9iAUMvrOwDT5GCe69gjMNvkOMkz1okJK+DXQCPpc8Vj0IOca95n3XvFW6Bb6GhCq+WkL7OyhVL75u3aM9LkScPZRZVbzDj4c8rvLAvfK6lDyJcas9+HoLvp7RJD7Dq8+8phVMPSRxvryBoI4837M7PJoyQD35lAA8BYqqPbw25jyv0jQ+4+KHPQf2tL3U+a49IZakPL1PwD3TpSs+wbiAvbEdtD3AA5s7ysuyvHOqgr3cCXA9HPUGPtqqV72a3MI9CZ40Pea9SrsGjjG8fSCQvJNBB74uI5e9+/FXvmC3oD00Oz29qSaCPMSORb2RWT46Zi5SPdHj2z0FZ0A9KsgXvuKIGz0ltuw93oyhvX1tDj5kBw88suejPIWz2b0CoU0+IlK8Pef3nT21uuE9MhM6ve/QHj08mhk9Gp6NvYRZ+7sryL89RhZAvehoGz2dQng8","xBqRvZbpK72x/lW9tPE2vYfBjzv27q+9teT4vXYwjD00aS6+KoiUvUagkj3drQK+rwEmvWtpFzwht028P9XuPUTSqjwaMGG9wbHjvPqM/zt+bm48W1CXvalWab3CKl49u2GFvMELa72yDw++WvRsvT1VFr5/try9/H6gvAdHmr13TC49yn66PTQP073CokI95k2rPUaj3jzH7VY96tGmvd7aHL5cJ8A7QjYXvcBfnb5EHAa92IMcvcC6AL48KMw9f6ZOPg6sDj2Aca89lZccvfZdSLySEIw9Lzm5vJRwvz3gtBw+zC2CvgMJG725kuk9sk2Ovf8ngD6P4qU936ppPSTdj70RzgS+YlvrvfOdSTxXirq9xL+8PeutHL54rqo9E091vc9PzD3a2ZK98kE8PMLjLj0bKY+9zljiPehwTj3vAQ89hGa5vUAlWD00tyE9E6XhveNQkLu33Au+27ZTPgDX8LzA3xi+NesGvhq/nr2Nrvy9CRkwOxqEzL23Dwy80Qc0Pbf5L70lqhO9YMgePpEp8L3DYJS8ECKGvKEjHz2m/qk9RBG7PQiQNL3OYIq+fgq4Pd5/jj3g2CU92IfIPXAIRbzXvqe977sOvso3fT2lpgo+LMvLPZxGhz13PIy8kfoWvVwf/72+FN+9m2qWvfNKpbwRlpg9KxYJvuERlL0vZOy8xoKfPTduGT5uaQ0+O8z9PI9waDvPm2g+QgkRPc5zGD4VL6m86JZtvE31T74Vo/M8JuP2vV/1LrwMWkS9Pw3fvaDU6T3FK6O9mH8xvjyhir1XDnO+RTPaPVtTPD7hY589p7M8Pcv5rLwV3Rq+nYGcOxY3ZTyJuBy9SMkwPn55yb2m6SK9q7OmvfUJ3rozkUM8wkfGPYJlzbqhGPy9bUEavrAzHbs/+/m9vEejPC38qrw74g69LN4pvBQmojzPHze9mRSlPdpcM738L3K+x66Xvhh7hz1dwgA9Z0cSvQGvD7z0diA+t7BTPdXKf77srR69IqrkvRrjgrnospM9","Ca8yuwRQ8z0achw+o0yGPQtlhD2JcqO96w7IviERlz25Ogw9a7N/vUEvMbwLnho+QutcuxO7hbvvSqC9xeEFvllxpjvDo7690JuPPoHFur0mKDO9CaSHPcaRwj0XR2m8ugBhvIgqDr6mvto8k+VSve+reT3LNts9ZGwYvGSgtD2bupk932IEvKgSAj31pqM9mrEBvQ3abr6r3AK+asEsvXnAIjxW2B+++cDEvcagLT5nQQa+RxffveKR+L0zl7Y9TZiNvQ/xSTwRzA26ogPivVPrBb2Se1c8idHNPfwukb2gxxo9hE+iPeO4Aj76ofE9boGRPPATCr69d5U9g0yhPfc8lzoXUZ+9bgSPPWZqeb3RrBS+yKGePXQda73EWxG7ZtHhvQJqNr2wcoA9DF1WPsvZjz0UaZa9bkSVPn8gCD5++R09WkMRvvQeiT1NbQg+uA+1vLOFGz0QveC8s0QLPn1tCTzGL/I8LoSOPVN/Wrupbvw8HcbdvQ68e7wRSZE9ba30vU0jALwb8Yq9kRqTPTC01DxFo6G974RnvMq2Ir3hr/687oVPu3qrWD1IqS29Xr1MvkaAZbyRxbK9K0iluw1BET3hizA9DDW+PBsolT1yFma7TPeqPbPlg76xUgM+FTTYvQsAo70PBGW+H19lPNd0Pzz+9Qa+YtKePfsJgz7BHNA9UFptPrXyzz1qF3A9sqovPqyCjz54Vj+9fZ4cvuqXOT2MZSK95LhfPH7f9b1Uq8q9KCM6PINBLT6HFeE9E/raOkOsTz6YBaM+rB2zvBIY1T22SUQ97kpyPboOfT1XphY+p21vvkrHdj0YFs+8H2ASPh9z+DzrW549iCbzPMba/D25bm0+wh/6vXrf0j0oWAI+pMamvJ4nCr3vI4M9k6qKPXBwOT2pD5+9l0SRvWPl4b0jcqi8bPOyvUfTkL3PWoe9jIIUPW/lrjvt7TY+Z3D7PfaeeL2h/5g9rrg2vfqwGT5PIKg9YScfPZPhu7uxlrk9FXesvYO1Eb2A4F+9","YvQCvjfVBD5tPls9EKiqvbj+hz04XNG9nuzLPAJGrr35SB6+Y+mHPGOmLb2NnzK+jRr3vUKl372zi56++yVCvWIgqz1oE609GxJIPk8vE79RhjC+PlwyOzJhFz2Yele+rnJ3O9sA8T2ZGla8lvufvODD3D1dWDm9S6+2vR3MgL5Ryzc7oXAIvqDANr21k1w9vs/svUjUPr4xTJU9qJCcvuPX1T1dTgu+kjsKPixRV7wJMV4912jRvZNRNbsdQQW+6Um+PaM3Cr75LFi9eoatPIdP570XkiC93C5ovSO0Ej4VW409t/GZvVauFL76rOs9C/YDvCVGI76+8Aa+r7G2vZaOqTzciw2+mIywPSclOL2Hom2+CHWSvE7FUj6rLDY9fmLLvVUwOr16vQO9Q+oSPvZ0qT3oOBG9fLH+vTHABLzQMR49ciHfPV1Esb3TuiY7z37PvKHraTzRPhy9hBoVvqEoF721x1K7AIVuPP+it72F4TE90j/MvS8NcjzZ53k8OcDXvLHMpr3x9Ay9lspdvVejEb5XxsM925vfPUQsC77R4GE9JxDrPHDNj73UB0S+MiWxu3WP4byrE4O9vXJ2vWdBCz6ulq29dCImvikfkT0Dwhm+UkHdvcZf8b70e2s90zivvLPqo70LBDK+eGVQO4I9Qr5xfHU++Fkavm5uAL4Js5Q8stBlPGR5UT0Br6e9/Gruuw17yb0Z8do8yJ/lvYFxRT24pQi+frowvou8Wb3Ak1w5lAT8PJR4ur3zNPe9NwcLPKAgi7umTEk+3Z+RvJikvr1VO+85v3haO3PCZT1Cvso9SZWdvcXkxT2VzPK8TjEHvoI5jz3qEQg8mmDQPQGvdz1xb++8aTaRvdRbjT22ZMU7CjmWvRZaWb3Wo7w9tmvzvSRjKL51Nes8ozDRvYpvWT4oVos9bbPqPdb3Ez0Q5wc+u6l5PT3m+L2oMbu9Utu/PTU8x71A7Ee+qCuKPaLilz5nkQq+LyEKPQd7G71PRZq9iNtDPTTsJj4o1M6+","ltYdPT9jhL7pvQg+iLoGPWOZAL6kcfu9b17CPaQmmD2VLRk+oTpAPjSAQT0lm2S9ny/pvIBMh7vuOyQ9a0OKuxiCBLtKsfm8+5Zivl9vQ73QoaY9fg6WPfGfnr1tkmo9uSjlvfgIvrrtHTC+zO1avoVF3LyWvFO+npAXvucDYD6wh5m+XOC7vaaTT743pgU+X8KiPnw1Nz40Jgc+3MqXvV9fyT0bVz07c5ARvkk9aD4HJdk9CCOXPKn4uL38oFy9M2VLvUokqTyIA4O8bFZDvgKp6DvPVPY9qB2FPZGQj71nl487VUuEPZIPOL5y3kY+/BNhPabQf7xXFjY943SovcQzIT7XMOu9fGKDvXBH4b06tvm8MbbEPbF0Jj726yY+07pEPRcABD7KZJO9nPhdPRAkUz2qLFi8NTbcvZprXbshbrs9YXFnPda6WT2fTqk9dz9cvdUDLr0AKb89ou86vUPd/z2AlqK9778WPlx59r1LtI69piXVPS8VIT3SFAU9IcQyPSUFt72M7ye8MP6yvbzrpD2CCRc9YcXgPN189T3Fr4A9T9qfPX3fVjw+pOu9BwrKPKkkFz71V5g8FocGvN/S670M4GY9HSI9veKA3jrsvzE+zEpbPaGUCrzFsRI+TtwVPbVpHL4O1IC9VQrQvZsWg7v4b509ffgKvcmagL2RvY08r0JUPmKr7L06Z8e9gqC1PHftgrxVwhW9IaQAvlCeWr22RBI+mpkLvVIb57zCFhs7Hv1YPT6YFT4u7NQ8RlDIu/ZsHj0qDBU+6ERsPfw9gbqiDRo97b31PRtZOr2peis9cxs4O/8xnDz1cBG98+DSPY0Kib2HczK9YYuWvWF1BD1Q6gK+SRV7vaAjtL2LWAi89eKZvQE7Gb2BBua99OmavCbMEj6tgaQ9h4+Pvi9Dsz0oKeY9GO8Svu+iET1esDc9wRGyvTiPKj6hDvU9IEJovWjsGz2zUA++pK/CPR3uD71Xqj29oJm/PRH0e71pcaU9WF5wvZyV0z2Jtau9","sjd3vc0hKDzxKQm99uoTvW/02DxkGGe9IjGdvUVDiT1kFJk8OvKCPeEW970cp4y96mkRvgLr4LxMSQg+P7iEOI8KHD1WvH4+twIJvWVHMj7PUiQ96tBNO9BjsL2Scxe8w3omPu+/Dr689Wo++7T7vdfuMr7IlT89/6mYulTZqr1CZXw8KtAPPXW08j0WQ+a9Um1HPWl1dL0g68w8HpNaveRqTT1vr8m7w+P1vJubkbzX3AE98gXGPS4fuT0xRBc9zElcvt8Y0j0onIK9SEkbPd3Wd7zc0io9yFQgvjcLbD2Xb5M8kt0yvvmJa75OijK9DP1DPWJeDj1INaE8WmvRPcm+gr1Vb0u9i7zSPVQ7Z7xYvkC8Y7tMPcCMw7xyhJG9mSaKPV6OWr1O4Fc8vArOPRqsHr08grC9/kR5Pr1mP72iLMA8Hg23PfGpqb2OYNw8y0gzvd1swL2u3J+94g42PKzJvz2TvpM98T3MPYwW0z1TxUk+s+XJvXXZSDzC70W8pYoKPdoPt7xAHjS+VDx8u9IDPj0bJE68IFYRvMfev73bzcu9Bq+8vK+ERz3E3+y9Qs0vvRmoFrxsaAS8e2/1PQ2IVb1zn5I9gPh2Pd/haL7PJuc9gNHVPbVxsr13fQu+2CsrPon9tj0TpZS9aMdkvrD4Hr5la6693nDXPeKb17ws17a9o+vrvczwBb6t3TW+MhVbPJNfo70pqzO+gGQpvj0gFD1ifQy+9Fe7vVV3Ab6CZIO9g58KPVs/eT2trys7pyqYu2eEsrwDBXW80U0OvrXqmD0B+ca9DppevcCKgL3eGPi9OrLUPBHS0T3Ngr466KXzPSyXpTxzTKw8nQYXvdVWkDzAYWe9sS+yvUBmiLxiJnG9oui8vcqHir34oJe9oSLovY6prrwWRiQ9QrysPTEe1jwXpaY9fHCUPR8pmb1A9sy9B4HsPDCXv73GTIe9Y6EYPkNrxj2wtbE9j7E6vVfdyr379LK9P4JIPcv9zD2DmAk6W5XdvbIaZTvbFr89","heSOPX4HFL7Eqpu9fFLcOgN1Dj38VM+8awIJPNJNo71Clou8Kru2vCmWhL1p2hO+ig4WPVoSDL0gmgo+ARYCvgYcvT3qiuG9U4A2vTbb7zttN+i8XD9CvScVVr0LVZc9e0rCPT99L76Zy/y9Tb6tPA2/5LxDspm8Ww0du6w5Zr3bAgU9k2G1PYsyyr2m+9y9R/SLvSfeBr2LqiU8YbFRvWcOmLzv17e9t94OPtl4JryFAfm9/HOzvbsBLjz4rUK8yAwKvVSmMD013Lq9CN20vWbtTr2PF6m9U0WiPhzXLbyOrPi9rPmbu0nnBT5Zp5E8Ln0uPSKqDD77o/y8yPuFvdCyWLsZO5+9RGlNvSlk9L1LiVQ9/A73u2tm9L2+eHU5pAWnvTkDGLsW01K+ICoEvtPxKz7CboY7cLQWPAh/g71ZbZw8mxUgvl60J73D6RS7q4s+PGzeXL2o1UW8QXPjOwPAl71JJ8g7nx6xvRzmsL1Ssok6CZY8u9RC4L2q83u6nAZvPUemtD0lEKQ9DuaOvpe1wT3bCAk+k/D0vMnVd73ngg+7XxLNvAn6PL2PwwS9V0+JPWo6v7y5zdw9yYB9vXZPFb1jQme8LM4LPNXkLb6LGGY8NaLDvVZoEr5A8pm9WLOwvbgBWr1aO+i9KddzvZe9Oj1V92U8lQ3RPNHMr749eeY7gs4APWTDwjtPdXU8ITT+vCW3sT20lAQ9uBmVPTNbxD0KRbe9YzWlvTj3l7zXoJo8vaaTvVyOOr6cFb27Wwi6vX2Rnr0JvIM90QXFPCeGsDxqsVO+1BEwvYEg3LzKboi8VdBvvh1Igb3YIJ08/qgjvSP12j2AJ++9gq6tvXz12r2fVEy8MLOpPJkVHryv2bO9xFuJvWJUgL3hT0a9GP9APQU4oL0K8CK93SJSPt3CgL0QFLk7H/ELPjLBjT3XIbA7VKUPvvHRhryHlsO7U2D8PLJvsLyzc7q9nWEfPuESX72JMBC8+1e7vBKM9L2gjPA7YYymO8IVkb2UdXi8","5YRevX50zrw8IgK+FpE3vmoFPT5zKza8DH2SPCOws71Fc9U7MWB4Pe8Ufb1Ynja903pEPTcB/D2Dr7q8e8NWOpIfFbw3Ew89bGMVPIeGVzzExn29hEVIPZyRmD2ED6O9w+ZfvQs0pD29ylK+XHI8PdlUmL0RQDy+0X8+vVDxBb6plYU8G6WSvVvp271qqFG9CCGsPTQlSbsVgio9Yr4lvEFDy729VMk9KbSBPNi6Az4qfTo8KmMqPeFCUj5T05y8WZYtPTP/lr3o7Ek+7yGaPcv5Jrz9jSi9GizAuwr4wb15BCQ9plAQvpLotj1glsO8KVNMvdImDr2bNCu+QahJPSX/XDyKihY9+vIUvkOGZL2blYO9jiacvERUxb6omaK8BLQ4vWtwMr0jHwM865MXvu6yqb3TYAY+83WHvC8mDL48ZbW9v0IGProa9L2dBF+8jHUEPlm6dTyaNjE9XLLUvXURej2HR+09XyPYPD+pnLxj3E69bWhTuyX1+TzMR8U681f4PZCYwr3T9AG+TuDXvdFuGr15SEW+il32u8TMe70gNxW9lKU6vsFzOT39uR4+sTQ4PT+r/zzmshM97oxDvsX37j1rbig9qKG1vYB1rb1vwI89RutYPX87XD1GZ5K9he+PvUIbUj3w27S8Q+z2vfsOgL1fI6Y77Ppdu27w5TwcOqM8I6QavqTRLL7cUtu8kUmZPZMw4r2XMMc9kFuDPci2br4By2W8mKrpu+Duqzxd59m8TrixvMRjyDy78gU8hCQhvtFg+722Mp09q+qMvTdsnz1JqO09Wrl7O4VOub2oumk9H5BkPdXqGb6ogMc9ifFUvcloHb3zAdM8OUDavfqfLT4VUC28SynlO2dnAb7jEB69GyCdvX7L0D1GDfa9sl+LvUwYDr7yW5m9N0fFvV+1gL7eZgu9iDT2PXzSkT05CxO+Vym2PWKgcb3EBZC8+xSsPLNZCj6TxoI9lwhdvhYM3D0rSqK9WXMavkq6Ubzojyc9PR/svDhRGr7xQ1O9","cj8IPYjvjr28ASW8StndPMqNBTwYde29ubwNvcgxiz2+Fj49EMw2u/7JBT1aida9v661PPbw2LxBKoW8dvTmvS8jdTt8CXU8NA5EPLEEwT3/Mik8kBeDvbld1T0UA+k9okDUPGrVNT0cdbe9CPV6vZImBL7BpDO9Ee0Dvvh8Hb7cH9q9lqUgPk78i7xPiBA9SyA+PS18Vb2Apls7h/OkPdtOyjxGTFQ9OziZPdOUg75t4ak9dDtPvLynpbxS2/O70XCTvQJ4EbxiOz4+XExAPWoFnL5gCQS+QvNou6b3Ej1M1W68pLwUvbQGOz35nfO8Uf1YvZ3i07zOxL48n+pwveg4l72cx5g8SI2QPDwURr5TsD09qaofPh7IjD6Nz0c+ebmuvbG6/71449I8PCQbvtAxorwnd2u93cvFvfGRkL24DwY+ZtVEvThHYT0DVZk+oRs+PnpzAT4Q3o09Z9+TPVQACb6vQ3K+ATRRvXhT/T0hWX29lnH3vMtLJD7VWn2+/dP6vLT1MT7FFIM9rj5WPG6a/T0GtJ+91V9evVjc3T3myhS8Z20nvnrHar0gjNE9Ta8VvebW+73pixW9F88wvud5O73uQiU++jLePTO0ij470PE9/m2mu2/sJb1AAKK84EUxvufTXL7rLou9gtBSPvIKPL2Dghy8yJ3bPR3LO73ZR9489lmavS4QSL0/sU+9nf4jPb2ImT3KWNc+txG7PXmXJr69d7e8wBmGvvPLBzyQyXY8/jYOPmwMcr2tEJs8IPwsvF/C8Lvkwye+tiS4vPradT0sEiC+6aIqPs/KLz4SH4o9vH03vjZRnD2REHs95A8+PewNuDz2py4+aGFsvcYz5r1SDYM8Rd6JPrQUmT3YtXs91rhjPQWTqL3a2Ce+ZQZ6vg+iJ704QgS+h6UPPfRWRb6gajG9h5ALPgn+kT2SXoa9dqo/PrnpOz5DwRc9E45WvqicaTzwApK+CJYAvvfKgjwAcGa9Gy7UvUFydDwjTcU7o9GTvde3zjsuZqG8","tzKTPdvoAT6YsBm+1LUzPvcFjj5wvt69MCsEPsdqc77D26W9BQQrPmZoAD6ZQw6+zikCP5Exoru2CmC+lZ2gvVGpMr1wVMk9KQiyPfuPcL2vMgc+onk5PRv6nL56iSa+OjRgPnuchD20EYC8j8SVPpS/Qb0qjOO9vMMAvQgprLtjzk89uz7hverGJrtCDNq9wTGGvKCmYb3/TRi9NpnTvElEzD0SI0M8vcMsvZc5Mj0ujEg98zrjPW8LsT70wYu9XmqgvSoyib10iIm9+LNaPdnkkL0rcwu9c4bUPesgFz5SzpS+3CpdOwnOYz76Yx+9fLD5PNC6hz20wXC9Ho6WPmx3D749NcQ9D84BPKmL2LyWxMM91RERPqrTYb27MNA9Iiygvc7NLr4j+Vw9aJIfPaF9az3WzNG9LLD8vCm0kb1HeSC+GF1+PYkhkD2Nuto8fT2KvRPQvjxOBMY985tFPWjjmTyemkO9rIxSviTBJr3lc4S+TXNUPjtAVD1/3D69J0VePQoxRD7Mw4O9o1GTvZU/hb3x0uq9ymuDuvLFDr2zXRI97aG0vVXGoD2HP+E9TCKKvZnFhb0gWtw8mESOPVPSZ77VbUM9hP1rvpFJi765mZI9M9eUvYPkp73hTcu8fqyDvuG1ab4DqvK825Z9voRWJj7MUti93vICvtHqNT2swlw9bGkzvQtA0ruIqzm9GDX+PBcrVL4c/jC8jayOvipMJL0qWDC+71LEPCxdzz3f3G89ncfdvQPw4r3jq4Q87T6pvZkKgr3RNZ+9OdNRvrqPhb3MwsI8E6ubPTJjf7y4sIA7feNJvRkcPrxhy209vAcZvmVPrb2Yo8K8vdcJPWuWKzyaTcI9uK1evWoOrTw3urg854ExPTAaFT2cpym9NqfqPR04Bb6HUcK9eMmgPOxd9LxBE4+9p2ojvo8mQzygZpK8+G+zvVahir4ApnM+2t4wvhos1DxM05e9pYpYvvKXHb56tnY9minSvexbub1dbR++jcK1u5cuMb7lDNq8","D6E9vH6mSL2fz2i7DB15PTzkAr6fItS80XaZvjryh773VdG7Dq1NPQamDj46ura7oVWHveGTp72zwyQ8HnoePWHU9z1gEEG+XQIHvgg/fby6ame9aTRgvX+EJDyEVZe9h1Ewvnvk3byMT828r9BuvGBdJr2WCOa8cauQvBZxD71gxSY9MD9FvUyJgD0D2c69BWmDvcr2tj0YqFS9WanEPIZGK721Mdo9FDr2PKQR/L0By9y9tyIyvlzK0T2Oxiu+10Btvfv/qrrX3kC9O8qWPSIrb72w6XQ9pYXcu+c0Nb2Vf8u9rXl3vEftET59OHm9lHzrPWLmc76ai+W8odVqvcO6m7yfSWg9SMDlPGRvqb2zTRS9L1tevVwMiT1N51G9l51xPfSdlD3BdRQ9Dfc6PcGZZr4xffU85dP4u36AMz0qk+g8XhBIvokdZb3h5GY9EhC+vPyKmr3SnXO9uZsnvfVEE7194rc8HSygPU3fhTzksRW+a7G6vVtZUr2hfhu9eSsQveTUHz0NBls9g8hmPaReVr2Pmn+9v+O9vQTU0rznNIo8PX7RvVq08Lw0PsM9qUxsvcCjo71hKeI8VVzxvAP8QL0meJo80z6lvfchnD2+F4a9/p/sPenys70Dp1m+f/hrPYoVvr4xDfa9O9w1vKc9Aj0rUf29ufr/vR+5mztYwpY8xkvqvELmRzwj5z296Hupu1pSJr5Nr4+8WMBLviGA6D3/dI88hIozPVGwhD3GYK28uHLSvQXMM761+yS+ObcWvmw37bym0QC9c+DMPEWZuL2lcSY9rAAXvlJtiz2yx/s9/sQ2vlglwz18+7U7dXRcvD+gPj0BrrW9SJDuvf7MsTwsG+i9zU/XPLcJj72emU++10ovvYw6lTwZaps9t6MJvTzASr0YDQE+yzqgPXa41b0AZiO8ozaCPRI0gD3d8Y88MR7hPVwkp7361BS8M+IIPPOKTr6rEvq9JcqiPJocgL2UrWE9iMwTvt9unz26fzy9aZ3rvUMFIbvOJBK9","X4bWPkxpRr34Eeq99ooPPZ1ufb05R4O9JFSAvmjMS71UKQs/0wKvPcrZBj8KNmo/xod4PdQPj72COzQ+/3vFvVfGBb1AtN8++I2+vccgzL56PFm9BCZfvtLNnj0HP36+3THlvZeUDz5zBrS7UWEXvcXHs7xSdGO9mCRKvgnEqL77XMc9juvKvVp0CL3ugS8+ORZgPTGYCzy7s0A+ZGluvXzfDD6t9TQ9THYavvrHCb4kGrI9jg0LPgUuED6oJj29MSRWvdUzmr4pf8W+TuurPR5Iaz34zWm+GCfNPReMwz5b7RM9WYLFvVbjML6zsbk96rWOvR6ENL2FE5o+u7xEvhBBHz1U1Ew9EIwoP5i6vryCfly98ByCPbZJ/7uRGc08WzoCP00xgL3es8K8VUqZvUoAoD3Gjcw+Xr2GPgzhzbwP1z09wABjvc4PV76in6G8y8ZtvhfYOj86mAM8IAsFPZ0ZbD2T/iK+LjuNvT/A1j4GVEc7PzE2PbPiML1xUNg9Xjw/vugyLz/w8MK+EEulPQDpDz+MA7O9hg8YPl+ImL1OKYG8DHoCPttXsb0UTxe/fYNDvsTWBr5MINg9lJeaPvw3hr4KO4O+17vaPkS5oj6Eqtw9R+xUva7yhL6ZZg6+eYrSvHUD373vGak+ckg1PlqaO71pq1q9F6z6vc6HMD6QPpy8zBr8uGr+IL6WmQQ7mx3BvcoUrT6w4Wo8rLKCvrhP1j5HDTQ+WM1OvnTDfL61GKM9eaJqvsPsLD/e6GG9u35GPnblpD0seSm+bMk+PZ916rulI5W+K68mvsY4ab3M5KQ9c7jTuzScYD5CrPa6FWKLPWXrSL7T6LO9XBl1PB8Uyr3ckH89ejDlvS3uw73YaKg+lrsbPAUUc73PCwu+YWX5vdAEHT9hXoy9Ua0wvogbd71nrTI9Z6dpvSAw772r7cW8tz6gPXaW+rxwMSy+kBH5vY2H1j1T+IS+E2C8vcMeij7iPcu9lMAQvqF6nT6OGre80o/zPXgE/TyRpy+/","UD4VP5RAd74hAzM/mH8gPgDGkD4qYpu+8BV1PgC5Wj5CkEo+2OlxvdLoRL31LxE/A2W4PEoxJb7KjXM+F70dvjJA6b0hjhE/OypkPzzt87u2zYi9a+cAPwuguT3p4cs9+6AJPhKd5b06FP69fkoCP8Fxmr7inwq+xr7Lvb9Hg7xwpSM+dEnTvo9cOT4fh6+9fV+OPU/8ez2lXkU8z/opvu7u8D74L7E+Ms12vsSG1rxMokK+CidrvtwKmT5IxYq986ljvb2k1bxR7iO+c4CYvYsNDL7HFFC+ElDavdm1Mr6YS0q80Cr0u/p0HT3OMt09qmWiPjcNkT0LeG29Gyw3vVGw8j33Kig+T2/BPf+eYL4U5gq+ZSXruykSb7z/zj2+nOEpPe3xeb2pczk8afTePZstBL3aAyE9zyY0PCDmvbxtuzY7ZUrwPDREdL0OQzk+31Sju2LUEj3CGpQ9LgyxPcBc9D3dx3s8H884PFmMtzxj/Ew9nuvlO+jADT4fvcY9wx1jvT0fDD63fJq935i1vUYco7wogDo9eX+JPVUw8z1WbkS9q5rPPNhW87rZWS4+FXAfPRdh7zy8/T09EYnZPV+7F76eYTW9G1oyPja1/7wwY5O9lPsJvhPKmT36Kwe+UG/EO5ymYr36Hay9033GPbCiur17eFG9goaePQiVCTpZa6w9JIUgvhRCeb0WxmU9CdhbvTF/ML0hTCc+QHeKvYWlArzcuGg9Mn9yvZIsLr3PuCQ9SH+7vff2k70967M9h2xKPT/7a7wUPyC6Xd4iPeaxHDwu44u9cb3EPZo6IL5Qwos96XixPN9wJDyTMKg8mXJcPDhKRTy+Fas7cPcHPF1gDb4nfbw9fHziutNThjtfTfS8nCKxvGjjk73iRSK9daqEvXcpTT20ioE9tAtXvjRQzj0xxSC9VpMKPC2Bnb3RLRk8B+vuvPUKj7yMlQo+356LPVHnxryZcty9mg2FPTIg4Dx2jFY9cj8XPV2SkjyGXIU9nMX/vVIH273OLZs9","ymKLPbkNsD0tOzI94MFEu5egAb01iCW9hThLPObHYL3M2RE9zdQbPRY+UbwmDsE9N8VBPiSqFj3spjS9NO58vtD2WTx4cCS8ghIYvMOyI7xWtgy+2fxwutz6cju0hxk9c3wivdJY77uaGVe6A5K4PbLTp72/Yf0843YivGGP9b1AyHo5j3GgPVjXJj3kEmC+YONlvTDAzz2O1tc8yeeivDD3Ar4zJKo9zXGwPRous70vWkI9qJ3hPEEKHrxzO0y8CLxKvd6W97iK1uw9FsDXvFafAr3ZReI9p14KPKv4ST2xlTs9ytwOvpUN1j1BPRq96g/CvbJcUb3rt2K8ka0APlMZbb0UvA6+irepPdscj7ugpnM9y91ivORZPzyhho49RN/3PB1xkj10LZm8IFmbvSXsOL1OUB69cLkivVndID0dtOK9yhaIvbQww72vJBK95o8hPdWSj72NyhK9Or/HvAfbB76PtoS9VAMivmi3QL0qxQS9DO4uPbBIEDzfjVM9kpeVPfpP5L2xjJO9ibAZvaz1uz0jea69OD4LPVya8j3sHxY9FmYuPX5tvzzToxI7YaXHvHliu7xIIxO+2Wi/vQSa0z14mcM9ss0cPjWDPb6pnEe8/Z1FPgFV97z217+7tc2MvJViwz3rntk8+0I6vTgPqbwq5CI9686Ovc61lTv8VpU8poBnvUESLj0b6D88V1tGPeDXCj304GS93+HtvbvPCj2IE3Q8tLmoPcMZlr1EJb+8Bo0OvlMzBr6QZ7M9wn/+vQ3Df71hZWU9Si1BvnYtzz0s6Zw8v7JFvdjv4rrRFxW+aaGPPWaz0z1Pwvs8g/g5PgAHIr7wNoQ7QP8WvgEQAL25g5q8Jje7vTUtiz3PNfG9Bz+JvTFMu73aV7s8GwSzvVthzDxPb+w9tsLMPUMaEL4i72G95jPPPEEyHD1LuXG9u1z5vLnulbz7tbK9VDBDvTPDzLzO4/y8y+KSPL1DCr7MGHS81AQ1vlcL+b0Jxmo7V8KMvDgl0D0DRBm+","k1ILPTb/V70OrTc7iMMaPmdWvrs1ngO90ITuOwGdkT1b0sC9dA9svG+aqLyQ6gG9avDnPchbB72UPCc97Zl0PSAucT0t8Fi936QpvnfRWT2zpZY8rGNqPUaGgbzxAim7Mx9lPJwNz72hcG27k6AsvegyPrzkq5a9EsGgPUjiWL0bTJe98/aDvXMFab0jypq7kKYrvu8XOb5N7jm78dKBvWpm7jzAB5e91pRkPaTJ+j2+1SA95JvvPHz0ET6ctdS88rcwvQUCwT1zR6u9LrUJPYqArb0yhmy8DkUEvhHPwL2BXmU9NC21vMNz4jzuSQq+Zad5vcoN6j24suS9wXv3vdm4Tr3fXp67RHz7uocGK7zAMRa9SBCwvWhcrjwD9EG8Pp8tPRzBRz3fqYe+uAHOPd8AqT0lzdW8sCmHvek4nT0lLQq+UoEyPiF/Cb7Er7o9sVqfvZUavD1ANZ49G/CsPfsbIT6iDBG9KCB3PGZByz0TmRG+FCCpuZzLDb1GP3e93REHvp8Zf70eUMy838GIPVi0Lj1HkMa9TeWwvTDypz0Roho9RODRvcg6bDzUN8Y9e+Z+vSsJST1ocp09togFPPkfm72zkSy9UV8uu6V4Qz17Tzk9Zu0/vbO+OjyvPwy8q2vivStrz76AHZY9nDGAvdc53LzzmyO+Nli5PYCO0L1Tdik9vo9kvipWBjwGBNE8pwuePUZmpT1ASo27Ev0cPRij5z0jWNc9dbWEvaOsrrwOxUE8p64dPN18OL7ePAg9dksRvusPJr7LTCy6FeEkvXL8A74eHAC+qnzAvUuspL08EV+9U94iPMN5y73RTDc9L4YRPUSrQr1pMjW84fmxPdZ84b00Fb495Khnvbu0uDt8g0e+A8RrvYPAP7uqK5A8Ofk9vvvnT77X6tW9zd+EPaXtF73uWA+8Wq4TPX19Br3UBkk9pP8kPXnPiD3BX2Y98JiKPKQoqT02Njw965C5PDRW1b36a7c9DHhaPbgaEb2ghx89qtOQvYoUyz0w1PU9","Dl0svkFXxz1Qaqu9bP0kvVhJkb3pNsI8tAuJvblYX75VCsC8J22DPNFg/L13e089ET1RvUDRj710aDS8wZyIvUrAwb11kw8+pJMRvZqKuz2iOle933OAvQru7zuLtme96j8WPRp9r71z72W9qAeCvLYyZz2gDjG+Cs6nPCvfRb1Kd4g9AbQ6vWS/6j03rRO9/yVYvX56Or0aHdm9KHsIvhL/HT7dGNM9qfnWvOU7Cb55kxq+QjVZvZ44vjooelQ8aP4cPTPRXjyGRmO8giS9vdS5ID57GLi9zemBvWJKg70sez89WoUivgR63TxlHv68GiT3PK3hK7syhC6+t6STvTZqSL3voY29hY7IPX5dB74GMLI9HVUjvZd4wjwYlii+iFhevOGN87waFJS9t1i3vPgd1T3K4aW9B6GrvdYu/ruZo6M8URJDvf7n2jwNWC49OVecOyJDQr4xo589vWsSvbh+FTqRduc9T8YjvXEJ4jzyxyA8oPC0vBnoDb3Z+3O7jSlxPXNnHj3YCru9v8CEvMTYYLzgaCK9hUunvV57E74iNuW73rgNPjiO2TyGd4C9CqveOzvrPz1zedE9LuO+vIr6jb1AGFS87wH8vBC3lb12ULM9aKFHvcDpVb7mpQI9tmqPPRVrEL29FCk9qJWLPRj4yzxMqSs9DoipvBeGwLxcK7S9JlMsPJChgL1lp6a8aRGzPSmilj2xRby8xWDuPJq8kj0u9mW+coUCPjwR1b1fu7e97ejhPNAbFb4kahK8qq8VPmRFKTwmcfi8s/tuPceloj2KoqE98yMdvjUs87wKnqi970NUPSAngj12ODQ9ZAw7vM/61bxI8Iu851dCvv3v9zoYZpM8slocviv7Cz4/48q9Qu3LvS/Vr7z9JrC9tvF+vLrpIr0P0re9k32gvCH3tj0TBfs9lwZNvnQOZT25nK49wkTVvYXmlr2+QwW8cYKbPMM1aT2mkLA9Q49QvHqSQL1Z4949pAP3vOmioTwS2TQ8q2hlPRvPU71jIwa+","mQRXvfJVuL1YvDk9rJQDvX/OQr6rses8KWdFPT9i7DwDO4u84MCUvePcjr0W5Eq8YQchPRREDz1gGQ69omI+PGIaI72AJYO9WBKMvSY3lL2HhPI9EDuLPR37or1TWrk9EfS+vVO+AL62U6c9iLH1vF+rKT2Es5I81BUavSX++70FlyS8d2CpPLvgsr30VMe91tUBvvyAtbvJwFe8h5ScPUJQMb4Ejja+O5srvoZbyrzulbS9Imw4PDFlPb0tMbA9EyEkPdVhyDyChja+sUzCPSgXqT37iia9E9kIPC7BpbsTAc69IDDTvHqh0juVHSq9ijDXvV2stb1coso6LxuKO+UtUr3Ot3e8kbB0vZ+L8b07wkC+mGQOPeh3M70MIfg7V651vadObT2+lps9K7RPPOPu1z2KzfK8/xbPvX8xfjxyUeA98tq8vBIOZr4whfw9f80bPUHXaL49oee9M5GNvTTdML3rXdS851dJPd1gzbwMdfA9iyYhvlN7nb15rQu+CcLuvLZMfj0oqKu8q2kfPeK5sLxYbaG93js5PEL6Wr6h0M68g2L7vAzgWLy6Uq29/8p8PFTXj71n0ZK96B9evcEHez1sehW8/qapvZlTRL4njIm9sudYPdK4I71wLrC8PrcSvQFDZD2TnRM9LNF3vuPx7zwnHqm8UwHnvSVmrbuFWpS9726kvQqOGb3DJMu9uOWJvcY4k72Urta9hh7CvMT/3b26G/88pc5nvfxuF70NbSe9Lu4bvjo8hb371hG+kwkivf4cZrxs/rC9xtOJPOnLij0aMbg89zaqvRqiFD0DshQ+cTwouzcF6r019Mk8fBKGvS1Khzwu0j6+Vp+XvdF8gj2h1q29yzqAPd+vdTweODo7UYGNPMhDtb2UsIW8cpivPGujjL37Bju+RTEMPGXlY72nhAQ9H3gXPZJhQr67I1M9TO+yvX0Me7yfqWm+KboqvdwwFL4pAzy+Un3MvTIcDL0/fXC9IvCWPUYSKj2XwbG86YEovoJTDj25rmi+","Y+Pdvfzjjby1Ul89rXitvA5v9r3Mjrm8oHi/vbZvQL1I9xu9+VHZvX7Kpr1BjSU+IH66vWoJq72/jJe7gYYpvnKDG70lKwa+kICFvcqUab1Q9Aa9x4VmvaW8hb3+05S8vPo4vqwoS72MIzw9QVaYvXLkur1EWKC9QFL4PO3ORD1xdqa8qqwSvfrPHr5EZh6+TQHAvR5yfb6N5Qg9dk8SPGjQc7yKWwq+BLAMOx2KYb0kznG+1wqLPC7tGL5z+hO945SZvX3aD77HCbC9puCNvXBsJj07By2+khMKPSdT5DxSfYe8obk7PcIiYb60Tgw8MoG3O3yYubz3ERG+WrrTvfPRAj1DFhK+5YCoPcBMfjyak8c9K+YtvuN7iz199iS+Y2iePYIBGr3/Swq+7aNWPC4MYrsVFTI8k+UEvjMPg7v22G0+PUjFvewgeb6u8am9RH2nPRpsB75Ikqu9E6Obvfp5uTwFRrU9qiJlPua3orx/kkq9cc2IPWRr1L3D/kM995wKvvb/Jb5SBN+9YZqFvVxbkDua6+O9FCwhOzdO5DxRobm8ShFbvB72l71d0oW9E12gvZ3jNDzuoge+G0lmvpJp/710Ypi8cG91vsBuD73w80w9h+gZvhKhnjwgvg4+o4jRvIbe9L22szG7s8ibveCxEj0jLrK9IWMHvQ+6c7xJ2229sSQ0PK7wVj5GME2+/ss1vvSQyT176fS80pzevfebsj2xZTa+QCn6PR27y71ad9E9YM+fvPJ5Nr0ZWRm7qMd7vUqHAb4V2oy98RW7PSqc/jyQrDA9kY0qPU3l9jysOdU99YBbvf2aBL7nfFm9T4/pPZU1gjy3ABq94GdmPkea9rwn/Do+wjc8vXOb57yEFmc8CRDovSIivL0NDhO8H56uvQEI7L2HkWw9x3xqPkBruLxK8iA9qe6HvUy7gj20Msg9uwtevf5S8L3eOfS9OZq6Pj3NZjwuN6e94TaHvGU6hD0h/C29sXX/PSQeoL0ZRSy95Hc9vvr5hT20zDi9","A/CnvX4uhz2LtiI+fhycvRtLAz65gsI9/n1PvYpbNDxPRnY9xs0OPJ+Ylb0azhC9XH5xPgqjhT0fi8W9QTYwu8Vv3T0Z2mQ9ajyBvJj74zoXb0S9FTnEvaT0Jj3dRFG9DtA8vngr5jw8JAO+IHlcvRYhmb263Eg99/WVvESk4T3j1Vy9CJtevjLBAr3Y6/i9MGKTvfJCX71/QhE8kPYQvBIPrDxJ+JA9L6gFvSK1DL5psEw8QZvMPQJNn72Jy0a+8b5FvszCiz359y+9Ra46vlPorb1uFDQ9y8AAPnZ7gz3wKka+Vhm5PWQQ5Dzztji9/lPlPPAw9TuVW8G8nFq/u1Ezzby7e5896cS0PS8aLT5fmiG9RfvHPJ/R172TG+49S4g9vgL//ry/mtm9I0lePtAX/j0dddK7aE6MvfAOh7znHbi9ssXIvTG6zz1qWlm9ky7IPWQDvrz1sLg9SXJRPfGvXD1eGyS9mxIdvtiKKT7aiwG+vuQbPhUqwryMbcq9+DavuvCtkL2WYGA89eMku+ygz7sPu/s+lP38u0nJ2b3ORSa9+OaXvVPxRTxECL898FnMPNtV1r0++LK9IjETPfz727y+Nug8xguUvTqHTr10uJE8HSMpPvMCz72559u8XYbiPZ3tsT3qV0o8agPfPB83kL2oMxa+lh+Mvevxgj4DmHS9DFwSPKPqML4BcQ09neBTvlGRL718RkK9ztLiPNBXkz6zHZ29qDR1vXTp5jyuv829DvTeuipC9TwEfLQ9Oa/bvaszAjw1clM999oTPN95gT2sB1S9UpoRPMciXbwqwAI+jNPHvXrBCz45bOi9U122u5e76DwEKz+9b4oMvi6KQL084569ULoWvl9u773YZMq9xALFPFSkH70488i80Rotvo2Crz0Bz0y+Vs8fvqtl0D2rFz09YGA3vSu/Xz2J3gM+BRUVvidMiD3xn8y9KjqhPiE3lr2jLSK8et8JPJTTAr5TpO899kLIPUqBXj3+sEY85xc/PZDPED1Dzt69","v+K1vYcJAT7Mi5e9DmoxPQVmPT2Iie89NYmyPf7U/TxwR4k9PbaMvFrhqzyV2Ti9CJxhPJnNMLx4oVg9V+JEPecsnz0ct1o8pwufPa++lLtIwy4969mqPQEhVj2lCgs9FEQoPSkgyz1QNCa8Ry9BvhiluTyTZ508w8IMPMTAsjtqBno9+eIHvGVMsz19RjI+fYe7PdfC3TxLnKU7D2O8PbJ6nTzL0ms9R0YbPMBIrD29XRI++i1nvM1sa73WOMY9avqAvez+oT2/1ru9qB1tPDj1Kz5jMyi8KbWhvdoQjzqyYns8lKXtPY8dST33y5+8o0BiPfAUDj4JikC++CgBPBkVFr2mbko9IKvOO8VpEb1Ze6I8wVojPYHiM75TPgK+H4XSPU4yJD2nvhU9kthivQBHHz1fFga+P3UqPdvjIbxhNXy9ER1UPtaeQTxey+89XO8LPfBKT72wiWU+t3y/OyiZjrzXfmQ8BtOtPYdtIDz9GVw9BlELPSJtvz0CziK9UYd3PdWZJ72yV6w9/GXjvIbkLT4lHuE9R0auvE7Oo715AKK9eKsTvRN4rr3Ik989lSSROxG7Bz5AAFs95s8AvDhF+LykbCs8la2TPBjbo70p7v09vq9JPSRlBL5g/UU9D69lPcvqHz0tTPY8NxHFvNCTNL1GYzM9ErIhPsZGXry67Dm8XTejvCFBuL3H1xy+b5wbvuVy1D0fdDS9CBxzvT/l/T08oMo8UNg+PQ1CUj6C+C2+r3y8vT2s7L3eFo+8dHIDvboRCz4VlMg96sVRvZq0tz12/u67z9aTvUPgNL1dL3g+z0FePFcqkD1widW9fKFYPmN6Y70itiu82arMPS7tIj1c6dU8pY6/PEvM7r36ZRG9kLhFPVOl4zwxYiA94sENve49ebx+vXA9WUumvFXbe7zrk/W8RaKpvVEknr1cvrY9OuZlu/1GGT3CLBM931glPUsmEDw5t6i74VbSPC32ADzX90Y8t3u+PZTQcjwCmDw9OpGGPUE2ljyBvU8+","e6p6PYPfy70arrk9MKLKPbeE9r0ESwy9+v2AvLX1iLwaFLs9s2N4valFxbypn8U8TbWGvNb8Ur1dQRw+Fs2lPT9GHj1H3t48U8oGvME2zT1ahwc9PJynPQtlRjxu4B+8wqsyPokP2D2ny3W9XdOjPbawYj0rzU296HK7PfX5Hb0BSJG8pLLJPZ8tKz4UqQY+Ie5uPc8OUb0MyKa9qBsPPcyk7j3DDEe9JcjKOoIgmjyb43c9Yy6VPYL82zzIaWg9VeWZPVxsvjvgvh88fewcPujQCD7hGy0+xFoRPSm97T3Xq5k9DPA5ui7R1T20Fsm7QrjqvczdiDv1vKQ99FO9u8YVcL2Biwq+QAyOPvNOK76V/X89QmshPtWwpL1XA22+Z76nPc7jVz0WyjI+xJ6Bu0RUez0ndza9c3EavM4RlL02jM49OwMovFqOjz3WV4U+K8RuPupMCD6RGGM9KDJzvS38kT36mjy7cJDSPau1kb1oQ4W9dDMvvb/z5r3fcyW+5P+Nvf6E572E7tM92HjivdEdXD0njro9imehPU61B76t80O7Qxp8PBT+fDyHFiY8bwJdu0zJ6D3or5Q9YWyCvRYf7Txhng29X1tavScrBD15ctS876WSPXTtnT0SQoU9bgYYPhOStz184kS+D08LPrGT1LzvVOc97Y7ePXPF1bxzQEe9mH0qvoszWD6a+fy9DwKcvaQUzrwzG5494pnIPNIS8j1li5Y9Jr0hPY5zS7saT4I9sYPVPd6SVz6kmD++bDWTvRJMHD2NtEm9Vs33PP63pbzfJoo9dSHFvSs+hTw9xxw8piRwvkVALz388Kc8xUfUPNjuEr0uT0U9j0DTvN/pIz1a8/U9+ySZvdzlFr1xPrA9GtoSPV64sD21GgA+oPABPRhby70wJca7uyw1PgGsHTssIxq7MtFZPRg7Rz4kxm6+8AHHvWYtND7O5RW+HHUevReCkjxxaRq+Mm0nPAtQCr338h08WeYLPox8KDpooS29Sxh3PiifCT4Gy0W9","8XSavQDmSj0IGMc8LwNZvYyPJD6pfR8+xMGRPSvf4z2KZCc+PgwMPiy7/7wFroW941hVPhFLkT0Ny+u7fRzsvLnpxj0odSK9P1ylvcQr/jzgA5k9uUGvOwTAL72/6hk+sOIZPaXiOz6uokk99w+wPAHMr7spdyQ9kiRIvOlUE77c0x2+qt4mPD7Z9D3DIYC9GgwGPpPnlDwCD7o9d1KyvI7+M71gtU49k1mTvLEF1r3f7Tw+/HvCPOM4Hj70dLE94zDbPYyI+L1t8pe9QRwCvfQCEr2ktX4+CkeWvfVDEb7VM9k8aj6xviwWIb7ZR6c9obucvaytub131ja8oufpvcfWtTx7ytm9FcY2Pgh2k7yqWbs9RkaFPm+DHDw2ndS6jj23Pmij670i1Ga9itS3PNtt5jzMpbA7IrEZPieQCr3u08O95oKhvJpS1z1vmfq92jCXvQFGEz7BPlm+gRG6u3m3rrwcveA9k+YZPc4LRz45sxc+L70qvqfjDbvrDaI+5WijvYkMsTweWkG+NWpyPuXuwD3yOpw9tSBwvcdEOb0nlYS9HZEivWpnPr6x7MI+r/c/PGKnSj3bihY9D7iMPRmUMj1qtXy9WP6RPrgJjr0ennK807KVPj1g6byJUSm+fY8zPnDQvL3J6J+9oI4kPlsENT16RDK9ZhKWPDhplzyksAC+twEoPQ30hb4uP0E+SedivipYkbzJ44a9+pRVPmU2O719yVI8ZWTpvfDVpT2RNAY9x2gaPiTZojyiS749EWPHPZUECj3liGa9Q6dyPjOUhD1ynMI9xY0zvRHGfrwQwck8WXUfPhoiVDzpjx6+khJhPb04KLwGPBU9BqMYvR3NiT2reZ49+dWKvVS83D0rwLo5KcfJPSHb/T0rtq899nAxO1pNyT0P5BW8tzvBPeR647vAnOi9ojh4PWs35L0BsqO9ihq4PSYoWD6+lAC+ywBIvpi7oz34Vew9O3kTvuE1Eb3LK6m9dcRBvrHBHr44KJs9kNl9vAzkML56DsA8","2DEkvucopj2PWHa9oBv9PLev3DzdceO9ejAWvV8j4D1YB5q9Kdf4u27vwz0M5iQ9Ah8Dvbz8/Lyc6gS+4QgcPS2Tnb1eahS7YlyyvZ2pebyM/VU9dFA4PUQVGL0DL3W9iduWuutUVr1lOFo8y40evkLqzr2qROe78yfXvdYYob2EB009ukn+PFkZ/Dx/qxS9lhQivUcqwz2jrXa+eDEWPf5CJj1BpQ29AzYZvqcxjL7ddHk9Hi2vvd2x1Dv3/IE9PxTgvEIBDj0Mlom+QNAAPTT9HD55XiU+jMpDvJhm1b2RwOQ9eTGDPcFA7rtLVhM95QMHPR3cr74TfEu8PIRkPNyiNL3gq4s9r+w4Pe/PRb5zXB69TIbGvbnbiDwtz8M7+Dp3vI0P6T2StQU+zu5LPZ5kZDt6Q4O93qU1vH+ML73Z77895BaDPGdm0Lxwpgu+PBGTPDcCub1Dn8q9nT4LvSd1kL233YQ9wfmzPOKwKb0d4DS+VvSJPUCgljtYFW69MWY1vPdfrb0x8wm+19uEPP6yfjyvWfY92eLQvON6Fru/YyY+aRgQvif4wz3Z66G9LJIKup9lP73C3Jc+LaXpvU1MVT0JbI69B10tvTHv+b0hfba697eavTNkLb4o2kU9D8GDvlXvZr05Aw++s1T3vPdXvbzxslm+kesdvs6zlT5g6Gm9VCmgPTYVi76yTMO7+4CdvdOhHz7fOsC9av+BPEuA4L1roSc8JqNqvrHAu733J9q8n3MMPkcZsL3C7DE+kGXYvE4WEL4ZG0i7hegQvTfhRT1BoQw8YXIjvaVnOT2UtN09JC2QPTsBIj0QGS6+OMTivInS2r1seWQ91u1Vvpi4FT5FYbs8JRdovoZgXry1ghQ9lVAAvQ6NtLxWDyM9Dwv8vdYRY70S8L89NrbnvdGszr3/fS298eXDO4txhr3Jxkq9qM4qPsaBMr4Glqq9txDsPbT42z2pqgG9ALnKvcONHb7KKbA8Y5lSPQimGr57KSm+/BijvX4H9T38RAY9","fG/QPR/fhj3tI/C9ZlnpvFRmDb7n++a9mVTMPF82sDzTMTy+wnoZvjNRtLwSzXu9n4fvPORl37241HQ80VhevXzqL72cauA8Bs7DPB27Xj5BctO9uPSJvTN/n7ypi9Y9ptfpuqqMLb7AlAS+N8vcvQ8SKD6U/J09mAs0PcxMLr5Bvcm92n4bvCvSsDtZU+48LeOXPGdQID25N8k9Z9ehvcjthD1U3/y90NHEPXcvG72WVfa8vdQzvegI17yWAgu8Gui4PD5UfD2+Kh++5MOWvcpb/j0HIiC+aMKYPbA3NzxQZzW8nqnWvTZ3D779kSK+8bFKvaZwNj0aaJO9akkpPCkVOL1CaKQ9Yco3vTYg7zxRUcA91482PQT0or5olz69hr2MvaNFq736dq49LReAPZgqOr3h6L29m7ObvHjp0r2Wfbm9Cn5mvL3sIz21kNG8unO2vbU9or1r4aa644Lcuwbbuz3mdJ094/IWPU9eJb41/NG8Cq5FvkZgDL5ZDWO9mYBJPItzsb2xn749qnNIPEAX6bzekbq9t7z2PAxJLr5FatS90uR0vfwNd7zUikI9b2+TvZT4bz3clKa9O2rsvCbPBD0mEmo97JnSvLWGgb74xPI9uzCkvUuSQr4KS569oNDYvZauGr7i7Vy+0+6BvNEb+j3uxK6+BdxXPZriojtEdD29ctlfPSIF6DzVTCW9DZvpPFSepL0mWTW85FhvvXq9Qr3OzhK+KgRJPtLFsb3QwTy+9PGlvWU4dr12wc87pDmivYQFkrx3O1m8UJeoOmiaIDxYDKw8w0K3PZdLm7yVmVk9Xm4AvZ4uuL1rdqa9zqcePIOPMbwPg4q94XG7vZL6e72E8XS+9Cy2ve7HJb1bLe290Q9AvlHJrLwe8oK7zP7ZPXK1tr1Ku6y+xNwCvd10Ab72fSM7h6lAPcs0Qr0hLYG9S2CHuyMsSD0bcMW9gbiRvXUOY76e89U8v6TFvGC2hb001xU+PFC2vQJw5r17DA29++SPvsY4B719UzC9","fq0lvV49Qbu62K69iDhPvHIWSTxC/Y29S8ctO4wYhj30ZfI9oualvSLEJr3dOCO92INRvLHeijzc5VO+dn9DvZcUfz0FWjU+VGH+vfaujjyFs5o9NvEbvVVSvbt7ie69SDWKvUGp6Twc7909eerpvG5jsT348aQ8tKhbPNJD873g+sa8CCiuvNZjPj3qORs9il3FPGDiwL2wvDi9sOEQvmo5ab0xhwi+bTBdvKtbHD6Qvn+9tnBevauSHL4D9Zg9ILuLvC+Jtr3mxPO7+CjGvcMRnrwr4Hy8Ci3lPIAZDrxDDD09hQSDvQ+rcL3oi/w90X5pujOcQLsMrAo8hqSeO690KL5mcH49Ohy6Pe3Vor3zcfc9oG+5vt16wL2rbW2+iUgvvTePMLzMZ5k975pCvuo0cb4u1QQ9wxAkvEL8P7ycr9O9quaYPRslH76GsHO+g1TwvTs0ZjztoTG+sPkGPdg5s7xhehU9nWHjvTb2Nz7tclq+hUlHvR+cT7viU9c900mvvVsuxbyTIhE8L3LjveK8r72zI1M93l9TvvSWlj1Y73W+V6kqPk3UVj0HWWk+BEpUvNsRKT1nMeK9hUUfPZYXVb1zn5q9YwQNvbKet705g7K90e4VvA74wb3xuRm+wPoHvZAbD75JUbG+mhFJvSOSgzwIBkm8x36ovOaH+L1XIEe9EXGtvbGsw72uFnq+F6p7vhFWFr7Ox7a8Wm4KvpuP3r3iMQi93G9HPVzCXrzNqiC8U5ixPY1Ztzzetwq+2rriPMIFTL6f3KS+tsEQvovoR74m38q949SdPDORpjxQVzi+8/u+vuS2Dr4TsR2+EQ+ZvBH/2rxpbyW9Y/p6vQfb3jwlpPa9RR2LvcAddDyjgAA8elO2PPIj5jw60uw8uAi/vV45ULylxUo9/PK3vevuEL6Xjqa9gCz1O02IpD0Xd2i+qW9SvUUCgzxBiBo9sZ2FvmvjXT2n7ZO7jkoNvRrCnLwb9Kq9odfiPUy50r2xdv09U2oXPbC3AL3TamS9","mXMivtQ5CL7yshy75ehZPCBsQL75NM694tlTveiXSLy2J5i9fnahPCj7XL20DmC9jlSRPdSgwLw9vFe+f9ovPkczjL2LblS+zXtpvVxT07wEi3S9I8coPbJz9bxw6S+8mBtXvBmuqL5sw+A8pKsmvZWWLTzsDsm8kzQBO17Y/rsnyRG9BWznvZlRS7wlViI9z+YkvjFalL1DPkY9Y7eIvrItxb40JAW+ztNlvl2Gg72DMk49kDSjvFfWUr97NPS6NhXdvU66lr34vmK9xg4pO4jr/7xGkgK99UBEProVer38ySu84Q0wPiL4+L32bJs7Iv1nuwf0uL2w7Py8/dZ2vsLBUL5E7749KrfavSkIQr2YST87P/L2vrs0k7zkzXW92eEOuurnCL3K0sE9R2/EvQghVb3d/ia+Gf+0PbDkLr0DC5S9Qrttu75aPL5eV/K9uwBcvpnYrryMiCC++GO3vK6EtL3eSXw86AqYvfE3Eb669ze9OLYoPbpwM77/nh48AIuGPJkQnb7xdty8iCSRvpwm1T2rQ5u97f08u5jCxb09zi2+nQKGvWKD0L4ZfYO98NxFviyBqr0M7xW7R8NzvZmh2zvjgWa+PE8TvYFqLr5FEiy+H7iRvqvZEb7Q8u29PSpSPP2s0DwVRLu8EDEAvuZ1bD06nMw8qwGCvITWcL3FERI9Gf2QvO4zhDxAkae9h0uCPQTuG70suL+9fGOMPZj9lzxoHak9xzYFvZ0VGr3KD1s8E3ulPMZjZr3+05q9pdievBwCLr5CGKA9FZHEPQ6sRL3c5eU9MYvSvVFQujlTIMC9vc4zPVR6hbz5MFi+jBXovWKadr1YoOG8x4U+uwa4Ar0TH9o9RX9BPfblBz3W6w2+pB6bvfOk2b2XNfw8R1FAPZQnY77oxG09fX0OvV+Forx3C3S8pmKlPP9uXrzMnVU8KvgdvXCmBj1QSYQ+gHiHPcJ8rD0JTWe95ZsXveRIfT1ss0m9bhazvXJ4bL0DEas9sbP7PcuGDzvlBH65","zX2TPVpDWLzLFVq+ojYbvXBC6b2CXGG9RMMwPtufBT2J+hg97KbKPY6A9b1WTkm9YmrgvB444D3prH29Zxs6vQbPwj0KnRs+rsJkPlQSVjySeuO8bB0Mvm8LiD231hC9Z09uvV9a/D3QgL09DBCJPX9Y87xAqlE9hT4/vRopVL2uXTc9p935vTSB1jyN86e9uvQdvvrUfT1zCAG+D4+svPif3L3Fuhs95xvZvUizZ708cru8qWkaPf0/tL37gyK+oKLhPd5yuLociMW99PaQvOaO0720bjy+r9QbvUEgYzwL3s88zullvSkOoDxyNxo72SSLPZ4hAD5B6rA9Bqi9vCsYmT2dTlO9R+BtPUA0nj0ZK4O9IR7HvYjUyr2SjzA9LITBvYZju713y2m9dwqzPUoB7b3O7ZA9uFo7PavWvr1RFN48hK3xPeGOkDyG6ls9P5G5vVpFmb3VBw07lbwXvOJxBr2WNl49shzePM2A0T0k2lC+EWkZvdvRnrydCii9WMPuvHDlH72RIAa+bVwZPr7NO72VfWa+Kh2SvbKHNj7awww9Nj7mvYyZXr3BhBW+xq69OrMTSj2yajY+gN2VvYDtvzzodQC99OlMvYEVTb3iVNQ8nbnyvMBK2Lwz8cu6LHGPPSGwU75D22c+X1gVOWA+fD0AlAc9r6eIPevLAr0wcJ+90wgbvlTlqj3StN68DHIMvR3Lmz3nizy9IwzbPXQHVLwXwXk81aGVvJVhmb29zBe9X9w+vkUw9Dz+Nxk9qbGJvYq9Cz7Q7sy9BixkvemaPD3AO9w9O4XKvbXrnDvnHeO9crz1PPS1HL6c6Ju9OfT2vOdjnj3eaIc81SMevTVL/b1CexI+UoAJvqT6cT1y70W9M55DvehkD7xRncW7cC02vmYZXTvzPA29EfWqvdLQKD5znwM8PVkMPYiYvbzVLxK9mu7ivHImpD0E6dS9zaAIvonLeT18aX+9PRSmPTnXi7wF4ZY9xVjKPNQU5L0g/C++lStyPJFKQr3COEO9","SzzWvels8LydBCe9AEY1PmGAWL6LWV6+lRtSvp3vUL5LatS8fBqUvB/kU71xRpG9CUEgvri+FL1VWr+9mYCIPXDXqr0haxO+7zsMPvQTgD2vs2u8NaWAvc2A3j0tXKa8XmoPvcY5Or4D01k+ZDSmPZ5mDD37wzg+cG9YPcwDGz2vAOO9BqaPPFF0ozuFGxq+QxgIPlcuiLzIUg09AbWxPaq8Kz2YeAE+mKwvvRh26D0THTS+sXdfPbcEr72V69g98OgSPtThbrhRxFS+yg8JPnXnpL27zqk9HdMcOZ4a1LzmHwq+E28ZvoJRojzNNs890SSMve3O7L0MEOi9VY0DPkQpVj1CjaO9PCKDuf9j+rsOP0S+FUcMPrM31zzFAv08Buw6vSXvb71IijC+Sf2SvTlHR73+J+y9n7OBvUoHmL3Hm7Y9GvucvVqLoDwEh9e9bm0MvR1aCb4SHca8yh+DPPJGzT3RIkM7DaOpO5vqWz0B5lw9QzmEPRM5YT5bUCs9qJvEPdXjQzxgHAO+WxkTvM10Cb4F7Lm9MHABvmKjRT0JNCm9xSJMva1dmjxdpT69Qn8+vRyZDrztFAi+Fgsqvnf5oD39I3U9o+whPPomLL6ttTM+OnKuuygw4D1LF087RW8AvZxUbD7T9ps8SlVhOzlNDr33OP09OAPVvE1tr7vtAqo6aYwWveABub1F/Ji9sA2bu7Felb0JnJs9MPH9vY7Ri7vK0gm+83+gPT+AFj2jDtA9F+zsO7ocwb1d3Gq9dggjvgwber3DmkA9KbSsujRhALwsp1Y9oDZ6Pfa2QT2F1+Y9uCaNvTP1Kr75LJS+04Kbveljk73aIhG8gIIOvsWoZb1JJik+NdMKvLjnHj69s9y74rBlvrWCm7y7qAO+qXH9vFu9g7wQoyM+c0NAvoIyCT6Kdxa8aMbMvMwZ27wdDkm9Qfd1O+1MiTn/jGY91uVLPft2NL0mA3o7XjNKPVxfdLwFBFu9SLoIvreFyj0Ccrw8OSIXPeankr0urM+9","v8mVvS5kmL3NWDm9alY3vJyj2r2GxVu+K8YKviRmLTzsMSe+fBgQvsd9iD1fAPC8UsDKvPQe8jx23p6+UDlavc0kL76BZyM9W6QdvXHi/DzCEZW8zOivvWRtQL51NnE92unGvZC+cL7DNha+mlwwvb32pr3Jgjg86jwGPZQXsT2zmmQ9gJcIPfnMdj2voWc97A0PvmDIND2dOdW9VzKePNwejL0xpPm9kg0yvcuZXj5aA4o9IreWPXh4Yz7Gna09KhISvcODG74frha+QbGevQ3IHL37EHc+nRDSvLZUATxLFnm9IoJWPS55zb3R26Y7LcmxPdaHnb3C9Ba+NfZSPQfnIb3KReC97KsPPgYXKj5nQh0+wcWRPrkgvr3hAIQ9SiTcveIZ0T2w1xo7U6crvMajl7ybH3S+P2xDvbjDJ72bOVY9kLHGPcnzW71JyE++ztAgPVbV8rvpV4+9hH7NPThtvr3Zk1U9CTVUPTlwBr7EUza+C4KrvTkYU71Bq3m945FcO+FhE729Z4q8tYYxvahi0DsylSc9Zv4bvZd6Sb3R0dC8eDB6PF1+FD7q6tu9vGSvPe3UPL2qDSu+7eb3POLdBr1nyuA9vS5lPtx8I70a1M89vYQUPWolR738uB29QyEHPl7CkrtxB127adqBPiRCGT14tmM9abh3PTgurj3dH468h1CQvD+2tT1HuRo9SiP8vQyhr72rH7699nfzPdXtDDwtSSE+DkNnvMckmT0tOf07IdbsveIpAT6hlHW7+t8HvlM3T72V1qU8mpLsPXPyjT0P9nG6jO2kvXeMpj03zwQ8oPLvvenuSj1SxGY9NyCcPe/Hkb39u6C9sJ2ZvUYu6b2tEEK+eEUPvlQPLL2cwPW8vouUu1GxZzybNBa+7B0jvXh72zzzIh69lNNQPmaUmT1G7C++W+DBvfEuDb5K+py9EO2KOsJ2Lr49U++8izNUPcPaQb0ayl48b5rSvIrg1L3I4sq9/QfqPVj6qTvhEpK9IQbmvfJS7jt/75A9","jejxvO6yqr3auRc+5xRfvVpMET6fzN494tSTvREYgj0MGwS+tkVhPJS1rb0jA+S9Z4CXvdIvLT5ygW+9ztHxO9kWCD4IhbM9gYmIvW387jyaVBa+IAoJPTBhAj7FL8o9Eb+zPLkKND4FdU897ySLvZL0G75UnJM8t6IwOtXxXb6KYQC+RegnPaUXrTzqr449QUFFvOK/bb6nUwK+RnTMPGhaFTvjXg29Q7iSPPuw4T1i0gs+9VcyvCflnT3ZXAU84YP1va0ihb02yrE9XmWcvBHitD1CnJi9Q4XivTEmRr3KExE8Y6BJvg7R5DuJ4ls9D4alPSSTUD4yfAE+QlUAPYSn2rzTT849cLyxve6zDTxm4+O8EZFlPgE4JD1YmJ678UzavWARV73Eohm+CoPJPXFNJr1wuqY9qS5LPmhr9ryk8GI69KD2vX1JOL1hGRG+MgLbPdz46D2rAB88zqziu8QtlT3Rp36+uLfjPYumpj3lDPA8o+MTPRBiMLy4ZmE9185KPPxPOj7WSy6+KYcrvT7xSj6qhim9nsKFvACxuDxM/ee9qZAOveMMS72f2g8+GlapOxDWaD1UHwe+e67LPDcewj0e+sw8/dC3vdLcG77Dawm9OBkFveaJUzzC4Ac+bf/HO65Vc72thwC98ULdve1MGb07LSW9PXnfPfZ1jDwvO1c+GTpTvThYQT3TNNg9OArHPe6LAb2LlYA97PUSvGkGUrxCWeg9OVkXvn8SFj0VjuM9Vk0MvMkKzz2Bu0g8mRXWvabPe716vrG8OfHWPTuKJT2x1y29oqK7vKjboz0QrLk9quVlPp6N0b0g/y4+4hoQPSqVJb0M+IW9XAWtvYANWr1vwem8RtnDPI53A77KmNg6tHZxvly1hb2JiDg+XyrKvWz2aT0wCcm9W+Jnu1w8Vj0tQvI8VONqvufVU7t+Tyk9/23IPM1zQb5xuOC9H9zcPTFDo72xAaA9UlFGPh89Qz29LhC+la2GPFsjBT662UY9CKguvrBI/z02/Rs9","+vEVPEuDO72pCmq9DrU8vfYQtT2Bjry9bXzWPFijHj7ataW9CYcCvtdBxT0U+Yk9lRq3vToeDD0Tlpo87TGyO6NRI77UwQS+4v/vPOOgsjyqcwy+0pOsvWZ+rz1v0pk9dFV4vdiaKT1GMMG7P2tXPYxydr5TECQ9G5CHPKsQNbz6xwe+w4gTPoKu9L2sbII8IScqPtw0tL3N8r09ycI3PczuMD5FNQM+xhGNvXLJbz5C4Vg9m8NPPXMkA76dBts9aFCOvXPPrb1XPIM9iHtevXz9mj0F/3Y9dNPbvSihfD0/FDA9PH4JvtIk5T1q66o975PXPNwwCr7VrUW8Ww+DO9VfCT6qNvo82OwFvohpAb77Uv28SBAWvlGffj3p4KC7wtfiOx6QIb1br02+J9cGvUjROz7q93q8WNs7Pg4P9D2OOIw90cusvBeJuDxOiUS9J7Irvu2tmrwvo3G9h703vRCJdr2Y/JY9Id7BPac2or0ZA6g+8o40vVjGSz11d4c9iuZ6PepL8D3w3wG+Z04Avuwtn73Blja83oDmvF6qSr5rntK7UwtuPebpA74oWDM9vUnPPDGusLtAlow9LkfMPVcRiLy1EjY9s2AXvduCGr4jhkk+KQD0u2z5Eb3pjcc9lUVlPoSBNb7ndzW+Ju6OPD4B97zPXRi+OQTIPNDnKr4wMJQ+EN1BvZcwhrn0HSM9o0I+O4VP/L14/uQ9SMc+PtG9s734yLQ9U/mCvaax9rwvsRm8K5HGvEedF77htKs93VAePqHJub2kOKg9CbiPPYnGAT5Hgfe9zPc9PqmPCD5CdES9+cHHPSui/Ltf6Lk9KdLrvQtzh72jQ648k1pOvS1k/D2QO589du6hvQAxDTx/oJK9mTE7PsAbjb1bQJ08C8mkvZxC/jxkp/Q8oMU6vYBkBz4TfLy9R9PZPRoa/b2mFui9GJ3wvLrThL7mAmk+1qo3PofZJL5gS427S7edvIm597yjeTc7TlRHvbuJ2z2az+A9g4/9PF+sYT21j4I9","ZgWsu4lCyj0Ffgc946XovXWs0jxrhDU+S3ThPacQsTysAS2+g8jIvONcTz78bQA9wKSJPUxdbbwksdM9zrUMPd8/k726O2w76ymOvfSBGLwsVbU9uwwVPgREsD1JqDq9LOkEvZMvI77gK1E+ihEZvgYjxD0k1zG9MEoAPgVBhz0e5327fYSYvf7tFTxzn1M9zLZ2PYC6IDy8QRi+vTL/vdqQZz0LYc49cyv+vGehHz6pl5Y93fr2vUmeUT2fklw9Iz1nvkCc1z0K2gG9tpp5PmjQOr1kJf08lMICPXo1HL4Vh0k94GHAvfeTVDo7jLO7iiGbPDVzyTzNen89VwAWvbmPQjy9PKu9SxoyvTj1zbxJEV09ZVBCvaGBpr2+bwO9G2GBvS1CyT16rps7TtgsvVT0Fj79I0s8LASYvZ5nkL1bESI90Uh8vCFZFz5OLQW+niNyPNfZEz25Q5Q9rcGRPME5fr1SrAQ+hsGRu2KtSj0o22c+si/OO1+YlLzQt9m9plU+PRxbsrwps/E9CeCcPR+rNr06BpA9LzddPVb+0jzVEY68IDLFveqwpT1wo1G9FF6fvLr99TyENwQ9Vu68PTXGzT1g8QC7om8VvWh4Tr1tlOS8aotHvRBZYT1I4gU7Qf8avZ21sT1+bIq9r5pAPTkD4DvuqqM8xolYvgwiQj5oU5+9sIdtvdExqD3wVX49I3GxOgtWDz0zfMw98XvXO2UVrDxLioU9TXAAPkLyOj5HEl49hkgVPaAb6jxT1B++v32gPZttIT7ZlbG9hzw8PMQ+tTvmLc089KUEvm5/rLzyGdw94kBPPupijT0XLRQ+j11cvn8Nb70Iew6+UUV1PBVcP70Rzk+9T7iGPQbuPb4aOyk+qr6VPVQOCz633zG8CE0gvWGXRjyYkOW83NzBPJeJ0ju24oe8xwKNvTiZrz3Vjqu9YLSVPfPckD0iIuA9D9e5vbiwtb0mPSO+tgSMPejc4zxyDPM9Gw9IvYbcPb0UHTY9NO1DPjvTjr0wARA9","J5bWvcZs6T1VjJe8dGZEveYG+j06Q1a9aej9u4FmXb7u2uk9CHqUPdYZb73AQlC9HLbePeBTgb3BIzS+bdIevfJDXb2QsDw9QdlCPRLqYrzn1Dq8iTuku7szhL0pFUw+kCECvlmAk70rTuW8FD7MPIJ/xTwSLqQ9I1DCPTrALT55HcQ9binsu93/+b3KzCe+W94FPreEY76X0J49nwdNvWeVFj2yzEo+ko51O/hBgj4NG7o6obkkPSqlVj0DY4y9tNvFOzKYsL0bnxS9WApavoMRpbu8oAs9RVpLOvv7Kj3rtZO93WOtPPefOD3OAbM9aiOjPYfVmT1t8oA8lMZ+PU17Uz6VhgI+eoLFvROqbT3a0c87q6fnuxD/8L0bmJ69HPGhPX2KC76y1oQ+ANlfPkhlR71W9Ac+tAj4PP2lmzyqepc+K5v/Pe4XYbxuhZy+8PqrvXi2kr7vKmW8RwHmPdCHTbw9mie+RfbYveH5HD7MqxG504sUvTbiyzyhQNk9ZKy3PFaEtLvLuEO9bsBEPiV2wD23nm29OaabPNBmN7wpFjk+KIqhPP0yg70+FIq9EgUkvaTmtr0XcLe9+1DsvGGVnj0Zqfg9OtRsvm3+Hj5JDoY9YkYWPaOeWT3+kGE90dNHvT0rqb1PuMi+h18bvt1V+T3irYm8J1asPf0qCDyBRL29/XNXvAKbWz4ebbs9D8+2PanNI77M8hA+mpg8vbGq1D6WS9U9QgVqvgIIojyiFSo+0s7kPRAukj4kmai85/BLvXOlEb4MmnQ9xBRnvfQp+L1SO5Q+Pg81uycS3D3z88o7CjhdPV9yHz0TyQ4+yJhSvYu4ejv1DyQ8YF+iPN1Wyr1sj08+02q2PXut2D3y12Y+/4ezveVvCz5eYeu8rES2vWD93rzrnJk8J36XPTIVwD3GtcU9yrPpvYC03D0PJIY9wouKPah0iz7p7Sk+JIqmvTDG9jyN/Q0+vqIUPqB6gDspokw9H4AyPQKQmj0NXik+1ARmvQa/xT1rYI+9","jC13vGqkE75hSV+9QFFFPigYprw9bWw+fYHfPJVn/rvMbhM++L9SvhjixT223Ii+F4HlPlfsFj6rlpI+bVMLPQbBUD1ig1W+jR6fvT1bcDzxwtw7cF89O4CiBTwi2pY8CNSove4qRb3j+co9EHGUvccofb51qlg9NSPmvaKcSj4u2qc7bKKLvM79sz2wDj++SpjhOzclhD0+hCI9DfsDPtDsy70wTg8+d14vPe8LbD71qga9/zMlPbt79b2DPye9rbCDPPaeST78DeW6y33svAPVpjztqJq8Ie4VvjPZAb6qlr0+TWvTPWgo/b2yKsg9ukodvWXAhTxmg4e8NqLKvisSqD5FN06+wHa0Piktnz04fo8+90Qavhfgzz3iQK09Trr1PdM0SDwnjEi9Di8zPnhHkL1TSUQ9l7nkvTWi6j2PA8+89pxKPpCsxj5NlnG9FKCru7TKorxa/og9JOWrvLV0Bz7eCgk+B/Pku1kvTT6EaO499NHavarHaL1iVQA80tOcPcT0iT0las09FNxOPvVSab0BOa09dW5BPUGq+DwevCI+349OPac4qb6kNGW+6qDdPdNthD33Usc9Wf7kPf8W0L1a/vs8tC4yvtZrEDv3JUe+IzbWvXSdzjzyRei9OlqfPrzSNz4rlN281rxEPaIWGD6wh4K8Yu0JPiPkS71gQ0u88yyNPKQz/715Lpc8OMeoPf7Vgj2Y8ZA9HmLovIk1+D1Ooyq+QFP2uqvPuzwl46W8QIJ+Po+rtj0n9cm6hUddPRSlBj68gjY990pKPjiJtj3DPS49Ay9pvQcMQj7jvTO8QUqpPI3+HL5a/YA99miBvV/xab7Aizk9+vD3PYyryTzBxXk9+ItCPSHYJzwo9Ca8c7A+PRrImT2BOaE9XFqMvY8mxryq8Te922qSvXzlDb2+wKw8tP/7O2qJcL01ymM8TuIgPS/DLD7TTMa9W2U5PcO/Az4g1uM8tIAavibLKLwYQzC+MVfBPHBJQz6aET09ePHDPanMqT19Seq9","vU4gvjfKW71vh209jfu8PAvLb70Sc7m928/wPE9vqD7gzK87RoaYvZe+Gj7c7pY8Fl+bvVFHJT08BeS67nngPFbJnrz7ECI+GzW3Pmt2kz1e/9M8msWhvf9yVr2Xvka9zmjTPTRAOb681m+9Gp4iPbZRoT18aA89Z3oEvLtvuzw71QE+YUFVve4UxLvbJqM9qZEBPI8Iaz06iFc9/0UJPZqf3D3u5Mq9rbLlvRgiHD2SOGq91b/DveNP7T2ryp+7NuTCu6Eu9z3SVw89QGKEPTkz4T5QTAY9muwtPlcsSz11MCI84aypvY4sF725jR2+aAYNvsJ3qr6mBP48t7cjvVJGAT0fOqa8nsWXPLFANz3m4Ze+2h24OwlgqTroGCK9huakvXx+gT0KlOA9xiXwPVAsO71OTm88MAXuvPs7Lb0PFY29COaBPY08Qb0vwI88+CnePaX1ND4Y86I7PqXcPJG4JLycQSA+2PggvRg0gj2R1h8+mU+ZPJZLsLzm6Ns81z/VPXHKUbyzBh49kaBSvVoxBT4Orl+7J1QmPc2gzrwAAq+6itqTvFybz714sYQ+7wkTvL83pb0wxyi+8X4VPYvfnbxZHB09Rlb1PTAk+Ty38Qm88udpPcLXVT3H1vw8iJhQvogAi7qcnVY9eTsbvmG83LxHcwS+YD3cPR6YYj2LcDy90vmmPUh74jzDm+Q9or6HvBltUD6/ezi9MwfSvBAA8D1GSr28/Z/yukrP7ztAyHu9gmEZviMHFT42AH49PTZLPfL9tDxJlwu8YlR5vV1krD3XwIK97UIBvcp9Uz5uKVw86LpQPr9nL7xfiMw8LnoXvDOX1Tycp4i8qlgDv5EQfj7paVQ9WOfJvX3n571wtxq+8cUePdiegr2f8G49C9FKPYJQTD2a+Ia80rEhvlguNr7szFo8kgwPvmK1BjzCz089KtE1vU3k+7sSG909qBn1vYBNA752qLg7+ZwMvtiEOT2zvMy7+4GzPbiX/73Ulos9J/OOPfqkKD0rUdW8","hr0BPh7aez2lugW9H9THvaIyuDqt/j07ryEGvhdI3zygOg49DdDIvSVQpDzmM9G9oVxWPTAGhr3Twty7s2wHvR4Z/D3lCoU9OzezvVZOQb6p4eq9qPlOu/o/7r0YYf+9XtBgPUYYED2HrSG+qiGhvUrJAD5NKcM9LC9rPc0JMT4u0wq+TsQXuy8bfjzJH7Q8R/AAO/WOATz00OW9OpYevhYn1b3sG5G9bBf3POA6HT20gVi8gIIHPenDCL1AzJM97VlUPXeMAj1XW3M+miwTPVTvLr6FE1M93Ru0vUSFRj2YYQe9xsICvg7SYjwT2F+9eMSiPfPwGz3dXzy9z6mfPXmbG75n0qG9/CAYvQdwdr2AzYQ593pvvTO9CT5E48U9FRw5PvYwyj10h6O9ziTFu5ul9zzYKAM+foCKu5V8tj0UaMW9eVz8vQbSZr4ARpi90tgyOt+vIr3ohZc9DQb+PDXyfT1UGoE8CXawvQxVuzqkuwy9rg7XvQHzX7zv+qI9CFpnPcz3BL1M2Q8+ZY5+u0GsCz4p/ai9O4yEvTY+OL1br+s8xMI5Pd1UET13weI+XgYZvjioSz15g2k9slYFvvzAi70OxLu9WZg0PuZ7dT1W8wC+Q+14vsEr7b0yx8M8sDmkPCBUXz16erG9pHc2vQk9Br6FjRM+pUISPvEF+73ygte8zS4HvoMemL1KT/K8IzSaPOdmAj1Am/e96uIDvrl6C76Ipzu+xznDPdTl2L2PZ9O9bSJQPAOnNz0zn586siG8vTnC7L1HFN495sd2vPAayz1uzFe9qXlIPHqz/b3S5x6+01jkvUW9d73l0fG90JQLPiB0mT0dy4k9RXO3PHhUPj0roeE9a2QSPnc4HD4Hnku+dkRnPn5EBr5zo1C8+odnvcjUxz2/wls8H/pIvtj5Rz6Yo5g9LJWSPjA/sT3oXqI9VnqFve9+rL2ibrM9l5kQPIKjt72Q9wo9SweJvQbdZLy7xTE+nQdNPcwFhr0Qm568JLWmPRZrF75PDxs9","k1wTPsVI+bxPN568+pCqvcEp3j2Dnsu9biN4PXDBFL5aRRg9qOYJvb1HCj6hgsI7DKZBvhjs4Lw7p1Y99eObvWC6IT2XFre9bFz5PWbmkTwwC8c9CLCjPnqpFL6Zpn+9231QvYfC5jtQ3n47sVMHveBH9b3KSpe9ntrJveIYN77iFTi+jQqEPTcO0T3u6AM+tMgTvhfXuz3e+2k8AQqwPcyFsD24hH49aqdQPTuvzr6N54Y9c+mfPeol/bzKEMS9gvV6vPFiT75/bbG99Qx4vuifwb5SdHS8hratPQ+j0bt0QPy5H5zsvIgG2j0+6Do+8ECxPagvbr3+Tfq9kTASPdeLlL0vVzk9owg3vrqDyT2wC808Q/W7OzsiH77xg2o+z+yAvQ2Qu7xFXlu9VP+7u92+lz1GXZ09kef9PW6xGz16yQM+yZ6Dvee/KL6B2yU+UamKPL6RzrxOVvi9R+ROPYjwMT3lf1e+gYUFPg3CS7ykqY+9Daa0PGe2nD1ZaDm+R0EgPXv8yD2Vaau8EVhhPGBLvb1oi9A98MZtPt2JVj4iXOS9CiY0vYMIjj0zn5K9x46+u591uj1G4TQ+Z9ekPam/Ab6ePF88MBgpO/5Bvb2SA5M+4/ZCPQTjuj09e4S9UwVKPe62sj23C0Y95zRUPb6yJb3LHOC7zhPdPQsIxzwN6Mu9884GPtgHP732Vga9GDwqPTXsQz1HWuO9dgOOOaJ8Gj3tc5C9GnpYvVR/p70tnSE+4X5OPRRBDb4H2rE9dubNvHLCdzvThQK+qSZEPbgRCL0nsO28m8qJPXwmtTwZ3Y09cNqTPQ0pID0j0Zk905GsvbygiT0nz5m9Y5mLvXMOSj2U+yU+vx/PPZGmKj1PpAs9XrjxvelVWz2ToD0+GE9GvWGzNL5oOUg8BkHSvSIPgL3c4eo963qQPf5po70ht9Q9JF9LPQdTTL32xbW9gKWMvRe8Sj4psfK8Q0+3vecstD1fX/m98/nFOxOcNL0BlLK9LK05vrgJVrwpkQo8","mGvAPUqnlT0axRw8XktCPVAF+71uYAo99wlDPd2BFL1A+iy+XuaDvVPfVj2F1Ea9/36Svd4+sb3M/BI9C5rguqnjgL1iog49G+MxuzrIKT77zD29KYb6vOI+Jz5RgFS9DiIgPPOdAT4xIKK985DkPCJYUL0CvuW81rJJvSdDE7zHIwO+p64svon3CbtTvbW+CSH5uxqSpT2gTf09Jx6HPMZGuT0lVCy69mxtO9+B4z3Mj6685m5PvZhHR7628oK90sQvPcNq6zxJ7fE8UWIjPaXnBL7qkaI9XokZvlNBej1OS3A9or8Pvoemqr2LHzA9KzU7vF3yg70DK7i8rL4dvo5ZbD3LSSo92X+kvMbobT0p6/09ojAAPDyTerzN2Om9534Svm4hLjykplS9CYhCvfZMlTy3/Kg9j5J0vl/LmL2H3409gMSrvJwJM72J/J69sM+rvNk2Bb6Rxy8+2t0avkT9pTwfoCY+1sBuvSZ4jTwj5eI8uZa7PQh7dT245rY7of2LPdBrCT6LGkC+LZOhPUxnXz5oN+29nb1ZO5Di5LwsJGS94owSPmReUD0UWN+9JY0WvjTtZ70A0s49rHsMPp10Nb2hhh874vZHPUdmcD2+1JG9HreEPWmBI75iKne8TwrAPC3VAT3ZIkm9ouABvuFZarzYPNa7LD4jPKbTSjy6two/vXvFvQ1upT5eJ6U+UIQsPgH5yr2sWAq+rl0HvapiFj8trVM9v5ItPTImLj+BjWi+8sQMPZl/RDzj/5A9c+YGPmGOFbqT3ZM7NM1WPnUHZbtlR+I8C8ILvs74TD3BEni9rC1LuUJvwrwTo5K7tE3Wvgb3KT1Yzc29ehgwvkrXm7yjDlW9ktOyPAyF7rw3ABC+u5EfvrGR0b5p3bc9IjCcOmoD6z11UeM88TZnPK+wSD47R609k9B2vmo+AL4gN6m9jLX1uKSvEb6vK7I+8BnxPdA+E750IBm8IxqTPlDHlL23Qhw8vlRevv+PHb7GqAA+P99nvXHwSz7RCu+9","oFWIPefPwrrSSc4+6fOVvTS/jL6efqG9Q5IGvqrKAL6iZLg+OgPXvVfNHT7kq5K8H/68PVLNkT4jBfw+kCS4vWFbYrt0qHC6EVykvpvF+b07aa69XJDYPk1yPrtAs2m8NVILvlMHEL06L/K85BVvPl/le71+Ooq8BGsUvrXskzvZI+i94j0ePwFqC76NCpA93qD4PhsjOT75dFQ+QZ1fvRx4Ez06qBo8vWS6vdtHqL4Qs6y9y/jSvYmLijvcKx69v2NbvuUAGDyK25o+cLedPn2WPr1udfy3I9z1PUNnEr3cTWA8rIYkPH2+Ez6N2Wg9qLpAux3iHD4ddtY9Hr4GPZaePDxYjT+9rI6pvXdPqb1j9gm8F3SuPtUE77zJWvW92Z+3PkndGz6PyNk8yslvvsaFSL7rqYE7zkmpPvWyMb5yUBO+oeLKvsLRu73nKjg9i92svRaBZ73PyTe+1fDWvfNZZ76Qsb69KoifPYuK+71SOSs+X/1FvVnbOL3l5w498O6jvYMl7LwgpIU9eaeLPGSyPj4bshy+bMLHvNSuGr2D0Ny+DVfVPoW3c71crI++rKCavJkkXjws4I6+O6ycPcEcTb26+Zi9VLuOPV4T3j1bs8M8BPVMPrhEub4LY3c9zYGmPY/5fr5HPQ4+EFV7Pni/pj2C6iY9gygDvlAcPLefdww/ILEHvqnU4j7+DF4+d3qePleHBjx2hHo++RMnvoPw971FmcQ8RSIJPvfBUj5WG749gMuJvRNS3z6TVUC+43SRvDHN6z4R2gE/+VETPCG/AL4do7o+ZcvlPPq687wrbvM9DR8zvfWcUT72ErU+pvTnvYPKS70CC22+HB7tvRsvcL7RwGW9c/FWPioMVz1e7hG9LiatPXZvLbzOSMm9ovhUPtDVwT55Rx2+oCjAvbB3Bz45Q5m93E3hPCybnbzaSBi9NBxDvs3XVT4tpRs9N6+avYKMhL1e6FS+GZptPNZLArwYIBG8oA4+vsV2KT6akas+LtXiPLyvKT1AglI9","k5NVvYQqcL0KY0O8nOBovZEYNr15q5M+QMDyvfLKxb7zLIQ92poyPQvt5r0YBBK97uMrPJkfTj0iOOI8thS6O4oIf70pvs095J6NPexfdb5IuqS8VKx1PeTk3Tz1DRQ+O3zoPSo+nD3bzbU9jIeVPRcmdL5AmiA9lKyivL/oEb7IKhI9jhq0PPrrYr1YhFs8/NWbvGhmSzxF/Ku9UXGCPMRyOD48PeQ9JtPjvbkAAD78P4m9YURYvdUv+jzN2Xk9uRciPZVCiT20PpO90ZVUPrINPD3P6ti90nVpPU5RcL2qVCG+RjJJvi4iqT2Zz3A+f2u0PWfJ8LxGKh+8H3cJPkGBED61Tsk7diO1PXQjxz3dLRe+365IvSjevDw5xGm9lj9SPFLqnT15Eu489IEKvkg3hj6YMyY+0yhWPgkauL0UUS696+SbvDjWeT1/OJy9U0NcvEj+6by7lpi93CHkvawqYr0nm3G8+pSbvU50qT2hNGU8wK73vNuqo71VwmK9sj4KvomyuTyUfsQ9t8ZnPceVcz0Y9s07nFiRvZJ6Ob10i2c94PbUvTGxkb3FTVm+suQ5O4m5/bwiElS8tdK3PcXS8b35yMM9x3xwPiPj3j3X4tY9JkJEPvIYNb3CFPG9s59aPTst9T2vSEq9m2OIPfqjeL3OtX0+e3NVPb9IY7wM84q9q4p3Ow2C7b1t3gM+w/1Bump4vj3YjqQ8lJoLvkrjhT4lFsE9UGfPPeAX2b1WEBa+swLRObzD4b3QWKi9vVEXPa1NZT5gxOc84JNGvs/QgbyZ9Fq9l2tpvi+iw70hYsm87JlgPQel4j3zOk89uhPOu3gaW71rAPI9+rKsvfJA07z+gpg9A4rFvbZmnT2X7Kk9PF+6PSwdCj4DNvu9eZiFPby7Nr2pt569FVEWvgTMiz60MHu7dcfOvWfNFTwxJRC+c/mbPXn+Bj7bcQ6+vkmAvVhYyT2l7YE9Mfr0Pd+kFL6G8Um+gFq5vecuILtLjcm9ldb2PZTA3L0+MSm9","a3u/Pb1YHD4cUuq9xg2SPWyaPT5cPPA9W4E5Pawqjryx4NK94pOBvXLJFr6Yo789R1G0vNHlqz2HvCW8yVfNvfdGYL1uAPy936dPPoL5YbzWUMq7sMz/vMdcxj01E6c91HnePc6Qhr2YpSq9p7pbPo+uar7NwqK9b2k5Puwvhr0+XAA7Fw8zPfMDRT3Xc6Q9mukIPTRUFjy51yS9aUMavSoC9z3lY6E9NPchPas+nzz5oxo9mw9IvYVlcz1nMLy9br/4PbHnlj1qaR2+Qm+vvbd/IL7axGE+1+lyPepR3rukmPa94M6HPA7sLj2W5C0+GVGAva9PH7zbssa9nTOwvdb9wL0X45U9LVixvRtjhzylc5Y8xMAuvjZvwD1P0Aq+mMsKvgqnPT2ptfa9V9kKvgvX673UpfY977FFPb29ebx9GxC9NlTZve1A2T2ZAvM9B1EKPg5/hb0ucuC8kK33vSxGhLyPJ069AiM5PH1HJj1RVAq+6LAXPDk/u70sBxc8dQiTOxaknb3SN1O9u2HePIa7lr2ccYC9Y9SVPe/UXD1p80w+PtSUvbi9lbxOWsM9aOiVveWPxzsfNRS+BSGHPRDhljxAJIi9UbmyvaQGsj17SAG+c5VEPL9eWj3C4iU9OPJOvMIBqj1A7Q89f3evvW/K5jtwy/A9sK1PvmLbrDw1x4G7D8GCuzEJRb2Y3LG8CO8avSxiBb1mAok+M7MlPXl577tMjow8PhKIPfSCTb3pCFs+UnaWPUDZ8zwvsdK7AbqjO8l5270tpYW9/K/cPRoOer2DbCg74Jjgvbt3fr3+jhu+wlcnvUdQ1z2D4V88IbDZPfrmsDkthAc9YBQuPHwMFz6BZaG924KIPV/O7LwpL1E9/EqAvs3CvL0hlwg+hnmYvVrv9j2qkMU9y259PkX9wr0Vef89/JSVPctxPD46FvU984T3PD3MXryrNjS8iPbfPIszB75slE696FRnPbeN+7xPkgk73YhjvVrGt72oQwm9xeCRPC9nX73RJbw9","5niUPcosMj3keaS8pRxyPWL5d712+zq+0g7SPMCFIL1p9ia9t580vl30fr0yvUG9wJoYO2AUJD2fnS++hHOJvY1AcT1UFpW9CA38veVyCL36tQ49DjtKuNYRt7sGBkE8y86ivdOwqDwQSCS+EhK9PXKB+D3NXfi9N+EwvbWohb25D9o6UioDPFrZMr6xW4e+S+rMvPM9RT2vr+O8/DN+POzOvr3Ttfo9h2OcPaviBr4m+4O8W+xtPVk6Szydrou8oCgvPIGDTrwMNRc9RgQNvl1y6b0s7zS+I+SOvcWIj73QqZ08aUBTvjXwcj1GNBy93Ys8vO4WT7zxo7G8YdCfPvwRUz2deK69RNjCva7BXby1fd69Zi/uvfXEmb3DDOk8jcTxPd9DzTwSQ7U9TUpVvg8PVr1XPKe8Hy96vrqHTLxkrJA6DiBKPSYXA77F3hQ9t6NKO5fi5T1BpBS91TErPZulCr0Iha49nDHdPWGWMb5WYpY8tYmevAWzg7wInoa8skntPD5n2r0XUjG+a2S6vYE+f737EDq9Io8oPQF07bwk6pk92g9KvaGbEj7G/lM9v9hSPLlEub0MRby5LRsZvdwuVr3EoVC9/RIvvX3ErT3pNsO8ZllsPJN4uj0eBdk8C14MPBzirTzWZQg9+jVWPU5A5z2EWKE92W4fvdYBuDoG5G++DsWrvTu8Ib5hSoY9oXrJvQzNzz0h8U0+wiDNvbHVAL7F2uw84tdIvmIhDr66MBA9HxSgO2Jzn70VFho93ku1PawoHD2suqQ9buI1vqYRVD1sY8I9X6gBvl2+2T1a28c97IIFvq1gqb15l3y9BfObvX5hsb2WPyE9svK2vJACo7zlAgq9/AQOPqf8Ir59Gmi+Goqru972eb09eLk98OH8vdKNtL1h0gQ+nK4VvbqeLrt2Wqk9SqCEvA0t+rsFJBi+9KQYPsh/Nz4aXZg8DBDLvSOogrz5/+U8u+jPvfBh5D3oVOs9jequvamNXj5utKC9JBeDvK7F3j1Yfaq9","CNgGvR/L1r1XynK+Gz9HPd/Uuz02s3q87SzTPNMDrz0DTIO8hyuUvargoDzyPEw9oO+WvQpdVTzz3s87gnmIvfQ2cj2AYQo+RB4BPtiwBrkfnJ47UAPGvd/j+j2AwgA94dTOur7mOj34JQc8mmSvPBu5dLwOCaW6j+xzvcXnYr0WTNg8SFiMveW7jL07Xb69EfvSvbDRxj2ZAAs+j+gVPIjj4r0BKwU91yjyvRvBvj2LAJu833IQvi3L773Z/RM9AnM4PRpOHT0c+1O+tcPpvSKx270fuPO9V8NLPjwykbsE6GI9uUREPBJtZ71k3rg9Z2iUvX8dGL4h5pe9eMZGvCufpT34IXE90o/gPHxzir0SpyI6ptiAvRGiEr2YQX882aNevhQZ6j3Do469Rty5u+CJTryAPl69k+wnvjOSA77BmJs9NFUQPe2xXT02D7I9HyDNvVXoAb7YHTG9BYoiPXUGe72eO8g9hMHFvdd+Ur1IGWs+guicvDaPcb0C9iC+diAYvHG4O70KBwi9gM2wvavtwTw5Cyw+JLLfvbwgGj44gVw9PcRIvobdBb0eXjI+dcAaPtTMmr3aUEM+Kk8CvkbarTxstjW+cxKBPJwzAbwtRqO9rw2avW39HL2sDzW99ORQvuOkCD+I1DY99ntmvW0w472H4Ao+Z+v8PW0Fi7lYwUm+4Kv7PfByc7z2dF09ToHBvAvYGj2K4+S8NCsbPl9ccTyt4Qu+zUr5PUxoJr60l6u858krPmgSAjzab3K9PgXUPT+7Cr7T/Ry+KD3zvaxiLz0C9sa9+7UbPizwyL2s1aq9XfyyvQrlOj3VpYW9fwyBvDqPsjyAWzK+V8IfPs/8vrwvmBu+mzfQvfJRFD3siG49yIQcvlrIib3rw5E9xg3IvfVChTy9nok9lJGavKaI8D08sR46qn0zvtGY7r2lXa47X0J/PUEGsby2NYQ9yNCFPc16uLwBcKK9F9y6PS7SMb1voG474I0TvuoUS72+xre9jT8VvAY4Cz7vLiU9","jPDpPYouqzp6gyA+c2z8vSUU0j2SaAE+XRqxPaXgE759R5U9Ew/yuxQL7T3S0bU9NjZxvYcwnb2UKhC92flgOxsZu7th5XI9UYdwvYw2Yb1li1u99QS3O81pqLywfIC9TBzMuwLxDD68YZg7kDMhPrFakryTasq9iSJLvVUoYj3MvIq9axAYPOWL/DwcZpG8NfyRPAvfPb10guA8x7GAPQKNHT5AIgI9rOEVPQNUGz1EvPc7yB6ZvPPdkr3UbMW8vjDXvYBmnb0UZuE9ZcsEPa3emr32TB09kAWiPMMF0bwCjsE9p0/UPW2B0r0XcC09AoWQPScusbw7nIg7eF8FvuW3RLyAY/I9i7uiPRp0DT1TrLs997RDO+2KcTwZggI9A1GkPRMUiDxB5Km9IwSEvSkkLLw1cpI8EHFOPZlYiD1HAwa+5dCkvRgWt70SZyi8QTNhPQoexj0kL3i9xd/dvBjxKDybj5E9Vvz1PENUA75VW4w9sLI6PQ9mGTzpLLo7bnGrPEsVcrwGiOe956h4Pf8hfz389hu+fDdOvIlPoD0sYV491l8Pve0OLjyEtR0+ZhTmO5zjjj2ScKG8nLVJPkXIhr14zSk+JAACPYq2S70iENE9aRUQPmhGTD69uKC8vGGyPOn9AT1D4BM9voKZvQxXVLv79iU+J4sqPJif6zw1Mco9ZS/OPQ5AmDx+0hc8ghpJPlcP1z0FFqY9L19xPC7K/DwquwO+LnFtPAYbkz22Rfc9n3pUO/NR9zw62ZK8PoYfvtxcyz0IaQm9Ym/Ru7YPNDtAtAq9wTsXPal47DmqjAA8cONtPHiFgT0JE6q9A8T1PWO2OzvPHwk9ShChuyMYZjzxEQI+QiWEPeAbpT3dCpk9au75PUM3sj1uZRu8PrKAvQe3r71hhnc92SMKvf7gXb0KaSg8Q+kpvpvTtD3lRNa9Gab5PXjYbb28Z6u7RU7wva1XGz54jKA9hjPJPYgJTD1mCKQ8xDNHPK1QV7zPLuA9vYlVPFYjrj3A1wE+","gbEDPgo7Yb3jd6o9U/vAPMk+i7yJ/f29ypBFPKlwzbwSjuA9SAcjPlg8aT2B6IU9/aIaPfjzl739l6y9Nz7uPOlbpT3/inI992qSvQ3YibtUpIc6CMSOPUIKzb271gk+TcEZvi33fT15ZDk8s8n4PKdUGj3nCRQ+u06+PdUdez3niom97tm+vcWMHTxfgxq+iyVBPb4NojszFQ44V7MFvg5m+D1M7ou9sVCGPX4ayr3e+aw9wVHyPbAwHz1JD2o9h1kqPJ9KFD4c7tq9xnvIPYhej7yLML49bB4/PZ3tMT1xPPg8J4HUOuiDeTyLDXi9aVjovMXBUb3T2iq9r08Avc8CbD7BZCc8UP09PmlUG77UdaS9wIyrvPdrij1KsWI+nUIJPtImyz3KDvw93J8PPmEJzL0FeZC9EYaJPEHemr0/1O09dDblPabGY70dmbE+CqF3vdh4dT4bGsA9hz8oPirogbyvPao8BAohPjOeYr3MJ0S9sBq3PLAjGD3BVd29402RPd5HMzwGoaI8okKfPDwfN71r42O7u1g6vmHFsj0/MqG9VX0NPkws373osI29TAv8Pa/EXj5e3Fu+mxSwPGfCh7wzJr+9OONQvffasT72mVk+gG1hPdKcgzuewBE+d20bu/OqCD4ewVQ8ab2HPkNM+TwCxCM9iR1+Pui/ID0hJAy9KICTvLUHgj48Z1c95+H6vY1Xoz27uFG94lBqPpdS6j3g2US94LSlPQSVXzydNQM+WyIcPle4Lj24jaa9XLGHvW4xN70KPg0+TROUva7jWj3bfo8+xvIZPQus2bwLN3e8zA9+vQLPGz4uKUs+0ZeBPUVxTT3OKfO8cBWnPRcgQTw1pPE9nZ2AvAKjGDwHPl8+0ZMmPm30iD7TYo46D+6rO9IROr6Jx0A92hvdPcEZfb143hK9fVmIvQuCIj1BOoA9gMUTvsXhQz5ZoZE9tEllvnMhUz3HVqO7b1sRvWARxbqIUJ69os1dPrRPrT1dLvQ8T35RPRrGU73X9gy9","a+1FPXbC0733c2+8lVsxPm+6gTqPflI+U3HmvX2Ahj1lkzw+ISsGveORxD0YcLs8KBaEPEruur1cWT0+/tADPuNoPb6ImzQ+FjUwvY+Jwj0aT0A9vmM8PhEaKb5G3GG9h2OLvD1dQrwes/+9fjivPSbITbt4mpY9AwiYPcMxvL0zBaA9w5QgvQ4ELb17Muu9MH2pPXalFj2IEaY9wn8svQYssr1zmD4+sx3qPBej5b35wJY9z4s7veQ+lbw8Xnw9lfvCvRTGhL2FoSw9eKoUPcCviL2rBoc8PVsmPqoaz7x6LS8+MVuiOv/d8D2OhGU+aweVvFNIGTsb2Ca+YgggvXutvj2bJAq+BP86PqjTYz0es2Q+ZAuqvD8Wjz0B7RU84qNRvrISxD0xMka+LikdPgawzzz+B4Q8/yDSPpxjg7033u290MIzPsVIST4wcA4+FlZlPUNrbT4eLve84X8NvRfGNT4V6x4+lHUePvLQfT5auUG9mniLvAuVijzraNC8KhpZvK4ogbwHqZA+P12Lvd99vb3o84w9Nh6fPF0xlb29y/I9Lw4xPiSSyDza28w5S0X/vQ9jZT0WRwA9A9BUvYtfTD1Eosm9yzofPsHmG72Jxos+2EkCvi/glz18aDm8U2OGPX2nTj2hHey9RyAaPj1rED6mUuK8H5slvTJqrD3SOR69FXfFvTXAXz7H66W9aLAXPPeXir0cA4Y+tJyqPckJYL6q9ky9gZD3PAtYM74f8NC5jMJYPb7Qh7waL4e81U1Fviqdar5RCAG6nKgcPt438D0A9jY+NM3ZvOx5Hz2txOc7/JWDPeBHhD0Vk2i8MHkYvvXm8b1Ty0O+bvIpPIW4yj2FdBW+FtFCPT821TyWFYc9PpcIvrTXjL1YXg6+vgtDPsu8hb1v1rG9lFjAvVq6/D2ijv68Cp4DvruZWL1kbjI9LdWNvTmnh77ft+28x3JvvkMdxr0UAzu9h/q5PUa7gjpPzPw8Sb4CvrvUrLyMObe6SoNavWtCDz5x6Z08","JLpQPRBmeTyv6cI9rVIwvuIv7b3fGxw9LjWTPC7gnzu0MYo9e+Vjva1P6b00fTg8eT0Lves48r1VG7m9ian5vCWaN77MseA9wlnPPYtozL0hSJS93cBBvSXKMzxEh/C96M+SO5jYFL7mSCe8SXbou6VHwr3Ehkw+y2V0vXSLozwOxi+9fKQzPc9zcj06YYM8aa/9vdjTMr7TLNK9eS4OPrEu9LuSU3y9lydZvTerljxTNHg9lGctvX5WCb5DbzG+80cFvtz95b3W4CA+avxPPfbmKL3WVRW+4bUiPpemHb1gt7+9+q0Tvb/z3bz9kMg9mnAMPGuCwj09E7m9/YcwPofxXD0UlZ2+5c24vK8+0L0HFwa+Yoi9vkiIDL3yw409xxZivu974TzS3bC9I4DuPfIgQT16us69uIdGvhLTq70a+7q8HR1FPcPIIj3TnCS+vDusvCNky7xQSwG8r5Q7vQ5g3D2119q9BPSvPNurND5OtcO+8k8jPYF9gbzejTy9szoxvsSTrb2ABx2+2okHvuj5MT5sEFS+nMZbvMf6yru2uYs9tQoevm2ACD68H9e831yyOzAF8LsxFH6+09qmPHMvt71EdQ4+IBxQPVMYnLxsgog9IlYcvsCwhr6YmUK+nUktO02XYb6yI327fIoRPaRRL71YsuU8F6XqvcWRFr6lh0e9LyVpPP31lr1AMtK99UaXuryDbb7qigC+TSFfvQhmNr4plAk+/xF9PIYLNr5OKgU9Nb5bPGNzND4XMcW8c0Ievpv8LL3Ucb09kxJFPUvR771VX2q9WtihvLvgED0RSxM904M2vnk3kD2wqyS9Rm5ePOl5Db6Y0aQ9Zh80Pnj0Gz54uoS+8wWWveV+v71+BoU9soDVvTrT2z3dygS9f3Y6vp1+uz0u00K9ZtjEvG3ZwLz6Lf28ohiPPfsmCr2iq269KlIZvjns3T26Cto9JSAkPt3fCDz/Ako9JXlhvjB+5j3SzpA7kLvpO8DkrL03Nac9o0uIPXcHMb3vAok6","jBGCvecAdb12cOS9KjQlvoyErj0u8d69wb8RvSKAjD7ZV0y+wYdtvexGHT4w29q8byoXvgP/bTzC6hu98uqkvETUeL5duDO9BZyevZlIsry38Qy90iMuvURAvD0nOq29rOGQvbj8MT5DBKQ9QdNRvYre6T2cK9q7IOaKPX5tQTyWrrM8kW+LPT0e3b0dqYK+wbeJOltBjDt69Ik9epCOvUkMwD4EmBs+MOORvdWHKb0sxQG7vOxpPaRoCr3siSU+2zVbvZ2t9zwyk4K+zw0Evmkyxr2vaIa9UaEzu49SK70pORa+576FvWR+0z0fL469vRq4PVOJTb49a9+951CMvolB1DwqY4c8kjEcvlhc67yYxnS+d2IQPv/LEb447im+SB4bPjFMl75RXVG+xM9/veOkxT3GoUU97WMXO0vbgT3a2S8+SCgyPQ4HRr5A5PG8UaKKPSJaJb3BWfi9oku/vcP40zzKMci9tyXfPYKmlr06H549CqTNvOZfVT4c3SW8ghPcO5KOjryLDQc+CMsCPaA6izyjL6e+MzcvPSU5tryv9rW7UhVcvqA9sLw+2u08gI2EPf7LhbxnMvu9Ru6uvcfglD2LSpi+Yo/svQkwEr2Wli69Gpshvo2qFL5MY7i9ix38vfp7i71x6Zq9uETPvT6N8D1TdnO9KGISPQohlLrRcxE+czlCPiP5qL43aaA++6dBvaSIU754jhe9jKpUviZ6aL7jCqS7HlUxvvabaL42Hl6+k9gpvhaK4z2RPnk+mmcVvmU2f705xK+9Vp5evdqM8jzl23E9TlWQvnzzdr5Y5mk9lesIve6IdL7r6C+9eD9ZvXoAuL1EcsC8n/fkPbVd2DxL90I+cXi0vIi8X77gOpw95/IrPA1hnz36+hq+7gaSvXrVpjwc1gS+U66IvtmcsjwIkko9tX+IPrGChr0YkEK93MZNPo8PYL31rg++eXQovIEpdT3nfhi9eRDgvUwtIj0kuB6+74UjvpW7Pz5U+ke+A81cvTJ6ZL4FkT2+","Juf6vcMRYz01QrK9W4BrvKTh6r0xEOq8z0BSPQ7GOD0v2vq77SGjvIYrsz2cCtg9IqtBPWYHvj3iHwe9tWtJvoE4hb2UbLg7WOffvL1xIz7Q2t68mmg3PT7/5b0Gfs08kX4Avm6/UL6Hn1a9V4BaPQhrUL7s88C9axdDPU6nUb0N0sE9Ov3ePTarDj48ND29nlFlvkFCv7zpH8k894itO3sIfz6hqzK+2LVjPaUN6L0np709nMpzPLFQRT5tINk8vrQ/viHqULp8Ocw9CuEdPeoIJLz0gDy+2TdmvKL+Ab7w6ZS+9jFbvWcN0LkNhpq9LY4uvUqvzT29vmC+jYggPfA7TD5evLm+vIupvW3dZb7HTku9reacvc/YbT6fLoa+ajq1Ppx2wbyTj607TADSPk5ErL3P/di+ZjsIv4ceLLzjiUo+5+rHvATnLT2EBZq+aW02PemMcLxkzHw9Pg1+vgM6rb3qWXy+pn24PXF8O75VEs48qD+/PoCB6z0Quvu8fLmFvVlSM75TEYe+FNa2PpxJiT3u4n699K+jvt+QHb6JFCu+/KHGvS20w7ymXam9od+QPvbaBD2i2VI9I6XyvKvlij7UPy0+d031vW75QL3rC2M+TuXSvXKO0b5KRNM+XMV8vYfw2TzTeaI9wXOGvruU9D0lSZI9c1GwPLXNqLpXRYg9Ep1PvuijAT0znEa+iIqout4eGj7VTNc6mcgGvhO2GD4pIwk9Tr3HPZr0WTynESw93qvxPFo0kj63Oj++RJtTvbWe5j3dhro80zDWPIxO/D2SftI9Sib7PDwJkj0aBSW++rRBvgCJvL3BxDQ+1RPePZWoUjzcx7c8lBEIPqo1UD59Ido+3nIgPlsAU70/vKk9i7rcvUeP1r2ohds9WDM7PT3sZz5JsJI86lsYPZHpy738eCM9d4GaPbxhhT02x4E7Fn98PVdzEj7lnZg9v65xvmEhlryFl6i+JSG5vT+6sD3SSoA8a3A6Pj7Wrr0S8Du+NLnPPfGFKT1+DaG+","RHz6uxqrLT6Avnw9xSAcPsNQuj2EQ5E+jLmRPCXAOb5V1Zg+cZJovrOWUL2760U+gSKfviT08TswJ7o+cp2/PLZbwr3xd569+9u0PZaCsL18mhk8EskkPgKSXb22k0m9C1yKPGwSPjzf4H49pcNmvFVyY76LjAS+gXKxvfNoPj6xhMy9JWfTvAwKUD3sbrY9zP53Po4ezb4AJTS+zHtbvlkFzj3zK1k+22zlPYf6cT7X4ic91dt4vYR7E77l6p28BT70vGqb7T1fDtK+TLBPPbZF7b3PYyo+nkIEvm/wOr3aFry91PonPmsPwr2x2nw9pTQvPkCcUT0aG629AP+rPvQrTz4Dbbq90EGIPvYsgT50/6I87whBvY93AD4uamQ9jWEMPVxSx70zlg8+xgDbPt5Nrb01pCW+V4eBPiWBqL22JBq9l70MPhPhoTzpnHq9YqOzvrY81j1rEXy+2UuYvcQkCr78jDE+1yakPc44cT49/PC9y1UtuhKUAzp97y4+YYqdvr84Ir1ZPly81bOnPTsQb76IU5M+tJlHPZWM3DsAZSQ9zXTDPo2X0L0E+f095bn3PJgZ/70LNQy9LhJSPpdyED4Rh2m9A4pWPVwtGb6ECJK8eIzLPcsMmbxoEg6+4pHfPbmYFL6fz4i9ir/EPsst3T0ZGUO+NJR4vVzB6zoNkRG9gFZHPknNFD0bNDI+lCPFO97Y5TsTAIS8T9JrPQf75DwPzNC97xmjPKQJ8b2oCqs9n7j7PZYu6z3AEqq8kMbaPVMsqLswVq29QVH6PTSo3r0u9xg9gGGVvHY/y7usZgQ+GG45PRuOhj0eVCe+NrUSvM32Nj51E5286uGXPe+f7TpBbRO+dbiwPOfuAT63/JI9CsUcPXDgqDxlt4c9zfKmPQOQiz0rUBc9heG0vCZ0a7yzwUo9fBZ4PAILwrw4Xc69Wr1evQtuNj37y4q9Whofvhxtlj04Ayc9QONMvQsVUz2I4049fjWQvQntxr0RQRw+R4gMPKYrCr6PjRm6","h3ALPav/CT3hjT69BPezvfS/nDsMGRG9q3e6vjCYkDy4et693JEdvTlcIL1Dp4Q75qsXvaepab1xqvY95u5gPTPAZbsndLu8ZPYpPknPrj1HvM06vHt6vOHoTj3zcZW9KFewPf+oyLsWL1K8YEMcvUG327sLD+c9i1IrPTPEwz1Sr/u8PXSZPbljC7viu+Y8JFk4vXGhQz5hfaC9A5GFPdHUdb23cTi+9pXKPHVOoTwrVKa9p00Tvh5gH71gHIm9QTkcPkm4oT1EdA8+4+xOvW+32T0qrbS9SY8xvvg9gD0oYCA9G+OAvKTnC7ysuv+82eyVvPN6a72Ciug9LCqeO0sv5rpctSY9oaePPM1fnr26Lw67kGgTO9QYkb2o1XM8bJSQPa99Ab7pWBq+YqdyvfF6hL5fhcc7gMw/Pexziz1ZvcG9adrcPdDilzzaphO8KtMHvVlG+T3ukE89BfJrvTG3jjzRzP69QMbMvU8+Cr7srUE9khssvYNaej14aYw9qW6LPZZmxDwqmi0+kx4BPit9LT1waRc8B02qPTNQKT2dRsa6AWpbOcbOsTy/sD8+Z1YJvvTAhjm3yyk9Xp/fPAKCSz5BAou7q2bDPNO+C76UaFm9V9PoPYppFD2t1Zq8iLFyvK4IuT79VKY99t/6PY6v1D2otho+nMbJPdDVA75aN3299pWnPC0JFj3sFm07WsSuvbg4yj3vzdy9bYgiPPMHOL6VOy69xrMPPlxTELxCSmu94WuPPUX5Eb5hEAI+tSkDPpXUVLyF4NW9urcMvAcr47xNpo29tA/hPVfuzLw8OiA+/7xDvhXGzz0yiZK9B77+PPk2kD3VBZe7SX29vWeTzL1lwOQ9jSYnPvVZrz2U5Ik90dPbPSLCTz19mJu9X3YWvmz7Cr750fk9xTYBvjH/tb3qlLM9YA52PECaeL16lma9N2E4PfMVnL2uBKK92XkOPYhnBj3pfLi8pT/5PXKCLD31XeU8XYwoPO3aED7eO1+9qx3IPE9OVT2V+Ca9","X9FcPcS2Qr7kCja9gl/bPasnk71/R6a99GnNvdpXmj00DqS9Di4ivAvziz3cOYq8ZbN3PJNMmzxTz5Y8B4mOPT1wHL2b5Vk9CGQIvRZxgr31CTW+xbRCvlHr7j2q3EK8c0hpPddVhb37cH6+kj7fPfdJk74gEaI8HLa6vSdiF73iJT48m5advutyNjwfeMM8L6TlvLCId72sqDq9hxaFvWIG2LwNAxy+2Oh7Pb/iOT6DoCs9GnGFu5kCgr7GsYg8dQmjPUAgub1gNRY+aMt+vf1SVz0HS0G+YEOfPT78hrwaFzo+g3p+PdpOEb69Lo++SgDPvUrfQruljai95CmPvHvi473eEwc+iCaaPG0aGr4p7/u9g0fhvRKQwD1nNqG9aBrGva7Nvz3v3xy9e+wYPZs+Lr1myv+78v8dPpKKk71lH6W9yrDGPfHEAz5/GYO9tX+ivVJDZL4Wm7482ll9vbBFMb2dwKY90E3nvQgsAj3LtU29OLjdvPKQy70GcHi9FCpDPFCJtb2pxZi+ADTuPIuYHr7BOY0+edIGvffaiL5lr6U9HjYrvQuyl715b6k9SqqhPRVY4r3xyqc7uCMEPi8T2z366Fm9YgryvbEaP77B18a8nfgBvUPOh72QetM9/2oBPQjb2b3V9VY8NY52vcNJ5rzBS7698oPxvZ9SND0Z2L+97wyvvpjxAT7P/5e9j8KlPXNvmTz+0+09o4C0PfLcrTyJPCa9ixjUPae4or2+j/S9bmdGPQzXtzzqR1e9+0ctPWZl4D133OS9Kw5TvRdaVL5DprK9kLQCvq8kqLx0zLQ8jjYSvhejZj4xn0K+UZnuvNlXVTzfmXq9457wvcViTb0oYDO+hOZDvoZmO70MKvw9FiOavQGCk7zCSr08sDjpvD+9qr0PUQA7xG+uvdRuZL0aqRa9QwA6vblZUTxI2AA+HoHovLPNQDzOrdE9QVSYPLWpuT3jD6Q8zxeCvhNxtT2riq++6j3kvSGOjDw69Iw986NnvptiHb1uGzQ+","ov5YvbzNS73BAnY99EBnuxS80r28BHU9HQewvfCkE74zhYW+5f3bPZ5bfryNKhy8rbpnvH3JXb0TZ7a9W/HbvKEVEL1ntvK9vt4avkuBzr1th6I8mnanPhePNT3n+Te9StN6POQTIj7UFrU9XkKHvWCbrLwpmRS+TZIcvh5AKT7LaIk9/cXAvaafkTyJWjS9EWG6vePhkj1PgRM9rZ6nvfTLDT3A35C9xENDvUPwn7zmTLA9w2ChPQoKzr2D2E+9rpEEPUL0Mr3Ypyo8pE8bvmr1N77UlfC8trtnvYa8mrwvahU9joUrvSQMmzyH4EE+SneQvBqLwD3AL6s8wufhPNhwH75r0uG9UPZMvlIJojyOKm0+ZouwvQVIJD2wvKe9MHQ0vn9dkTxlThy+WCUuvmEmX73dCNE5KE8OO/1PAL3gxfY9cJ4gPYC/Ar7k/6g9mCNzPfBdnr3NMJQ8Y1nKvcal1TxZe2o8w2mIvTwDmr0cb/89J2FHvZ5agb0XFyW+rGthvdKNpL2j5pW9jnW7vA9IqL1OB3M9PAnDuMvvpb2JTJc9BDnkvaXZF74TtO89A13ZvQexXrw2Jx4+EgCbPSElRz03/wu9gy/bvC6PWr7ArAC+mNNnvXIvi7vTuJ+9SD+CvT5l4rvEmbc9884nvT/mD73OVHc9tAQovvj7SL4UDAK8/LTFvV/L0b0CIb49vVCIvZFboD2BRTc+eooVvmkPW7wR27i9P3XwPUTqQz2YM769vo4vvlmaUD1AAfi8wNwDPo7/JD7DvA6+szocPkhDNbwryjG+kzvCvbqMPT3lLr07qXEjvl7IEL33Gxs9A5bQvAvfe72HR7u8VpClO/A8XLxqT4c8r823PWcKmD1Rvc+92sCTvWVtLrqoI5G9cKn5uraW9LybiAa8TJEMvhpVOr3pC426qnDRPRpPzbwsLAm8gkuZPetR9L2RCcA9X+zrvaVuzr3l9Ba+bLqhvA+/xrsjZIo8tXI6u3dM1zy/EyO+sxY3u0GCQL1IbvG9","c7UKvUIjCb14xIe953qKPbbNS7yEGoU8KgtKvb+H172Au+y9UjrqPIsYa7x9Iwm9974LvTiNzr0rNhI9jqCWvQtKWrzXETs+TH46PY+7tr1mMna9ugycvN/YOj6YaR2+kijyPf2WGT2iVTE+wCaYvYu+4j2JJ3u9dj5jPLI/Gb5xpzI92QgTvtqlvzlXSDG9XArZvIv+DjzvBKw835vRPaByIb31HeW8ah2jvebuhbvMvdc9lUQePQt0Wz31gGq8XeGtvPYguL0+lzu8NEhavGodEL0z+9m9xo6NvBQK17o/KYm+mOxLPnvUGb5z7wC+SNmovH7Oar03KJA8tTWuvZnvob1+ET0+j8SHPbLlPD2o2ka9g04XPh8orr1w0Dg9GH81PpowMr3ZgXI9sGfwvSAZg7vKSd496VNDOqkMuT0qZKK9um6ju0hJ5r2/gKO9SjHGvYFjCb3H9Rc+qSC2vd77hL1xrZc+BK6SvQdtT7xsQE2+wAiuPBRxoz1R+hq+Uj1hPSI5m72ebR8+QdOEvY2iCjz3qJ88cQsTvYjNmL0OwgK9vQV3vguRp71j7oQ+z3TfvRhTyj0G/FS+f6JQPcmpn709NNo8MzjzPQiiLL7LutO9MJWFO2mQirqI5Nc9GoQbvvLyrr3Pg+a9I6RJOsXGgL5plzI9mi8LPQRMvby6zmk+1qvxPI/M0jynm789jC8JPVit/DzJ0tY9aFs+vt8U5r1OaxQ+47uHPvRmxT4tGLK8CdCRPWndAD7g3dA8tWCkO5/oiT3NAwq9B4WlPcqJtzyUIBA+V1JIvPoz0DyCwuM8l6nRPZEe6r1Yu5y+Wz6YvWQZPD5Rmre9djinvZP54r04OOA7kYpIvUGb5z2F1yM8xz2QPGXpYD3OTBs+YstkPSE2D70sUnE7aNIPvWrMuT2niBc9FS6lvT8ieTvpyPa95owxvk3mDb5cn4S+NL+Eva5VjLuO3Co+GxNRPlu6lr1bJLo9vX5+vd5Lx7xXIPy8zdS8vcNNgTweFcK8","tt5RvUDXHj5AA1g+ElEqu+qyPr0UkXY8ps2dvUzWAr6mUFg9sF/CPZQbrr1yFUy98EpIPgfFLT4ZEGI+sqXxvevf072Sj/+9Gxq0PaK/x72zS9y6OD/mPX5Ao70lC/08WCKUPLSgOb28jqc97uMBvD8ubj0RQoC9dUWmvGO1oz1owAQ9liWcPv3Ihr46GGe8hYjzPGGlo72ZgZC7cLqtvUD5eb01Ndi9+B2uPdtFwr5AC0i9KmRCvRVdnr1ytyU+u4t7PbgZAL6nxBE+yGiqPe9fe7xaYjs+G6VYPs/XvT2uVHu8YUl/PDk4ej2V8BC+pvyDPfNhIz0JJyC+fABgPKe1ArwXfjw94nOpvdZPUb1GAIm9Y6+TPcHdjL345sQ9ODy6vHZvJD6+2WM+mKp1vWtehb7BVMk7MENQPqPhSL3fy1g9yRFkvUb9lr0BRNa8WfsgvCNjrr1h7lS+o1ikPXQgjjvE7hO+ITqfPcuV5j0Xm5U97dqAPKRYcz2lFbi9GTCHvbr8h7ypRTO7Co3avQSVDr3mXaA+jvaHPbvZur2iViS+gHaBPoMeC72g54W+PUklvWy07rxCsxI9KwICvrJ+ob0xIYi97uMkPrDktL2Gb1O9KmAJvRJRJT75NCq+M4uZvdneEj7lcLe9zwDFPUIUHr5vmIq9yt8IPiUA/706E1490U7xvVlgSz5l9r49/q6/vKU8Nr1+yNw9IQIpvaAzHz40SZE8uM7TPPdhJj6Px1c8p73EPIQmiz1ZG747wg4wvPaDm70fNOk9QEgsvYiETT1MH44+P+GgPbk0uTysXrA999atvgj6s74w+tI9gfv+PL+rOD1MzN89FwrEPXaQdDzBcjo5/cByPTo+Tb6TybS7LJpCvdJQYb3Z0VE8BVYyPscgjz1nk+Y8u7sYvUJE6r1BI6e8sy+rPJxmYL1t3vS8yJ9ovUV7aTxPsXE+YzODvkk6OL3ir2G81SaSvUynl7yruDQ9oaO5PXGjXr1piG09LOU0PXBxnzzHpHu9","6ZYJPawPJr3bcgM+P1iiPXXIw731OJS7Sz8xvuG8ujtfKg0+MxlZvfUF3D2V0gY+qU3LvHMXkT2ISfq9W4KxPOf6VT1AXdS8IMFoPbqZpT0VfWY9PzdePalDEjoE2ha+AIzlvaHYVL3UxFS461sPPex6vr2odIO9PyrJve1l/TwW2qa97B58vILGEz1ETOi9n8+qPciBeT1Ux2+9cI1VPfD+QTwuYAq9ZwOYveWRM72Xrtw8v2yIvbN0EL4lq5g8z9k4PSl1r71m/b29FfGgPSllKb2CKZU9mZXavbQRZz36z629gCzivcyp6jxN9uy8bGF+PQ68Ar7Zhka93I9kvXqLSb2tiRU98i6dvJLzCL5PGG29A2yqPY0cPr7Lzxm+YT6jvF3vPr0CeNW940TSvIGTjztxUCM+gDepPHz8jT1w48I9YwqJPfmWSr77RW29ezhYPaZdkzzy7rI8+5ByvbPsnr1NZgy8Yu1YPGRbeL2IwCy9+j48PGgEvzy63q89ANEBvufaf72VRak9BNUHPLyfWT0wgqM9KLixui0XiT1BlpC8BhcKvle5p7x09ZG8R2DoPBDZ1L3zk469MmlOvYa93b0dCbw8DIX8O6eRWzzJr0u6bdAAPuCEwT3sDlg8gQywPbcSibxIyDW9fBXyvcEkcb3C+Ey+yoB1vYZmBT4ZX0M8DOawvXdPy7xYkqm6IX+lPDRZ/LzLkMy8uq6ZvbOFtbqMrhW+bd5/PciYpz2xmTm+zlSEvANYgz1u8Yo9vKL8vXhELj2e9qY9YIRxvf1biL1/U0m6kTDOvU2JE709ICy9XTHZPOYK+jolyQK8ktjMvSMN3LzpIKa96Py6PdbZRL1WwLs8WCNDPa0pGL5NUKG9Kl2QPchiaT3Kypw8O5vcORQLKD2x9My9Pjv9Pf1Djr1yn/88FukgPSl9Ur05UhU9aDd7vWKBrL3sq168mFArvZRdGb7cPOk9OIqOvDv7lr0c+mi9N4FlPP0DsTwTbuW9wOSZPAG6njzGlWI9","NvK8vLSnQL1L65A925/2veEVwDwRBlk8zbCXuxWNYrwEOli+oQtXvV7mB71874W8Osk0O6JPcD3HHUg7gfx+uwxrnT12iyg9F+4eu6eAQTz3jBW+qJJkvV7Gl70m+4e89wCxPS/yDT6Mo+e9F+4oPYE//7yqSjy95T+0PAA2/zxjaHG9BHMkPR7Q+D1258q9ivFyO3hKCTyDutG7nKwBvWnH5Lw6oay881QpPXAKS739eF09Ub28vGWOxj0UDcC8J0mDvPV/wb0BmUA+iRkRPV1cMb5Nc6m8L/cMvVEACr7ybJG8WXAWPXqCBD0qOcu8BYlCvVDWDj2Hym49UxqKPcyzSrzhzqA8TuDhvXoxgLs6YQS+x266vLw5h7zUWcu9pEMNPhpRIb6LOGG8gK98OwuUBj3cKN89BxSFPW2abr07ivk9uv5hPXDVsbwQRCY+MtoAvZp0SrxHSqO9nKtkPRY+sD1JtY65ClAVPZZHlLx3x208HySwPUUKKzyA2tY9q4f8OytjhT3KuOM9FUWBPeVF07xymT09xCCBPWKGqL05wpc82yHrvZq4aj0bBQm9ilxaveuPxr0mYoG9QY+4PV3Hc7yfHFU8AwqSPRhqaj7FgNs9HqeGPQfJpT14MWQ90ZyAPZXRLz5P18W92iKYPrAViL1efGS9+slNPW01Gr7Plok81kq2ve/ShT29HaE9NHq0vSI8rz2zGpg9mUF8vAHTrrxK4f09bsW5Pf+aBr0MCR++stVOvSQY/LxKQIW9TFFsPTIfkDx/7KI+6udAvfOsN70FndY9gDIDvjqXGL4Xosm9ullIPDwAeT1QEcI8Q0EYPPkzs7wKiWo8nKYovMPVFD1WPTg9y31xPYm5yTxhvek9t0SzPZOQmbyxsF69jpBIPexF7j2LqIg8TMtzPnHx5zyelAS+fmaYuaSrBb59zSW9hRAcPfuBpr0daas9WRX0vcDsCT5IYOK8jgIAPZZHqD1Cdx2+5liCPPR5tj0bAKQ9Obd/PRCY0D0tJuY9","cgkUvmb00r0hc948cjQjvamjJTxPoAO8qs/FPM7itj1N9AO+s/4GPgtfiLv00zC8bJTxPezrLzuvV8Q9Mr0OPsHDkbyb+ie9tPcZPZLsvD3Idwu+wBusvSbIo7zruoq9rIC+vWQ5mDx/zvo9dN0XPf61+rzLVJ495SNnvZe5Er7yrzA9pXaEvKgbEb1KPaw9mpGMvJyfQ76OnF09rDipPWo1Gz02YJa9U6mFPJCo/b3EHgc+uLxqvXLXbD6mZto76lTuPOeJijxUGu89Y4ZDvdsZKDyNtMy9l6KqPZevkT3NDPU7InBaPevLGb0ZC4a8HrGivTo4iLxs4AM7RramPWI3Cb59JMo8aTDfPcYpdjwOLYs9QBo8vpeEXTwxAn47K0JavC90hLvtluc9/iNrPTpxibv7yIM84bwlPg1awT03e5c9LZ5fvf2bMLyZ2PG8ws9FvLGXCj6yW0I+w8eCPQDViLyZ0iq91RVYPKiyJL2NeoK8WDwkvS3cBr53L5m9NnGuvMxUdTzT/CC8X2Bavc2Nuz1gn969bjeMvbhb8rzKAQI9qLPyPEapwDzFine+DurTPAQolj2yEXs9QFMXu7HozT2FuUu9QlmOvp2Prr3r65O73v6lPUYPv7zVM7O9F30FPkJa9j2Y6Lk8oh/7PVwucTyAZxY7qb9EPRoxkj3zaAK+XSYIvXVl1T13+Da8gqTOPJVxgT2o+oA+k3YwPnu+PDz8HaC8LbWGvWVZK77tnh4+/tOuvVpfI73f9H89fSXcvWRcVL2k8la96L+OPj62qb3wOpK9kAoEvnv+YDxfHqM9J5s6Pk2TIz1+VoO9E7UHvNtDBDp+7e69lpM0vap/dL1z6qO9xPMgvM11pby3YXM9f1ilPLY+RTzgYfe92BSBvZbxGz1S3fK8u+jWve+zsj3674k9sQwjPqoUDb182I29FbrOPZrPA74NoSs+fsm7ug8Snb15qf680b0CPkJCjr3iZAw+nt7lvbToyLvv3V09N0VVvdeKIT2mrS6+","0EbRPTT6Lz6LfAo8HiOGPWnh8b0jsGu9r6X1vWR937wYJgW9BJowvNeW1D3rLpQ9ffQBvi1rOr2jc4y9fh8APr5KD70X54M9+O/zvOXbPb1WvEq9uhgEPtbvUD0xNqU9DfzZvdVF+z1PoMO9l0GXPYrJSryIkJ294ecBPVKhh70Wcv29pjAlvVGxNz7ed9q8DSVcvqIoEDwQNqI96N8jPvhOFj3221M+OqFeveW/bj6tMP49+jCzO+OIwr2/RYK+CfAiveNMiD0TCUa9TZzxvUWl0TxM6Ae+qU2KPL/nRT1G0KO8JHICvrZiTL0OyZg9RqoLPpCpBb7F5js8x1QJvdeYAL5jUD+8u+Y5OseXw70Y7989/V50PX9xhzzgitm9IjMnPSx8Az4uheG8Mz7ivNf3FD0RLjs9B1hjvjGmvz00N6I6lHkFvXfVwz2+66w9vNOvu2Hzsz2J3pc9zWhkPG68Gb60Oec9slJ0PDHk8r2o5Ac+LdpNvdC8uDpnbTc8hbGDvQ/TlrwaDou9SmOMvX8Elr3UeWW9yL9zvUjXiryRDw+7YyRgvZzwg73wX769lvODvVPFcL2D+Mk+mFiRPIye0j1i/5o8mhbLvSw5sj0nnK0863HmvU/Dvb0DMgm+Mx3rPLI1Qj6jeUi938z6PYUBG7799QK9TVXrPdqVHb78RdW9sxEMPs2fRb3LCbq9L4lovQuH+DwEiAU9QRaDPdPeDT76Ta08Tmx5vXi0f72Bp3k9dh6wvWuMbT002528QSM0PWgKAj4ixGW+w9N1PRSWlT066ig+i2b/vQndg7u6IU8+3Sq0PdGNBT35vTy+Cme4PSkbVb5cJM49BWdSvXE0oruYHkM+Qc2gPdxWRb3CcIQ8hhvzvZ1Vib1uwAa+aTtoPY1sDr0o09A76wUEPcBktTkjo8a9rXWkveVGDr16r0M8fYotPVLr3z1arGS+n85NvbJzAL32ATU8l1XfvMpmTbkYte29MV2BvXHI0L1QmSo9u66EPJKZQj3kMm29","e3oGPTd6yDqnt46+qKszPQd7ZD7XPji+lg/KPb+BeT2zrhI+XFtEvsio7j1RwhK9bLkIvuYrvjv5aee8GlewPUiE/7zqSqq9DfYUvicMCr6NmWG8a0sYPaNaczyVDb08MUr/PEyK3j0RKTM+3FMmvkp2CD2be1m9BTY9vbStDb4g74U82cqqvu8IAz12h6++JqYaPeBFXzx7gwC+PrQ+vpyNB74ZrXc+anSrvRWotbwBS409T5+nPTHVv73cpS++ApFDvd84FL0sii89eRHvPYUBnr0ymVq+YARPvfdrUz34FZw9SbI/vnZKKb0C8cy9Yzj1PQ+kgL3Tl0m+zRdbvbRyhT4nxw2+/amJvBDHrz0s9Ji+DKeWPoWcA71wLs291YcSvDIU2bxsOo69OLsQvk2qFLsW9Ks9HS4WPaj1Y71Ib8i9N4HKvefNyjxRj0q+yw6OPTCmQLwmeEm9gjCyvNR1AL5ZIw49GTsbPOcl3jzxnKe9VBWNPO71jD6ii4w6XTGavdMKCL75QUe+vRv5vRo+kr2T39y+B8fgvbWdsL3bdE09QtZbvlz6hz3Co66+Z8CbPZGCQD38PRu+AML7Pf69j7z5Iyy+Da95vC8/2r2+1ye+9cmtvRMy3D3KaZC9TV8GPXkpHj4CVNE9hVBrvQ4uwb0fSlc8CqHDvWNhFb7NZqA8v+h7vWBYq76Qsu68kveMPbPgTz2jlxA9ac6qvAITFD1fjPa9726FvjpdiLvhBj+8D9NtvTXYqz06tSm+U88xvlex1r1QVh08GgKAvXHld7yKYgy+FZXRvisQjzvBKQA9si6ZviXpAb3ihs6+LcEjvqy5zL1XSVy91pV9vq9pGL2LAI4+IpPJvXiLFD68HtW9hMDiPSmrGb01oIC+abhSvZpWwLxutqQ9nyqFvh+Yyr0SStS9SMcAv1Tzgb18Qy890+p+vWC32Dzo+IA+iPHmvR6/Dz3520i9SClMvKDge73Zxk0+zw4qPtdOGzpzIBK+AkpVvnA3yTxSJQw9","hPEvvhfdi75/VcG9vXwZvTbV871SbgC9AQOfvSU2trxLCp88/vUBPQFydr7JzLS8Ce/IvJpWnT22fqw95jt9upNImL25rrq7SuDyvAW8cz5fv7Y8NCqHPQOFg77cpXq9baOtvcYEY74ViKe++A8rPPUVSr4fuNO990iKPCfvOj3pSXy+e1+6vvltIr4lJk+8i2/EPXlGZr1suAg83HL9vADK+T2D4Io+05TMPEnpOzul7B69j3IyPmML/z2oJRm9wRPmvQuAEL1rOi2+CcJXvswS572gfEi+RwLnu0hTz7x7nWe+LxOYvrVApjtKsyk99l7uve70mT3Tg4695yPuvWwGM723+jc+yyFSPrOC6j3kmFC9NNZTvafxDz6Qa34+CW3+vVcfVb3HRk69igWmvBDlKzzDCfU99+txPOIetrzp5PC9wHfHPStQCr4TpJE9EavivHsuoTwCbLS9SdkVPKFy/DxtOYw+wsHavY96Ob2vaCK8yvi7vC5Mmb2MShe+JJYsPVy147zuJNg85UDPvPXMKT0bIry8S+idvU6ifD0od5+8OuwPPiI/qb0wNmS9rCkrvfT5Pj2e/YY9FvQbPczm6T3oCfW97QK0vpIrij1QV2e9iJvqvQUNYj26P/K8lvWBvKhZRz0B55e92Vcjv57HV70Cl6Q8j6KxvX1rsL2Zelq9TssSPQpHkrxgwsO93FbmPZvpE70sOQ6/wlLSvfnKo7yba9A9sWlaPqXnaj22mRi94UUwPQgEpDycOtO84I5+vLzP0b0BT4U+6ia0PLyW/z0gtLg9rJGpPReuML23bvA8Gn2mvKjjV71UEVA9AUNDvqQX2zwLOoi9/RzHPZfkfL0vdvg9tFPuvbJr1T3HkKG9hdmivPVwGj232ba8OAU3Pmvht71Yr4Y9Qo+FvW7qTb7kxYS8mr2mvUhBa77dEEs7e3KVvZPPMb6agMc85j/dvTcoPLyWySg+zzitPRr4sb3P72i+u5Jqvan2Cj2zdAk+xpODvftBm71X0nc9","fKwxPT98SDvJi3S83ByQvQFifr3G3k2+IqE7vG5QBj5MtZa81LyAvOTBKj5p75892HwNvj5SxD2SAY28GkQevRVE8TyaFx29KIjYvXOcMTyv4o47VVSJPUdoGz51ams8GYSrve4q5r0YXwa+HMA5vZaQQL6eF467HnLkvfnPpT3TvdM9I9oEvll82D3K0My+fzOHPfUfAr5G56w9cBRVvFJALL57OBQ9lAASvoWucLxVogy/0OYovQrx3b2SgJg8kvzOPf9JKb1HUPU8+8QPPldV3DzM+h694+E2Put0ML54NBY+pDc2vt/DFb7IYW08RKKBvbp8hL4anY47+V16PuEEoj20yRg9ij3ZOxZC4T288F6+N7E7vk1qSr23OA6+EtUHPmdrwD31YNO8XEHhvcZqgT0fE0k9oxcTPqOzOj1E27U97hiIPWSWxz39WNi8AvqCPA2nrjyjK7M7BykKPv1AkTmtecs9KgsVOkNsBT70Fh0+Yj/mPJmxIL6ATLS+ZUVnPVXkkDy3tjE+WtU5vnZQwj3eeKs8eJQGvQdPFT3ltCK9DjK7PZZeRrqC/AK+wWXvvVp+oj0ImaS93L9gvZ9IML1XSWq9mhW/vr0Mhb1kbfO1+RyEvlt9Az1Rv4A9wzPFvA7ZLrsuOSQ+fkUBPrmYor1yaTY+PDELve5HNrsrF1Q5QncCvT/MjDw01ge+Js5IvQEkTr6QjQs9MfXXPT8ZLD5o/7q9zanFvM2Zuz2cyJA9S1xDvJzTxDz51A697tmwvQQhaL5si429s/eRvkTfmb1ulRg7LGFBvqZ2Wb35MhW5Vif2Ohqr0T1sHms90J16uqvTHT0b41W9WpMTPga8mb2gZF0+kqruvLwCNr15sbs9Ss8BvOKL1D12gSo9eGztvYcwFr6Ve8c9DiHkPX80Ab5jSV29iAdLPSTzdrxOZX894/ilPc68Er10Q3U9znXUvJwJ1j0/kam9ZRAkvrYqtz1bSUY9dD/QulLKmD0pps69GMe7PaUugzzHdmY8","6nD6PQivOT1LsVI8qnk/vcXnNj4CxaM9+Dk+vRe8Ib2izIC9cW9bvSh+Xr3zK7s8N4SKvV9Gxz2RTUm9DhVwPZDTqTxPR8A9xN+rvYwfobslo9I89qKovXVrkD2eVWE9gyQ+viCKZT1WDAI9V8b7uiXKAz3R1ZA9ugmRPUcGar0TT3u96w0lvV9PtTx8dHk9A7YZvhUt7j3vCiU+Q1W+Pb8Hcj3ZLqU9bC42vGj+z70Web+8RaGMvSylvbygx6g7NbwxPZfjAD7LM+m8cGf4vdMsaT3benM9CwWuvfSmbL3vPWe90EswuxUWFD4PqOq9NM4cPR12b717sNY9u83ZO1Q4sj1q4Ls91663PDOq171S1wC+3FSTPYeAjztSkS28BqWBPQJ//b2vpnm9si2GPm6Ttz0XJHi89bA4PcgOTT1WzR299p6WvoK6Pr2sY/w9wIJwPbMfujsUVCY94DcXPCSaAb1D7A2+XSICPJAE37uNqqc91NiCPY8dgjtHRdI94POqvfO5MT1lKQS98T/vPFEXyz1biKg9++GsPJo1571bxqm888MvPQ0nJT5l2nA9ZJo2PYGk5r3NQcu9O20pvH3fJzuoDte7qzCGvNKQBb49G3+9V6MHvhG76r3Z2XU9LOESu3wH8zxiJS28fTHRu7i6HjyRMaI982KQvKt67b1tVSg9NnPGvef/r73813I9a9KVPVU5gr1y94o7U82APALBgT6vPZq8EANDuxHvtj07COS8fWMevT/pnT1WgYi9A5l/PR096zw9v5w99PRdvaub7Dw+HK29Xn3TvZlwxrw4dZ89fFYcPSME6zwez7u8NJDmPL69LjwkiRq9rakQPfx51Lx4GVS+Ip90vZruAD2mG2s9ELpqvXyEizxc8R49NEgNvqFcLL3Arc69TPdHvkULYL0cFgM98aIKPWgPmT2eiqe7bfWTvZPOML5393u9sdRCPQaxw703Lym9AVw+vUR/3Txw4dO8HgKOPa0ILj6HoWg9iEYCvInjjj24cdW8","STbMvTNa/z3YVQ8+oYxUPVYBTb0kI9E9rRr+vS4J/jyj4029TLWEPnchFD6oZj89k99IPSBNIj3EgGI9ooQAvUshsD36Vec9iaLVvKzedL6gjwi+fmc7vXT6r7qg2KW84Q/OvTJCg72KBXK8lbSbPo5uGL4RrrW9n8O8vJ3a2j1DA9c9c7MSvrFYvDyppsc8VjiEvZSOAL0HSy4+v2q/vZtJ+7qmETS7oDvSvELDoTyvOXO9hzqUvBJ2G77DFt+9IXSbu5mOOj3u7sm8LAeTO2AKTj4ym7U9UUuNveVVoTzr78M99425vLGBAj6zBo290GZ+PTP0OLypoc08xnKKPR5Nrb0VNl+6eWsBvjBI2b2FgbI9aVfKvTILD77PDIw8oKqEvch8hTx21Mi8wdTlOpBPo73ouXG9J5TyPVpAk73YEoW9qw9fPGTIj73D43+9qyLAPAAJ+D2UCU+9xglIvd2t6j0i8/Q9x85PvT4WYryjBV29j/nrvb4Lgr2KtmE9aiAZvlW4vTyQ/8m9AUMevcdXwj3tv8s90CoWPqcnJr3wifq9nhJbvrBw/T0qfFq83ettPv4omD2Pe3Y9ETQXO/Z/E76zVau98ZyrvWvbB70AVWI9ntsvvtcldj5Tu5g9fvfYvYirErgwjD6+2E4pvRD7JD4FYWU+X8w9vSUWkz1hU0I9WtNyPd8MZ7tocNw93FSHPd5SKr1KJmc984jdPYYjIj63XCk6DXIivtDs6T0xzA6+CCGhPUHBxj2yr66955XCPa1Jbj06ivs8J5e9vb7Nkr3dU829IheYva53DD4CEE29iupFO1dmKbw5WhW+St9xPGiMgz0dkOg6rn5cveUVmjzy0xQ95E8Zvn/dAD7s/NW8EsIAPQb8Z7u+Dy495rylvWvWvL2C97Q88Z4IPhYajD1CgaA9trWBvUT9+LtUPXI9j2bJvelzZjy6hh89XqWrO63MlzzzXhM+ouusu+94Db0EkU49+ScSPuy0/LwVL6C990/hPf7GAT31dZi+","l2lRPSsvCD5YUH68cERZPV8gl7whwc09tlIXPdrp27rHljI8VU7OPNhbAr5RmTK9FpuDvWjigz3W9Vc9YJBDvdA49j0sr5K9ihc/PS6yhb24B5a9DaC5PclywL3muqq8twuBPYGD2jxq7zA+unHevTb4zD1ZMdg9YoI2PEnj6b2tREK+BwTKu51c6zt0+4o9vx4APuLa8Ty3LvO8ydGOvcWaMz2VVOk9c3UJPrYmOT2vEBc+8p7ZvFYigL06c6S8iRy4PcCkAT4rPA4++rqdvWUe/j2gqjk9HnTevXUz5b3/NTs+URwfvewnDT3wxhy98eF5vfSKm7yIPjC9qd9xO8ZtBT70I/m9xGdMu1SB2b0m3iO+CiMzPgktkL2ifI287JqTPNH5yj0jROC8y+3GPb+wVj7kWsS8KGopvf+Qjz0cKgs9qq/LvD8kt7tB4BE9RgfMOoKAKL2orc08v2lyvWvLPD0gzRu9xpAjvnb37r1+Wn69wbDmPfp237zL/Tc9dAsIPv1LFL03WrK9sTEJPhNkYL6OIo88l3yruyvTFb1g0fa9OKfKvS1uFz7NGc88cr8rvWwZ4L1U6SI+bBmvPSMiPz0JWIY99M5QPkECdr79K/g94STKPNHatz0EMmM90iK5PakKf76xWZy9fau4PSx7G7wIBjQ8llKXPKhrhr1Gmx2+fS/ZPYYUm70HwJ68wtVAvgAPAr19xoo9YLSTPAdhfzzYbA+97iIJPgxrF72sdWc9+nvvPcfQL72uhKs9joVKPGSEdb2eB8u9oNINvaPJaTzkgDG+kpsJPTWbXz121ou9IXQevaGltDxHmP28hP+mPXKS2zye5xG+4wTavQXX7Lx9bLw9lI2NPZyQi70WagK9af+lPXOjpzz15o89O3EUvYTz0T0UZl88OB7ivUXcgDuyuz0+zD4lPi9+zz3luTQ8O/iivUCO771vtQg9MDUsPr1ibz2L9AG+qM8TPnNNLL78Q7m9P7BgvdD+Mz4KNhs93Ub/PQpmEb31gAE9","sxMzuxISYr0IrTo8x8VLvXBYvz1hLP69QZtOu/K5Mz6vxTy9J2lWPY5RDb7sVzu+xD9UPY2YED3UQfE9cgUwvvlnzz3O6Ga+lqIZvbnM1TyVmnk9zn0AviwAaj7c+9w9PVvrvYLHnb0NJYs+hj7yPf1n2z3RiFo9VUqgPXj6Kz4MK0k8vUKpvYi3hD1CWIK+h6kuPXVxur0BMq87zy+wPBlfGr3pym87WjsTPsdjm71W2b09G5R+vPT2Tj6ZrNK72FENvWru/b0xl5w8rDxwPEPeKD06FLc84hoOvplfnr0Ra6m91+gVvByV2Tv0yyw99eM1O9PeCL1++sC9esuOvrrICz0KWRq9HFCDPXlpUjwfZCM+FssAvYI/Tby77ge+QNDOvbALzz3dWBA+4f8sPe2zt7xXq3g9swYMvssrtj1ETE6+HTK9PXZQmr5tjM692NN9vT1HGT4/1eU947caPh2BpDqsl2A+861Nvj+MmD4pq/o9WgaiPcwxCb7wJlc+ik2mOx4B8L3DORO+yASUPVka4bs1OOy8+0RiPZQgRzw270S+LEsvvKJNHLy2pPo9SloTuag2jLyP4im+8JaGOytt5T23phM+3pILPjw+Tr1FICO8sEeTvkerXb0EmZ086k+cPXyj3juh/7O9HnAMvbmCuDxqWv28wikAPdNtTb3gsnw96fljvah9Zr01yhK9wUOFvY37Zz0Zs7c9vjGAPKeD972sSBA+E19hPXQBKz7mpBO+JRyJPSCjwjy69yI565kmPWZr+zx4Dzq8/FFsvrHFpz0/4dc8yC4RvakdLL7akgs9/mraO2lBeLx05zM+3Bm1vWR2Kb3bR969jFxIvM8VuLxZKpe9mEcHPtA1gb3k+x49uu5yPEXmgb0fqPg8AgrPPGgFgb0v8K894GJNvYDQ+z2OpVg+85icO+JZzDx+VXm8KzdZvSs3uL0a3Do+CTYjPpGL8D1vPo09/vXIPZV++D0excg949VWPY2SCr0E2W291PQHPvs2NL1qXgK9","WY3gPbkMFD12vby9iDX2PFGZDLzczoO8l2OOPaEkBD6U0bQ7auuPPUv/jbzyE9e8sLJDvd8ULr0xntQ9/pHPvBAEqT2qKEe6hQsBvHh/ET2pUr68CfqLPROtQb3EcCu6O04IPoPou73C1HI63LAJPfh22D3O/4a9XNaSvMXGyb3LYtA9eTQGu9cl8jt8WbW9Rg8DvYhsrD1YGG88oIOTvSaNtzwKu7q9hjL7vAl0BzxaLBW986kFPRaiaD3/IdM8y6Q5PgiJ+zzwmCq9C5qNPPuVFL6cymy9XmORvZyA8zvbLqO8XEojPbxgnD3oCC29EjsdvEYwELvDCiY9pUpQPYe8RT3dgoI8b/h+vRtkdLwgCoc9/LAUPi3UAT0FiEO9U5ztPYdjEz3hCas9W6vUvdEVvbzhLC07VNnovWtlSr2vmoG9vgCnvU+BIb0JUTg98VhOvIZIrD0mK2u9irPEvEEiED4GuWk9ubIrvcndhT5D8hk+aJCqPHOP37y0Lx69IiYaPHSwYb2KLxg+lnITPkpx8jw2zi2+M1GUPVpkyT1TUMU8NPGsPT7DdLyK/iS9I5aIPE0rB73pVCS9ye5HvZQGwLwImxs94lUgPZ8RUjl2u/m8MZSyPe0sST3xL/s9ng/gvVxd2j2GYHy9/xVVPXk5wL3L2Zw822A3vZ9kVj6xkOa9rFQOvRCEZTyL5le9YSG2vAHMu7x4EI48qYprPCLuSr5gTdU7cBpyvZRRVL28xJI9NRn0vSmLpr37EXo91o2YPDQa6rsAC/M85jRhu6HjqT1AZtw8nG0ePjTjCb2q0Uk92XMYPQ7nN77/6xw+fwKCuyV8Gr3fVV29FAXePexD3jzbdfu95twiPXX5Nr2mNEY9pj1pvH6dyDxZrn496HclPpCI1r2e8ZY8njNxvi73hr3WzxI8dO5FvWF4MzyVOlg9wT9+PXCPTz2DVZq6Azg3vnJK+r0dVuk8B5gDvOxFcb3S7E29ABmbPCOfbz1gJTA8DFnYvQW16zwpsyi9","zLEOvY2y/D1TZ729+eEwPiAJobxpgiu+tz4oPrZyHz1izGK9blI3PYeVXD18wLO9BL4HvCiZbLwIPFq9q9XKvQvwCT1dgsi94Bo/vd9zQr6y3ic9YjJsvT1rKz2Qn4O9vVt2vD9nUj02Hx49OOHpvMepUj1uOKC93zSwvUUt2T1D6oc8zxsQPWJqVDzAYxY+DAAAvbqiTD3BOhi9/rFrvXsJoLsgOA09xNIUPYXJZ702t6U935YvPuGhPL2kNSQ9URi8vUiYgzuNih8+ale4vaaCOj6fjpc5FsnrPSOZjjxHH189ypENPpIjB74BzrA7m3E/PHjLt71HeRa9PjJcPZbPAb6++uk8XBtUvs7vV73sghY+71wAPBMUO75HeBI9TcLjPOYhBD08xti8+2J2vfR23r0m+Pa8XTkvPvQMJT3SfHi72uJavWk/RT4ATb67WrYAvRlTiD2Ac7k9zc5oPIFjQL2ihzw+Af6FvTI75T0zipU9A2ICvJ7QiL14nlE8STofPk/AhDyT4gg+jW6rPGCUkL3Jsl8+NRTYPZYmhL4gu3s9D+2uPS4M6TpSaR2+6SsHvs2yAL6RFOA94FWuPnYASTzKsSW9kn0gPFaWOb3Lz0O9jRmJvcdDR71Lzwe9KZC1PMi4Hb6F35q9FROXPZneAz0hHqA9FMwNPouPbDsaZUg8laqEvU3LZr0UY/i8mF5FPaE/1z0uO4e8Ch2CvbzhMz5bJkc8b/dMvavUmD3QxmK9NY3lPL5Jor05njK+qnjRPWfu+b3Kb8S9aKF4vQJy9T2x/he+PK+tvc3XTb1pzmC+gbzfvfFNmLwqCxy9Ao/5uCJ3Fr2E2vS8cSssPm0lib0zuxc9wE+vPesjRT2DWV49Jd1LPMBi+rpROvU9IlO8PVQ/Eb0Yolm9WWynPV1dKbyn4Iy9JmpFvselvr3e0CO8dY4SPGFdUbw/vJ08ajZdvTbUqD0/+N69YQETvh2HLj4Rjzw+JG4PPcjq9zxJlDY+blWAvZSwrD3JcV49","e9HlvYhslL33EI66PzIgPQUvqz3kM2S9Ze0pPK8I5Tw/UUY85xiiveL6F76tzzu7pJv2uwUcA75OXYK8z5IyPkw0NT0Myz2+1QDBvYKRMzweYgW+Uf67vapbHb35N5k8SnqBO5UYg74TRME7zfy+vdgRpT1LvSo+YkbVvVVVAL2a9Wy9RXZmvWellTvftvA9r2RYvYr/jbx8c5i9PISgPYCBXTqbjpg9NiXFvcWnK735Pbi80NpSvM8tiDxsoFW9Lh8gu6OT7jvhTr09oSOJPV4/5b27/co9OFC/u4jx8z3nlQC+41prPXYKWL3WYue9SnJfvTVMCz3O8sC81s6DvfdMSz2EQzu9rYb4vQWtgzx3UC2+n0dlPWMkAD5qxjy+Qim7Pds3Vr2u29a9SD6LvVXslT0LgjA+op0qvSF94Ltr6Is9IMBVvfYtHj15qLo9N68uPAxLQr3utjS8rJ9JvRYPpD3x5fC8H+XAPQ2rQD48g0Q8S9nsvVPloT2EhmI9RMIyPfbQUz6Bpga9bb5DPTDlkz3MXh+8Ki+hPfqwlL0U6lK9AVYtPV0EDzybxHQ9nZ2GPR97Az2zOg0+RfKZPYgc3j01wfO9y+PQvXnrHD0DER08KMB7u+qoObuB4C093xfHvBjwHj4BrQU+l7zBPT5Sa7xG+6E9XH7XvQZn+Tz2Nzw9POPbvNAaDD2F1XE9iLaxPU0O0LsZt5g+vFhaPfoYhr0xKcm8Yyjsu8Ezn72vnMc9ydqlvbX70rzWY1I9QZyFPEl+1z2uOvM9hjHxPMQuCL073lG96mNYPh5cFj1qnM09UB8qvY0kxr2bPvg92OsEPc8SLz6GGeI91dLjvMXROj1Ge4e9m3LXu3cC7bxXfRG+4+JtPWoFRj3jz+Y9/8jKPclyKT48L5U9Qkk/vs6zhD1m9cI4PEjRvTfztzzFPZA9Pfh3vKny1z0NcDG9wsnhPdOBh7vRIie+ZLKzvEbZkL1DbAg+h/iuvQeL7zzU2vU977ARPkdasb2NC7o8","MbS5PIDOCr6kY1g8H5jLPc/G9zxoMKu94+F1vafLIz00tZE8tleaPWJSST0Y6LU9pF3DvTySFb1MhXq9Ccf6PFRU3j1tF8i81YVkvCY6QD3NwKo8/yfzPR435z2+GWM944IaPfLvgTyF79M8UgGjPReAob3TE7K9HqTnPCLHmTwdnJ08QNeFvGIkQzyx2mA9UQ7FPcg/jb0TCpe90zv0O+O61T2GGWU92VuyvC2+ij13Loa9WdAMPYM7ozwMc4k8BXuJvRbeCr0Pt3+9tduKvUlhS7xLpZ08ISkaPUyqpDw4kDI9sTI2PfFL3z05xys9UKERPcr8sb09eqY9o5TuPTXFeT3RnII9KaU9Oprhcj13tou9H/qyPVew/zwcXlc67d27O6oEBD7qVgc+9yZMPPSfeb0OTYY9rgG2vQmkWT2P7v692XbzvDJ/Sb0UBzU8IdPnPXjxEr5YlnU97xkrvMDjtbyvzCw88Z8Pvs03bj0wHPA8ur5aPcOGb7yTBC4+tMOKPaYmiz2FOHA8pJ6dvKOmiD0P/688YZ0dPFRhcrwLAwM902y7PQWwgDylYKk9kLteOz6zrD12jbq908fcPY/Gjby9Wms9tnAWPnBRnD0R/wk+Sl15PZOvIj2wyyu93gPVvB9BVT3F2jW9eV3svNiMQLtKbr699Kg5PTrM17wlZfm9Reu1PcM48D168/c9TEKMu82Msz2rfDG+UNiBvh8VMD4zUg++cZvEPeiZwL0tB8m9ArGxvaJVL70Hwz29IGOmvSJQkz0MkWi9bt7kvYUDGL3U2HC9MGI5PfK9kb2XnEw90dK6vIGPJr04B5G9mai9PX6eIb6aEiA8shMNPrkWrTya1ro9lB/7u/3iTb4EMyC9spILPSivJD7sVOo8oEsXPkuBh73maMW8+m9CvTEBLb6bLdO94D8PvnPvKj3AHDO+u4UMPqTBCz6PTYA9b50ZPtPXIj5Qq+S8GCFdvkROFD2GIkA8un53vZsGHz3zOJo8IBqDu8JNHD431qU9","+WapvQiyOr3FBAk+yY+wvYEj9r1VIPi8mbb+vPNbPr7mDPA8nk97POMm1D2kXCc9IXjsvWFUsL0fIoY9fKmTPSmKHD253HK8rkgxPiw4AL6mPjC9q8BOvarbzL1T/JG9p81EPiiZjr3f1Te9dS8xPZaKh71jtbQ9LOMpPsJyxb2NP6y9XSZcPBNSmTyLzfG9YsH0O8hpKLzXZog9gzbUvUgsHb7Blpg9ginXu7DfAb5Fi409JpRBPYX8Bj5IGoS94nejPTEMMjtxGZa98gK0PdKp7L0sOsO91ii/vdShGTww8Mq9f7ELva/GF76DHVk8TBgJPWeIDD7WvWS8ygoGPZpS3L0WAZ69GGgdvLC/1Dyz+gq9jbVevVOIf7yUw648WRPVu+pEYL0r5ec9q2kpPkgTvTxB4Vg9WoPIvVWj771QfFw9y9nbPVOaCT68Xr88YykTvFHPn72TGT89gHIFPjUc8z07JMG9nj6SPDHdWr1uud69uY3ZPeXHtTx8nNe8nSgGPYcp1D3Sf8o8GWHwPe2TVL0Z6CQ74TeHPdh3JL3UQDQ9NmyVvOhouT0GWYG97gkKPhVgVT2csxs+RYmFPSsyvbx334s9aqCIvYlR6b0azAQ+YJsiPoGhwz17T848comBPWXRJT5CRDs+A2w3PWrHrD187Bg9tZXuvaHWgL7hOa06FFO7vDLJojwm0oS9dUhDvYXNGT7tm5C8+ydwPel9MT2cm1K8Z+wHvUnqprtMrb69JI7BPWuK1jwIsPy8NTKPvVmJAb0ZJXq9GZ8/ueI7iLyyeqa81aoPPpyqoD2hUPs9UxPRPOR8+rznWxI91iVaPXqRJ71v2i49TE1tPP7jc73KaAk+/EeCvdaYIjyHdeg8ZDGuvIWYLD2j9F09QJ9RvWcTyLwy5048kzchvd7yorxMfkc9TLc/vfRWkr19Qxu+zHuEvTiNyr1rGbg8L00hPTmfKj5R0ie8+j1cvff4mD1PuKK87yzZPSTctj1Ozvy9pK5nvTEPo7vnHPw7","xcKEu9V42LxmDTw+KiXBvDeEtT1o56G9w2MQvq1Kgj6Vwhe+oeTUvV/w3L3t4jY9mV1KvR6Z2T3tqyO+idSDPZT9Oz0V2LO9pA4GvuQ7Vz7SUYW9VxJEvbVYQzyjN+m9t6ShPQwW1DwtCFe8r36lvSGh1DvdB/g9nP20PecWcz6a7Kw92/ayPRunBr030SA9gUZ0vW3Tjb3ONwY+RpXdvEHOxr3rcga+Ygumuzq96zyKaqk88tSrPQmDPL3MC5u8ENNKPcBebD1CBwA8O0yPvothEb7fnNU8oiyjvdxxgT2Tywy+jGyjvHswz70MqCE+N850PNWIpr3tSCe9dAY5PVVAdr311gg9U16UvZBuSz0wWlq9OK19vFpb9byaz+q8G25jPIoRIT1LznI9DZNSvVgtwj1WiJu9mDT+PdQppzxFgM49K/7MvWvXED7TUFO9sL+APUnBvL0CXxo7AX8XvmqmAz3XC+S87QvsvG96f713aeS991TRvE2P2b0kRhe8pWKzPZ7xEj30OL88RPidvV+hFT2IJ/I9pp5cPc6E6T1q7Qe+nrGpvO5Ngr1Qe5M9S6bUPccHtD0GFwO+IBkavdc7S72yKyK+UO+qO04FYT6Yj/+9Nsiku4c+IT4lc3U9Lx0ovVeKQL116i89ZEQXPp97770ocCI+s7etPcxfnjzII9i93ajGvdbAdjzbwA6+H0WdvfIs9Twul/k8FxDIPPA2uLwVdrW9779iPSjz4ju3Py09orvVvHtTmzw+l0e7P6sZPgGElT0zuIc97KFNPWeqE70K5+i9MIJGPTt8hj32G409ApCGPf5UFT7V3LW9HnUzPsbXALwhaFY9RFjLven/oj2rO189btMEPlKei71/mFe9lSSCPQhoHby/lbs9dZT0PQ71q73+deK9vC0ePvrIpT0mvYM90Y5RPoXCPj1BsTk9ilRpPULeHT1MQNa9Pombu7+GWD234j0+EM1NvfV4Xz1I1ak+tmK6PGAHGztY7/K9irYkPXHdhrvId649","R7kNvYeimDsz3k29Ml2quspyjr0hpoS9ix4fPYyICb6sNrW8xfNhvab0Sz10KQ44ZVEvvLboNT2oES2+GqacPU9LtjwnA6G9CubfvXhe57lSOea9/uzlvdxNoDtzx5u9522VPA4arr31Bw2+gXaOvHYn170ryN68vk74PQVF2D1D1/O91ACCvPqo6D1IJ1c9UBGmPZOFGL1RyEC9efBivRLmzL3V+dM9YUVuvXl0eb32LNY9yp/LPGGkArwvFCU888NDPasbXL340e+9/8EcPCEpLD5dRh2+3Z7evfGsn7uw2am8LibLvMDBHrxOKM+9vYGJvXaGeL06ZCw8BHTSPWN2Zz2FEv+9m7CbPfHWjzwoacI7tfb9PdIpdL5Jm9W97KkGvl+X/L3uZbE8RB98vRLogT0bAO48Tq5IPb9FVLzh/GQ9vDjnPNy7tL2l2Tk+c+LIvdS3172LqQy9okj7PDap9bwYg3G+S1jBPX6S7712emO93q8/vrXs5LxUmfG9wRGjvTyfGb00THc9cmeQO/s2zzwLv767OWEEPrxT1zyd1So7HzUGPgcZXb26AFs+N3v2PCD1Zr6Nc968/wsAvk5Q9ztzc4w9AIwWPNNzUD735vi9b/chPQ0et71uIEa+5wFaPWSlMzsO2uS9RqBzvlftyTlsP4A94Ck/PmTQPrzIua49yq0LvYOet70HEZe9qJhCPFPkxz2/tBi+w3WSPb/aKr53CEw9VH/DvFGjTL2vgJG90KMSvA8kZbxmb8a93XtDvLUc5L3qpvq+8XoUPYxqpb3S2W09Okk1Pf1qXT0T3cS9sBGDO9Axjr3XFHa8FsuaPQJikr3rUpC9nvwwvqaEEr6uCBe9JPunvbxU3jz9pFm+3F9/PK3gDT5IlQW9uCrqvc4J7L0nynE9g8mPvBP07z35iiS94QRLPlE2Ej3RSiK+qR7kvH7hKL6UJHO+0jmTvv7PRjyrKQy+sHLjPK4xwD3KuNq9umY1Ow+dpr39R9w9ftOvvdOE6D1B5pO8","65SFvQa4R73WEwE9ygXgvZ8/3j3Xot698qjIvWGUjb3z0k68YFBxPYNgNT6CsEw9m3ewvcOFXjyEWle++HILPrexp7uz8fs9+y7EvS8vl70rrqm9xY0+vcGLHbx3qZa9qCdVvlt9LD1rAuq9+nHDvQRQUb4AlnI9+0lQu5SPvL2vDZA7zvOcPRR1er27bh6+nmbuvVhglz36CZW9neQ0PTtx2D3Waca9N3RqPfihDTvm+Hi+GgkRPeKPZj2GBsi8OceTvaW97LxSHig9aVmMPZ0lj705JTu+CwXbPOBmm7wyozs8rEHsPXrHOb74Hgi8AlcSPbk9Pr5XWf69S692vksALL6zaCE+UvSqvHmtOT2I1mQ9CI0WPaSCAb14vqC9NjK9vPE7Vr3hmAy+E93xvZHzlD0/oha8h9Opusimmr0y2MC9idYaPQb79r1ncOK7d7uJPMi9Cb4egTc+o7YuvkToFrwmw069meoJve7KfD05EIG95XIdPdef0T34qYG9JwSbPKx6QL1eaQM+LkkNPiljjj2VLN69wDcFvc4iKD1KxZ09v4jfvTpi97wefBc+vp4/vhEV2b3ERGS9VU68vfeN772lS9U9BrsXPkcRBT543Wg+M/s6PsUanzolUGE99ktRvQpPML4Lw1+9J5hhvNQkd70n4BE95coGvY8cbD0r8A29KfHtPMi1m7yOqBU9yA/TPfAhNT661p4+6xREvFwwyTwyYja+nVoNvmQiljyx/689NyabPN8tqL1aTqe9lX5tvTDrID17k/M9HrFmvjP81j19gIQ+MQDwPa7pOr0Rjyg9D1iQPdstiD1TmPO9iNn9vddVXL1PkQK+8AA5PWjKer1lC689qFl8vfpoRL2eDOA8JwcevQecDLy1/oU9H8iUPmnknj2xnhg9ltQYPNQKFz6TdIE91ezPPIAesryL9g08Vba7PXlpgj5Wb8+9Rvwnvtl94z3GObG7MmsxvpWij71+0oy9GFEQPt57zz2eHQA+zwcSPTP0EL55El49","uqbTPQYBHr6VIKu8ba7UPXnFIDyxuNS6oERIPpkHxr2qLeQ9vIAEvBPINb7yFXY9A5lyvQZUmjx0YpQ9AutgPZwufTybzsi9a8mSPSg/6boYEl891f+HvhDnizy1sx+9wA0ovHKnAr7hYmU9afYPvFr+Br0R1Li8ercIPMxySTyQez49tqDRvZitXb51pX49fWWXvZavCz4cR3K8AVfnPBoGhL31XXu9oODUvdqsbD7JATk9mDaqPUf6Nz0iyty8H/saPnX8IT1/GZ69VtGMPC1zCr0PG8u9WRQFPvHXir3Nv4S8rXskPeh2wbp2qFE9aX6WvaMv571/d6M9n7cEvLkDxL3uZhy+ESCpOy8ItL2Kh0C94QPiPRugq71DwDE86klRvkY7lT3EdKI99GVkvhKQab5d7mc9qCYLva+f+D34k608wJaAPQgY/TypgWk99HubPWeZ+jvaUfq9M5BRvAPEnzzt4Fg+BmGIPYTMGT7JDSo9x6f7PJgJ2j1sJk68tPw2PMNuND73Jt49VPS/vXPoND6R0Ea+ZedrvSdz4b1nAR8+Q061PVMuir3mi1S8dCz5vTNlYT2ZX1+8MRE1uoC5c7zJHo89y6WoPLSMML0pmdK9XRaRPH8TL7448GC8DKa5PNX+TL7mDfg9bEa4PTs4HjwSZro9bXGTPZFjuz6A7We9+2K8vAwyZr2g7j88SbTePRqZKzzBcUy9HFqbPbX6zz1/hYO7DiQXPdmy3j33FDe8hF5avRdipD38yD89GkExvX5dfD1uebe9AZaYPAoSm7xhtSG+sQbavYDuG71Fys28eHOtPWIzGz2bzQM9K5qYvaELG7tz5Ug6eEg/vbyGnjyt1WA8YwIYvNCC6r3lTr49vcLZPQGU2Lwc+oc9/vMhvGuIZjw5y1U98VKKPfaMRjxWjic9zp98Pcm0MD0fWi++tsbAPCLPOr7nfUM9ddAzvW6s4L27uys9chYVPUx8M714JH28HJWbvf7gGz3bMMg9WINFPS2H2rxOXbS9","SpsyPHPrTz0Ts+S9jd4yvikT+b0jKMQ9cA3TPa2LTT3almM9SWgmvR4iQL7LX/U3lREnPU/Un7zmGBU+mYylvCk6zr0Cc609drXXOzz6gT0dW4q9bDgUvgSp8jxbSIm9DsGfvVnowbyIDAU+FSGNvXCjYrySBEG+3/DLvQCydzsUmRw9nYIUPUQnwz3mgLU8P/MhvIH2m7yNIz8+4dMGPf8C0by/AvI8UPS2PdvHtDy2MXs9nw4mPameeD1s2Ly9Ctd/vd+y1T3Nu0s+asBkPX1zwr0djEE9s81RvafFNb2v8vC9xqy0vK2cxjrl24U+RQ69vaa0X71h/aK9AwDEPbn7J7obOcG9kjdGvd6drj17O9C767OxPXmZnz4JFrA8kaPmvRmIATzmfaA9jgRNPC2/bz1ItCq9G9nxPXoA7z3sb+A9pwuBvc4cRL4APGg9V5MBPZ0JEz0KTNQ90LFYvca8oL2DzMW9EgixPRLyuD1MLw8+kZNnvlRNgb35Boq8RzqyPXyix70rKj2+1WGJvfpaE77drqa8nP4fPt9nc70hT6K9Ko5gvYbWDL6KYds9x/2FvZzFfD1NO0Q+oIGVvflAGL5qf3o9UR/0vdFy1z06ZkG9mq+BPTGPiL6Q+r68whI4PXU1uL1cm2Y92EhSPWUKuL3hx2g9RqOgvZfI6T2wIiG8+aLDPTjnFL3kI1K850W9PKHo+TuS7os9lAWovatGhD36PCw+R5S3PUTHnD2qzyO+aDo9PdfxqD3ycS+93eIyu3kg/D14Jgm7xaoePjhZyT1m39k9un6vvbpcgDyU1p09Jj6EvRWADr6/xqA9bD99vrBiXb3NL7G9BoW6PZt09bzjc+m9qtjGPQecJL1Kv9a9bkWRvhyylb2UYAQ9Dwf7PUYTjbt2Hdg94y0HPo8AWD7d3Lg8e7hYPtNVqLzfGVg+Thb9vLYrzT2/e429rG2cPIz92Lv5mNs93HMkveosCr3UcxI+uQETPLgUOj3YEqs9yW+qvTp7EjsazH0+","n2yXvGjbbT5/9xC+KobSPTfRmL2S7Rg+ggyXO/n+Wzrb37E9PBUSvo3FATxV8Ny901C+PDkYjb1HqUU9LTlXvQi4gbywrYK9e1mcvrw4Sr3VdYu917WdOzenCD73Ik49pQODOaV3ub0vYZG8ik3dvb64zL3oecI9Ba/KPUlsNT75g+U9ka0rPakuh75AWqM9nLvivW7inD2dq529v4DpvMygxb3c7Bq+qaoRvss45TwLRwk9ebZPveHM2r0K+pY9QhqOOyuLXz67ybS7eCvAPPrU6zxLKdy8zX1mPaLfYLyNSlO9+5LOvT+aDr4yW6k9gTJrPaihCz7YVn89s/pzPHAJoT2d8Fe9wJGRvr4CSL7pHWo9qU1hPe2rpzxZhU++6llRvaYnx7wBjYe+2XxJPfGHIb1R5h2+w05lPW7RIzs3fKG94+AIvqZPwDwPtpW+MfUSPVU1lj1vrtM9ALJcvSWxHr03Sas8WBGEPjh0Wz0v2nE8ltiGvXnDUD2RG9q8rly9va+j9T2+kGo9mzZRvZ7i6rtkO8E9Xsv9PB6gD770gQs+YuuRPfzmNDzm98W9CGSLOR0dR710byE+RMyrvXRuo71EazC+NeJJPZ7gjb0t9oC+8bJwvRIwij2IOCc9agyUvQ2eoL19GUY92Nz4PY1C7LxaCKo9XCxtvWU6Z73Ev5g9t78zvWGVJL3r0gy+L1ZGvH2VjD3iyqQ+joWxvVJRv70xA0Y8DqcaPaZaq7nwdW28KSkevow1kzwajB2+A3WuvV2f4T29BCa+UiuKPYuEQz2y/qq9FJaZPQMB8LzQfQw9eq+IPAhI57y5px6+MkvCvTjjSb32TdI93EUZvfzd5r259h87tosGPaV7u72Jr5K92pJ3vdj2Ir0vPCk8ggMePUI8VT04UUO73WUpPIi9ET7oBJo9XyPIvGkzgr53c6u9gxczvokbcb3/nDE9agRJPsZ3Dr61UsW99v0IvUXW8TxInRU9l4XHvURnwr1nm589NF6CPn2wW7xet4m9","hdLovFLhFj2j1h2+Th6KvluB4rz6GZK9lcPDvTBYEz1ghyi9qaInvD5cNT1oXL08dhmxvFlFTjz41bG9uXuQvRm1Dr07KLo6TlwovDDtkD2d+ha95nB5PaT+kL3FHIQ8TtMPvgUFob3nsm29jws3PdfCdz6/IQ498g3BPbT4Yr5SU8y9fwqtPUi7D74T6vs9INM8u0yR1T3BUrM92g2svWCfPLyMHb497aasuzxOL739x4E+r/KYPWIKJT6IKJq8gIWaPa79rL3pUyC9Z52MPZNHwr2VLKQ9KTccPUjOgL3vnks++mnQPtDa97yTl4A72H+vvWMjgD2Utro9X+dovjoDNb7sigi+s5FVvsnWzL2svva9uOHFO8xoA71SYga+XRBfvUSBsT1MTgM9mDVPPcjrib2FHi09kbOiPaPnV704hSg+BltJvV3XS74KzLC9y2zMPQV5/z0ulAI+CdlePTDZwLxA9qC98DMSvQQQq722jOC9yVXDPeVd6bzsQLw7YPlwPbioBb650fQ8wmcPvVM0qT0RlwS+L0VavcnBML2TYIC+bZeRvXreoz2IsPe9thg1Pjih1D2KReg9sbi+PD/Hxr1H4C89MCtwvRmRIj7OCRC+MO2CPcXn/rwSYoW7sZbCvWOudL27BMa9PrWPvb1GE74pVdI9S3d2vSaK0z14vGk9Po74u8lu67ui1Hm9PD+TvX8cHD2AlF2+zQ1GPbPoGT5irtc8kPrxvYGqgLxU+4S99IPdvNuI7T3e8MU9DL2DvjyMK75HrU89m1ATPoKuZruKnI27odglvYR06b18Tbi90NUXvqHQcj39mYU+EoA+vmi1cj5Cw1A9iXs0vTggSLyXiNk9xWQWvktLKj0GDZG8dVcwvbJemb6buaE9aropPQSZAr7sbRA9L6a2vHq1eLsnK/29pQYCvnTHh7yaUEs+VIU0OzFupj0wdoq9vfxgPVYGOb5W8dW8Q8iQvdzkSb0s3+E6pCvLPVm15D3ySjU9PI4rPtFhkr7uBZI9","aL3BvSX1wbzTCmq+8TMJPqvMnDmLmAO+6gbiPhTgsr2Ip8q9c7TdvcvOs76CCaa9sZTsvRngHr6/1do9HpoZPschTL1r/fc8+Oe2vodxbL7f/Rc9nOcfvjp1Br7q/G28jeiJvf2YBTzwTbe8d8gNPuR5jj1s++k9cJRjvQ/Ssj1qVso8/bx4PlYXhL5YhqM9c/xzPQZRl72k0oG+dSz9PSrJEzzdyF++tHN3vW1/Kb2O3Q++Fa36PdNzyL7Awmy9/8LnPc8eFT6oT2m+B4BWvgWkzTu6fge98gpgPOhfXDxc7RU9OOdcPYDDXL6PI2492p24vNI7Zb6a4jy9BKnXvTtdvj3QeTa+2ZwyveICmTznR9G9Y1TGvfNfs71DYpe9NZ63vlWwBT1+xju+vmGUPQx+ZT7/x5S5lG15vYGMkD1lTme8Eyq9vqywqLz00DA9FkEwPVR9nL0lc4I9pTAavOhPwb1MQYW9+2NJPSfEkL7WKQw+UG1hvBN9Sr04fcw9PS6dvbKIdTyZzEi+TWxyPrwIAj2UfFi9f9iJva4IyzxGEh++hd+GvJRLWj5EUa29X8gNvRw0Kr7xEAY+vqsGPmWrij3kzxG+asIEPhJ+572DqFo9t3OuvSNaQj5q2E4+/IXCvac4Kz71t5Y9vj77PEGcOj7fDaY+s7W5vUZsbD2z9oY65CiDPQZYNr7D9uS8aihevu379b2ZqLK9NpqJPVW0yr5GWro9Pky6vVaTc75NY5I6O11evXl+wr7NdSq+DkcOvj8chr6/OWQ8hUG+PJG5rrz1moW+Xd8rPTFA6LxkXdq9YfJFPpMBfjvmPfq9wj3wvSXYGD0Mb4K9kzpdPOGWWj30ZMA9lfJRvvCXw769taO+c0s0PTvh0jwvS4E9JMx9PSXSwL3kMNG9rNsAv3C6Az7uKcg9LHocvaAPDr4pDAO9m50ZPfkrlj2I9YU+/TRIPjmUR72l9Ha9m/MsveIt4zxqzFG+FSbGPSDxTb0vFdG8WSXBvTk7AT1yswq+","O5WoPbRH2j0fiFW++XL1vGCCK74a+lQ8jLAlPvnmgr3Epci8O/X9vJe0N71ZeWa9rACgPVL4hrwdt8o8NFDru63JNr3IUQ08Io5/vbTuBL4u1KS8EZwDvafxUL1gbo894u8UPJtw3D14xUU9HLaAPQJytj002Mq9Yx6lPfkEE7zY3pU8gVbKPdY1873vUIC95fuYPdgEej38WZE8lRbEO9ADF74rBPc9vIwjvdO+bz1po3O9vnKSvUiHHj5YdHA91vmZvUF78z3VzL29w9LOPSer9j0exqo8+XoYvS/C2LvJ/PG7V9CBvamnwbpXgew9dPUqvXT9ZL1ckq+8jk9WvTt4Orj9cTo9MqrevYjKgz2D8E4944dDPes62bs0kHG9BX/XveNArrw50xK+7LzjvdAnmj1G7ec9sSy2Oq833j05eBE9AaIxveoQM75YTvi8L2+vu9Aq4zxDq1Y+Qce6PH8prD0CDQ290GCfPU4YzDu5S6U8smy4va721b2pnYi9anCDvTxMcD0zLp+7FPbtPIA/Cz2WVDU+3fVhvHtO0L0SSPc725JcPPVbGT7TUwI+OxaFPev1RT1cVbM9blbQvVliKL10rTY9y1dMvB66ML2odJa9yrMzvq6E7z10UD49rhAEvQWDSL1apPQ8Zo7RvKANp7yMwAo+sC5tvQCclzoknBA+Y0M4veg/6D1qkUU+yZGUPTN/zr0xeLa86Z2EPeAf0j2K7Vk8eJlOvQx+orx5rEk96O3iPNWW5r2s3IO9RMn9vPG4mj1dQly9r48fvWiUS71vRO68mETjPaEXdb095Xw9s2cPvaoEAb740RY+lUVQviQmwz3FNGW93csQPE8lTbzJ6Bu+HtdTuuvXfT6br5G+YkAQPfeQ5z2tFgg9syBXPWorJz2UCh28QSuAvYvXcD3GAkk9OwGCvfMumL1ZhQu+8DbBvfzbw7sjFMY9zXGCOsWPx72ba+I9p6ajvb0FIry0WLo9n22uPa53YT3w5uw9n00DvCeQgL34qR29","VjIKvMDpRb04HTY946iGvVMJnz2GCbi9SSfHvU49Ib6jTqS8CvE3vgW6pjxyBt+8unDxPTkTmbwPCWi8aIIMvTMM0r0F75W86nAdvsYeO7yL1YQ9oQTpPdb+kD0wtQ6+ZGc2PFSM0r2+nBa85IQMvpyKdD3cAss95RCBPBsRVr1psW29LE0hPoMYLj32f+i86CgdPlKZJbtEXZy8BXilvT3wm7sdeh09R0gtPo0GGr4tHtk9gimwPZP0Eb2T5O28vQKgPZlKTjurLKS9fyKEPDOnWr1hUgs+E1zlPEtd6zt4tY691zkJviPvFb4/OMy9f/GrPbKfGD0EpZQ9gvb1Pb1kyL2ymg4+cqKfvQiBqD3iKQe+UQyAPR/oID7AWzo9Zo7uvbqjIbysnoG9GYeYvPr7J7sxWYi9CrhjPV/Au7wA6c08BC+3vaLyBz2dhIc8tvHjuxIrmL0hJc49sQY7PQbuaDw9NXS9j3vAvKuJML3fUBm+zwnBPVOhjz0wlgy+r1xTPU6NBT5D0D88yLDIPe7eFz17zL69GgdaPakfszuHhw6+Kq+hvLbJbT3ZmgS+75aqPKmwTT3/CAw+I5mCvabfer35roE9f4ZRvtmA+b2nkOI8uISTvWbkhrvoles8A7WCvLqWzT2z5ua8Cf4aPnO9G73A9JE9vlaAutgGdT3Ahb68cEvQPT4DCz4sGqW8WAjYPXCUrTywllI+oDoDvUnaKD0gxew8K9uEPfuz4jwbVc09B+mAPKGdLLzAjC29I1pNPS2HCjzEfYk+y9MYPZMx5jwr9gk8Gjq6OzkbOD1UMDc96dEMPYcX1jkjYGI9DzI4POklyLyTw6c8hOu2PRPssT2gOuY9xeujPea7Uz1Hd1Q9s3GTvUWldD0OpOA9VeIgvSf9lT3OEnk8bkTePbA2kbx1QCg9QNSyvGuuRb7Y5o49NZAMPe2I8jxz92q90x7JPWO+0bzwpoS9UcIcPdUKFTzY/3U999zKvYAzhD0gLAo+CCVsPRU5vT0Odmg9","gdo6vPTBGr5vQ7G7A60xPkVqET2F9zk9tYFyPB7Pvz20WHa9a7w3PXV+5T15VNQ9Mm2QPKycmT37B8K9HOdVvVRfEr2acQO+7XFJPGVihb0O6ZG9J4q6vQvJVj5vvPE7CpnPvfeFRT1YUhS9E2EjPg1LLr4hpo093UzzPcV+Cj7xlrI9cgSDPYb9wLyvy/G9dAMJPGN/4b2TbGw9SDp8PZGQZr3rvui7bTYvvTzmXb3MQ0k9rRm9vEtSorvwLIK6SLg0vsYJujw8gWU8rjwCvvybTr16dno9J7tlPr4RfT2X00O8rJLxvbgjCT7PDSy9ZwmNPWUyn72nNho951poPu42Oj0dk8685mpYvZ+elb10q6s8YS4VPiILkj1vnYQ9Mrujvghsjr246hs+d0E0vOt+rrsaQJG90mCkPFEzNbsf2qy9nb4oPUWhZbwXjIE8MU7AvaQ/w72PVEI+dib5PBPVhLxYLzI9m6ruPZo0eL0ybD89VeZqvXT9rD1gpsQ9qK53ve4qMj2O+9k9nEA1PsXEfr4+pw69+xU4vXLaAD5iPgE9BKRUPX0ZAD4itXO+scEXPnx+nDytics97N3QPCUabb1HT7684Er5PdxSsb1Rjo89wkHkvY9EgTpr2UC9k52jPXNLfr0+2Ac+CyDPvcZDJDzl9WO894LfvVfu7bw30cg9mXNAvaGP3b1wIh49/voYPr0rMr40Hjw+HGfCPYPHtL0zgBu83Z2kPfuViT2PLhQ9se8qPEVfiDrWzZI9mFxgPX7iCb3e6Ie9abEXvrn0wb2Mp2S7fAhZO3BkqjsKiw8905efPRyEVL3IEGe9qRRuPStGqjwmX+68IxEAvbCher3ausE9D9XQvebVNz7xoAi9ZLyGPePFeT5bFmu9EyRwvbqVKD6K8To9+LRRvW/ZFz4Ue388ZoNzvTlQdz10lSu8QU3APew0I74XjQe95uRbvgghFr0Yale9NEI9vih22bx4rLq9G8S6O5AyBL+qjIq9m7BTvrT1eL4ZnWW8","oL4mvchbqT0NCAc9L9fFvRG5Rr10Lbk8Fo/dvnvg+TzOmRK+OWSaPdNbfzw5ZLY8PLTUvD5jm70s7Uu91fWAPRXoAb4vUF66p1QmPXgtLz3w5gC8XcMnvn8XSj75IMq9o8uRPMUZJD6bbp49Wy4vvUZu5D1kfb28CfP6vZwDqr2/nZG8NW+UvdF7Jj5xykC9FNIavjxRBL2rx9e9el6vvXqvr73shvo9QZiCPEX/7L0ohRI+8JR3vRUvYz3i5Dm+MsqSPb88Kz78Ju29JEFKvoT/ET7OMEg9nqdyPOLMoT32muQ7YBlnPSmKGT5QEj09xbELvrpeOb4Y47g97rT9vABw073AtZO9gB3dPdcpHj40zPE9aSMAPXmTMjxCQvI9XKLjvAbE7L3/u809iN4DPjUhVr6yO289XwEePiSj6TwNzTM+wjpYPatQ1zz0esg81BKGvQv0gj3NoPQ9Aeu3Pcg3ET13Q+S93oPBPaDwvb1ZxDC+gSWrvX2KuT265Ai+taUovWkoLD50EgU89bcXvoCsaz2qYzs9czinPYZfljz7QNi7apl9vnK/XT12wUI9pHk3vuYYAT7usky+CRsePSEKHT5DOJS9OhmlvYXXNz3Nzl29RYiUvdBqQbylbZI9PTt+vV/Ig769dki+6CwCvk04cD3QgI69IfYmvMm+r706Nyg93hGqPWoxnT02WVc8JUoaPvOAE75U4Hu9+5IfPr0fUb1DqjQ+y/TZPQjAyr2JyR09tHGavanoiT2mRPK8IL6VPUFKOb0R3+S9K6jGvB/IuD2C4Qy+ziQ1vtqcoT3IpiQ8GybevVRuLr6Q2lg963C3PdvFozzLSDQ+UFI2vpDy7j1/n+W9yiaxu0xcvj24Mbs91eiSPcK7Lj6q+bs9CRnfvUSgITyliOC8P2SePqS52TyCtPq9nXYRvWQp6LzQxGc80CELPjgTLT2sEIW9LmtwPkdxSL7/84k9Ao8TvGSm3Ty6TCY+wfliPY3+uLyFhnk7Ci8bvRO8k72y33k9","dCvhvKj7CTxeElC9loWcO2Kh6jsiRhG+9X20vCgYej03/g8+UcwivnwU8r1fdgE+iJYJPlVdTT5BEZk9Qqu2PaqTzz2n9o+9MacJPTFXKD47PeE8sdAJvsPKar0nCBg+NztjvWGJEr3v14A9ecjYPAeFKT7wa987+9cePh3zJD1BNBi8jXLMvWl7BT6Uzg2+1EKdvTIAVj2Ua+C645HZvENzRT4zPfI8d1exPYGgaj0Kojs+Z4FSPW7dNDwBNcO9g83XvOJw3rzqzBa+WATyPVgt2jxjfnE845VRPR40KT0gCFs9KsNFvRV5ML71L129/RZevQ1R8LymL0i+WulkPXCLkb2hdTQ+37pGPT84k70WURG94j+AOzRuC76j0aI9yz8uPZF3pj0NY2K+7XGbvQ0EO71/Jsw9P2imvegOIT4ht1e9vcxsPbRmWT50Cum8DzYgPqaAj7z+SqI8q/mzu1cMOz6tY5M80T2fO41iHr7UUhW9DBYAPZW+NzyrdES9F+ZAvcb3V779RUA9jrOBPNhokr2Lxea9tycHO49SZbv1aec93PhIPvBXmL0aFLw9ZJ/ZO0iFmb2sUE+98uJEvhyItT2Bqoo98Z1/vUqs1L0jMjk+EiV4vmuBFrwxR3A7eCBoPZMfxLuw3Vi9QJ3QvY0ci71GV4m+lel3PA/Oub1dQOA97z0/PfpvJjzWmHS8Zesvvs4AqL3oTwE+y+XMvMI4bL0WggO+a4UDPtfJ9700ami+eC0CPqK5KT02vI890x45vcrDFz34MZ69sLAiPmNGBD0EVxA+e0erPeARoz3VOUU+TkKovfZfPb3wdh49uVhcPvKPYL3uJ4K8grHVvbeaAT4a0V09BYeOPGLNRL4dRMW9RWISvhHReD2l4FU9JbtBvaF/7b2lQYC+r1+JPWYctr628A0+bxVQvpR0ZDwnQcw7sLn/vWrjUTycvTs+DleNO8SJD74jBM89PRQjPQfwAj4dR68+yHnuPO8ZMb2iDEY95tyKPRNEAj734iW+","1iggvjVBtr2lyMy8llucPXzCFb47IAw+vM9OPYKWyb0nMkI9RUu8vI4DNz4NWgu+ohkzvUz9vT1GPtU8gd8QPn67DL3Usni9W4K+vZXU37xxxMQ9R/iGvcwpEb4DYAI9K0eTPZBirT1t5C4+iDmvPL5r8r1KzW+9q1SdvQIWvL5CIh09hA+GPt/5dT2jBhK9ao+WvUGi4j2EIwg8YWgfPOAKJr7ZEGg9Nc8kPmGcsz3EwW6+Qx7mvX4egjyQySU9zqHjO3uvvT29Y8y9W0MAvOD26z0TBdi9Aj9rPbPwn72g+H48wkR5vL0qgzycH+K8JU41Ph2Sqj1IeR295HqqPboIHr1FSBs83ZI2vBqGqT1LfWa9ym2nPMeuQT6Xfhw9tUlGvP3MgLzo/W09itKmPeGDmj2Zo3A9zEQ4vPGWObzn6Pa8TA3SPWDyjj39j+Q8t8kcvXmpiz2S9w28IzC+PUKhuzxV7kO+EcDovNCj6r1CBc89KUkJvupYCz5EX7o9/oNVPTui+b2oyFC9QOyqvYfr1LybGhY9VTcnvPJFhr285Qk+HmSHvQmkA75SOjY9T6lSvX+r4DypAdO97KeTvRepMD0uZko+PhWmPccfp71BrJs9EETLPSZjlj3jFMo9xMGBvU20MD4OFpo96P6Cvu5ahrxDQUg9H/m1vB9ozj10zGI9j4NhveZUjLy8fAQ+7rXjPT9jS7yUT6u+5gExvTP+hD1nVZq9pjAbPvY0Bzy+myu+KUy9PSBk+j1z24g9s5gdvUOjtzwn04A81keJvPPGgb3PvuQ86HqRPXRQCD7Z8os955LAPWFnfDnHR9o98Om5PVtGm73SFUK8U7PLPERBUT2jQ9Q9uJx1vHnN6bn23bM8uaGhPaNfkb1IaR69XA67vQBJEz76ho89S90OPmOhPT3suQU+2GzBPe/ukb3t64w9dYaYPcHkLj2FJkK+mVJsvWcDGbywdvI9USOIveoZgj3CjWA9WNDVPLDmCz0t1449ZQ4fPdCGwz1AJmQ9","uyLtvFmS+j2ASx89MCN4veeVXLw3LV+9bRiZu9eCD73r7a09wP2BPVdUQj4ZJ5g9DtBCvZhf4rth0Qk+fg9DPHcJ3b3RpMc9BiE0vMkvnT0dxga9UETfuzYaG763lYa8G6H3PWBIAj1mHyY+dR+/vZC6rT0Ahmg8USjWvOxgJ7w/HVw9ihD9vO2bxj0STUE+4kvWvfeOtDxJJyg8Y4HMPTTVHT60naW9iV4wvpzaCb5kcSS+UBw8vPVtBz4q9r69kyyru1GTJT4xgwW9K66Avdn9tDw4lvk9HVcmPvh24DvFska+2mgaPY7ewzyTSpQ8TlrLPat9p71A01u97yopPisSDry0NIS9mMIvvmeJwjwaewQ+D3OlPtyNx72r8S49RcbrPI2rPb6/ohg+DekDvorcQD2vRxg+30x5Prku5L1/Gf89aRMSvVADNL39D/I7w2vqu9TDMTstGIG84Ci4vShCh7y8i0o+e1tKPlGuLz0WS5+9LyAQvZXDAT4N5rc90HvIPIcJkT7YgiE9C9KAvT24Ez6YD9C826DAvUqdV72nuWu+95mCvcw/UL0sx4s9J0ARPsx7DT24qHK8KNt6PQJ4gT1yNbQ9nzbnPRdii71/+sS9sPvXPeWwDj1n06O9U8axvMX+szxNuZY9//P9Pbcyor0OgH+8b7CnPAtqp71BY2S9ANQKvTbJDD6708y9bXwQPgLZnD1cL0S8sRR3Pcx6mb0fafM9auqFvkFIF70yqA28uKBNvdrJAL7lywK9P8nxvcX35j30pwu+Lf17vsQBsT2m6cg8rPz8O79GEztxLOa9yP0ZPevgOzxmhRW9LV8wvW9/Yjxvkr88KfZLPeYQBT7Y6n8870z1PVgV7zuc79o9hlwXPaCxQ75dnjc939qrvc+XwrwR8ag9sDaQvfU7fjvn3XA816EePupa1zw2EMi9rKtOPXRjpD0uICc9QX2RPfwL7b11ATs9QTflPQLme7zjXh49DaFBPc0i+72w4rq96NzBPTBREz5MYIc9","q76wPfHnfb1o7Yi9lFHQPVdYNDyG+Hg9SCwVPhBZLT5yBw48bQZbvct3kT3+/Sq9dazOPZNjhjzWhm88zQWOvfYlqD2qQy09Xokuvnitsr1hsYa9XBWSPT+klL0wmV+9P8z6u24bvr2j5kE9G6B/vYBwLT3MDtG8y5dPuj1pjb3RgsY8q1pjvYTu9DxxGB2931tMvebSU77fo/M8YZgePXjirzuG3aa8lO38PfM6M77FbgS9TVSFve/Wczw72OK9G7nWveT/QL0K08E8BvaHvem3gL4vWcU9kSKRvcOAPz3MN2a9C8gfPhd14z05mt88DvYAviiIQT30N7O9GZiYvckOST1/jaw9pgeEPVbmeb6wsk8944kJPbUasryLdo29RJIHPeIgF76/1hG+CmsBPuEKkL0blZy8g86Wu1Jx0D26Okq92F94vijEqr1Ek26933QAvfKcFbtlOUi9sTiEOMRtBr3AMA89l74ZvhPEwz0GYE0+wk7wPEUNcL2AHOw9JJkIvVbCvTz48d89+MuVvWrrPb0GPFc9yj96vQx2aT1u0ec70SQLvaxfnz0exo09jLQDPrfeNr2RAk4+W9XVvFMGQ73gtWu9VKoYuzJsQj3KJs+85vI6PRkt1D2IS1G8FjOCPt9UnT3W+c89DflOPJV7nr1bdpy9twMNvr+3NL6n5Cm9dWsmPgUhFjootiG9es9rPfXo9rzA7a88a53wO+2+Oj1NUxm9z9KtPLDJGL2IgIu9jy7GvSIuab1qJVO67eIPPsPiPLyHKKy9caofvJFBhT3U/Fc9MkntPN8Vkb1lRrO9flJ1vUTKKj6CPCG9QaEDvXoQ5D2OK6y6ztkUvVK3Yz3Ml9m9aJ0WPvVW+r1fZT89tldSvUAwsryhD5K9qCAGvn5f+zzsMwy+CoGpPOcHG70w0mi9qUFxvPYC2Lwf4AM96a/1PXerQD7oKwI+ayWdPCXn0TyOAp49SpucPeBFD70UMQk+A+EBvqsdnL3S8IO8vBwDvj3nbD3YkmY9","1gLZvbO0hjzojDy9AIvmPMPn0T1MFjm+G6gzPWFNNj4iBL88/4RUPUDW5byddx++m2bCvfuL0Tzmci49COQ0vYV5dzxq6EW+nMIOPgJy/71ezYC8JQqTvGVk1L0WIpk9BA3TPQXKdL3Zbls9GZEkvGyVm72OfOs98mLVvb15Vz4HOwa+2DqWvdvPnz1DZLU9hLoZPeyzIL2wWJ+9/LQ9vbsLxD0Esly9Ns+VvXUFWb4Up6w9TsNLPXU0ID1bV5K91IVGvAfeQL0dLgA+EzTLvTvGKT19CKi912EFPYUkt7394fS9MK6wPR0LCj7+tYu9bFs2PZFbWL6YNIg8DEabPcCIpj2tlb49qkNNvmB9vz2lqAa+V81OPc5R+b0HPfm9uA20vQtvhL1A9bC8viyGvEWr1DyWF9i98/+ivdgNPb0cYQi+sXiiPRb40D39uS29Git5vSingr1s2Lg6PxqPPaMK9L2fisY9MQEhPb13cTzsUAY9qbd3PU6wYzxF4YC9tXDYvI1KAj79vZA9u6OxvZ+Juj2+wK89Tv5dvknzQrwsfNI8CWUKvIibGr432js9hYtWPnBvJr6ET4w9AXZDPvfRPD1pCcC82eEqvsYGqz3n8hg+RgobPhD8Dz690NQ6fLjAvMTEDz0Or5e9wJhyPa1qOjzWw0c+ZeoovXb1vb0SFfc5fKYuvTdkZb1kr8G9X4KCPW0MVT3nSKo7BYRHPGhTEr6gck8+jG2aOz+hGD6a3+u64WHFPSSkUTxbWqo9EKhHu47kf734Rt08xqFHvh7Kg72jtdC9gkI1va7iib0uKie9TqFmvTkHKr7NDxi+BKxKvVFsLD2d5tc8Aw0IvjUUKb0viA49zoNrvTOPTj5Gaq+9E0tcPuOB9LuhE/+9kPQrPfPX4T124fU8PSPJPIljAjrcWLg9XaolvrPfALyqVP+9QmewvT+gwL33u3+92KcPPb0yKT3/9Iq+tL1xPRcCQb01HnA+Dp5bvHBWsr3DYQk8wyV3Pb6w2T2ojEi+","Ood0u2PGh7uRGa69W3fjvewh+73T9Xa+wAulPekOlj1aamU98oqvvaqmub0s36E8cH2LvTwD2b3utN49NLB5PfQbOT5JXr29JTXlvXEaRzyUmiw9wOg/viMpy71rG228JlNivc4GZL2G4p+8mEGjvUMhA74YNKa9imrEPWSzTL02Chk9DMuqPX37VT0acV69kRy1PW7pg73V1IK89C1avTR4dD7esWo8etwbPWYqgj6SaxU++VbzvLvtGT233Ci90+dXvdoX9b11OrE9/bFdPpMohL33lDg+kWuEPYh6rb3GbeW7jvgcPf36xr0pzSC9+ikIPpCX+jziYZi9Oz/OPTQlBr7mSD0+iIXAvfYQLD6MJqG8h9WDuWicBDvGsEI+Dx4RPfboBL0BbSm++nD8vbzhS71C9Q6+8vElPSzl+z031/E9UPblvAuo3rwc5KQ9D4cuOyRWWr45YlC+tBu5Pdjhaj0z4Ya7Yv7YvNgNjL7pYKE9WQmSO1V3p710Hcm8a0o7O8HOnr3xzAS+4ph1vIXqOT4MTAY9z+k8vADC4j1sa0O+TZgxvUL4Vz11g5u8fWKEOUCbyj2IBMA9zna5PS7Q2b3HIQU+N4t3PifPub1UdMm8aw6WPUigvT3at2O9VEWAvHK9wT2UjP69ATmqvWrt473zwcy8+0eIPsDGUDzffyY9rEnGvBs4t70DCas9sspcvUwpsb3E8JY+t1NEPqQDz7w2bpe8PlyoPbMNtzz5dLe9/4R/vZ74UL5fP1W9ID3bPXR417ui2i29avKIvQm3XT3/GIE89ja6vdzjw71ep1u9vTDsPBkduD0bEMI9MLavvRLAKb0uuC++MeUTvdY4Q7y7KxS9ZUeZvAvukzyd2aK9wDe4O1rQOD0q8/g8OgrYPAa1ML2toRw+CeufPuAVIL7Qx3k8KkN/POcomj0vzi47A3ahPXgzAb22MgW+kltFPqAHOTvhFXS+ifpyvfHmnz1XIEu8cDLEvXrN0D2JsAi9HjyWuxzrwbxoJnU9","PTjcvC1FKb38FoA9rqW0vQ9qCr0KGQm90M+oPfnS8jzYfLG9Dux7Pd74WrzLpUk9lq8yvqh+Sb2VtJA9AV0WPhnnJb3NXrM9RH0TPSMwxb0jZKO9mcv5vAS0Jb2MuJq9mWHVPM5BubxS7Ga9R7aTvEdP1b14WGO9lNGfPKfBoj0IYpS9vDAzvsI/6T3laG68vG4gvgA+973DGHk8Fln/vEZ5Uj6+AdG8OaRvPZ1BAz6YIU6+qMo/vb6jsr13C4K9eXTSuilBfL2aeyk9VvrBPFH27rzhwKS9XbeHumA1L74yGlK93Wh2vLN7yz3CN6m9/PWdPImATr7s2My8kyoCPkBjDr1oihc9a79ivkrM3jzMdo698kOBPtMQjD0pUVe93Bf7PNq+Lb4985E9JMkFvZ5eVj3acda8DFE9vuH5obqnKFK8YlHQvamYbr1bozK+U5juvYb3GT6Ys5W9lQrDPIuBzjt8Dj2+0Uu8PSm5g77povK9xeGvPYDhFL4EVy6+R4sdvs37ND4Hg8U9ptb1vYE2RL4z+uG9AxEHPCc5TT29v6a+aV7sPeDSLz7WuEq9wR2RPXPZkDzdGAS+x2DKvJDLFb0yIAq8Zhr3PUsXUz0kXYq+jLyHOzZNn72N62W9n06WvVk7Y760Fgk+tNK1PcNnD70A5us7bCijPUrpx7yrN0S+aMeHOrvMP7zj/No9Cbk5PhwLYr575dO92LQxvlWNuL77YW080ywzvtcGALzIO5E948WBPZfQSj4U/q49CoIpvuv4d74GvBO9fSv4uyKPgb2vWV6+tOrvvZYOYz6YbBu8+WzYPY1VNb5QAoe9+TgTvtLKYL59Vkg+qX/fPROpvjycFAg90z3APbfaXr7W/qC98XVeva3/pjzj/Je8yz4dPgN90z1gk7Q8FgavvdCDJ7704q09qBalvYRTlT1zcTm++kcYviJc172AiVa+ZriFPnVMgT2oV6s8oWa3vpuWgrqIA46+mE58PHEgab4nQt49Q2ZMPWgRBb0RYXO9","uNqKvR2MtT3t2Nq6lYLZvKruyT3Jp108t+DNu9icZL2ntmC+QRZjvFlD+L08HQc9SQluPZdkgT2F/+K+cnyHPYSUFL6Dr0S+e6qgPkakYzxzVbY9pc3FPTpsjT3vNJA9++8RPb+MrL1N8788TUeavYmsNL10ilw+6Va4vV/bazwRiw08aSyVvudhTj55Kp89MFCRvtXsGT5Whzk8rpqMPQyIqT0FVMY93COAPT71h77jyPs82jjCPbRghj2GDl++iT1NvdbiKb4sBxo+J7E2PUf6Sj5q4GO+VdoIvcvcFjs7XXk8iSv0vZZzm76GrTo9/6BAvkrxsL47Cui9wau0PdtoCrsbdwk9ZijePc7tGL5lcqi9YtKZPeo9nrsv1OI9E0QDvpfGn74c2BE+QSK3vaWBgb5J8zi8iyjQusTEU72BEg2+afxCPlcX/rn68eS9mdZMPQXMOD5fYq89v5P4PD37B75wUBc9UOfPOwncLD0s6YE+WEuFPdM1Gr0gafU9W5iDvRVL7TyeyCI+SoeDvtiTdD0TFSm7vHDcPepcQb0Bmx++i9RSvj4U1b6hVGI9ZfG7vmwhtL0tyW6+hnznPRwKhr0v32+8WEA7PWf41T23XZY9iJONvnIQCT5cpBq7uuoSPZPSLb5xYL29/1pRPcElFb1oA5W+XD9Nvp0IdL6JdpC+nCnnvSxKd702StG8zYSgOhBAg7yc0ku+uh5SPMoHqLsng0w+oZKnvOfNjb0RybO9L2DZPI6Bgj45tLC93rUOvhMHhr7AX1e7QbkQvrqGxb2H8zs8E9LTvUs4aT4FgRY8NpzkvJlIIL5Me6e+PKo8vXKV2T3rhPO8PcuavtiPmD3yn6w9zFY7PvyTUb4CnFa9x8IYPY6vxj3OLWE9F7ndPOPwO75yj9C9hn0Uvmji9b09Nri95a2MuQb0xTzfw5k9rhuwvmo3Bb6qTFs9E2yGvrFC5r19fb69YjwRvk22uT3va569Pa6VvaqIub3NDA++o9+nPYE+GD08VKC9","d7jlPY7vuT2Z62W99GILPlXuAT1Ajy0+zsUhPZCYHT17wA++6L7tPLU9c70I6b09jGRcO46crT1OwSy8dLblvDjaEj4gjVq9rCYQvkDysjrbgIi9d28lvWszcbw2yKw8CdEIPlSo/Tv6NWy9zpz7PL+91j0OYH69twNzPbPDd73kGrG8wwuHPvr54j37DDY7suMyPSD4zDxml1U8BRuNva8aorygyNk9Lzm0PJ/vKz6l6O+9ytOmvaCSxj2CsbM9QlDivRLPazz2psu9vxqDPiGdpr2k7G09Aorku8nG3b1t0u28QasQPrjLSj69RQ6+jbbiPHEQf71vzGQ9KAK9PcNDgLwDTyw9kavxPNqCuz0DsNg9jExmPX6FpL1JTHE9D/BtPYBXvz1HP9Y8Y1LgPcr6Jz6Txak8FvajvZzJyLoxpTO8N/W5vV7gmb3ikAU9h6hcvbQxuT20n9Y8g3NnPVz4iT2g4wk+KH/wPNX6vrzFvgA8qNP4vbG07b2XnCy8AIoLvbQPvj1IUkk+vv29Pd0S+r3dcAE95ZHMvKKe3TwPqR281suXPZtqnL1gqLQ+Lb3HOyzoFT01nrC839whPTFlbD0vSbe9YUzPPRmYkzwzZ4Y9+T+LPROjUL7OS6w8i05VPdZtDzyv4NS7mTv0O38sqrw0OQk++q5QvT7c4j0JyQM9grNFPuebdT0rIQw+lOs8PnO2uz0thxu8Kz2CvMYj0r1MNJQ9cudaPX1kmD3MadG9RvGtPG8/c73rYA0+mwGqPW26nbwcsbg9qXGfPTre2LzW3A+9+4a4vWDEpT01Nse9Y5w/Pn7YYr0WmRy966vBvcML4bz4XsY9viDEPe8ODb21Ii49EQbQvTzJFLzxO5o8CP+TvW+vLTwPZKo9zuuxO4DoML1UJFi8N+exPM1ItbzdKq89c4baPRlZor16h228FSHlvXCLVTyBkcS9FV8+PVmdND1pYwU8alhNPbdSHr4wyYK9hP5PPd6167tJ7rA7PjECvYJcs71ukwq+","F1NiPsKLRz7QEDG+a0HauzkIKT0cjA+9INR8vascjL1BmeC9chqMvTjwWj3/2vU91511PavrhzzaAlG9aAqrvSoBZT4LHA+9O6AFvotdqL0aAhk+d38dveRxSb3d6bU9vcOoPTyAj7u1TPa8hv5dPZCLs7wOafY68AxTPVE4+7oC5+i9pHMWvgstAD4UmgQ+dm0zPV7OArv2yA+9LSRxPf9MNT4fZO2958rzPPnWd777GZc9kq+CvI0cj73dsou9M7YgvVpbNj7BSHU++ct6vNpxuby0HlE+I5aZO1SQtbzhukE9huv+O7SQWj3EY6q9ZwRivVQL5zz+t0o9OAOMu9OcFrxo8pu95XTgPMIFFL2NAaS93d4NPEu1BL2fO4A9GzRVvf59QT7NIC694LRJPdM4qb2le4g9Mstlvd627jtQcw8+FvA3Pm0jvrwUnaO9P0Csu1W57z2vuuG8lzA+PetptLu+lf69k17rveVXxr21cJK8RllZvYx/f71e2qQ7Ocb/vGKbib1tvAg7dCKivfSHID2sMDM9IrnHPZ0kOr2tb845oFGnvIy/Bb4lC927RtIAPgn2przI0dC9qVbLvekSfD3fSK89KBc+vtouqb0g5LI9BwQgvgrP6byQ3he+4YnsPSUwWL2pEP89NdemvjftKz2knyS+nzGTvTQSMD3+rAe+xCK5PeojPryminq9VdfLPa7ITr26yQy+qpsRPDBzHz6UWHK91Kr+PXLVuD1/g6W9GpORPbtiij3yEEu9Nhk0vaC3DjybLqk7Pmi4PKzSsD316+E7mddQPgIsQDt9OMc9oOwXviJgYj0dyo+9DM9CPVVf5b0NehK90ihvvDHGtLsUPZi8tIMNvk7HzTw4nP89FNJkvfNOuD27dSG9ESGXvXVmNz0M0Iu89xWVvDXGlTzdPkw9ES6vvCDe+LyTOva8Q9AWPuOtz7tu+mk9HRI3vmc3eb3EwBu+2c0gveVuIL0Cy0u+pYPRvcQyRr3eHFs9nJajPY7rAz0oLhe8","c5eDPctceD6JgLk91Wauuvc067334Oa9hUvVPVkcVz3rIyU9C2hJvruXnj1+fyo+A/bEPZ7i4b0TavQ89H+Hvm6dJT04CjQ+9wfUvK9Ibrw4onk9DceUPd+MVL0btx+9C1iAvTzjiT3A11m9XzF3u+UsdL32xy28jIE6PL9Fp7yXNZk9h5kRvVjc1T0rTDa8lxYwvlJrgj1fQo+9CMvpPIVsKT6cq6m8WaWZPVek3T1IQO28Ay9mPGQavz0zEwm+Rj/aPQHasLxZbJ49wWXwPT5mGD3G6wy9QDSTPUdmhT3ChRu+wuFgvOBxEb7UTfu9UjUVPiCyZb7Tiok968phPgjcnT0JLf+8QxVvPcng3bz6gho4f6NBPoWOxD10+RW+hGpxPPH5wTwZtuU9NyIzPcOazL3PMgE9pLasPeUTxT0G1c69GUPVPet/KT6UfI88DJtRvR/OP73rICE8TtcjvA8FxL0HFck9J68bvmh/cb3JYls9Avq1PfxtGb0plem9iKD3PXJNK772EGC85w8Svn8oMD7f75K9mMlqvVCuzz0HepO9IbHhvdKWG71wfuA9uDSQPVAPcb3hFzc9Vr+mPSd7BT4R+7U9h0AovkiM7r2fmG4+fXLPvDKQFb24LVY+IrMEvobJCD0RzNU8jGUMPbnUa7zIm9o9pApaPVkyIjzXwDW9TQWjOy0WyT0RkS6+Fi9YvbBZzz27QkU+bDBtPGUQCbx3UZ08ExKlPdi/9b0aJ5c9POkFPTSc/zttuxU9bfwDPtbLtjrwIkw9g+UfPOK1LTzbtKQ6s++3vQZhtD1Ke6U9MkwJPhVknbvDroi8vGS3PSDUBr4MxXg9F0xSvEH5Tj3zMVS9UhQivV7tOT7pFDu+75fWPcyEEj3TPIc7QRylPPAPGL0PtQo9E8CRvecqS71nqwu+ooMRPuWyPL4wBZ68DNCePBsk0b1EtS+9pxt6PRIOiL0wFYY7VmUnvF4UqD1sOjW+CVPxPIrlVr7LucK902cyPdC/lDx/ahM8","LWuJPQhGXz1/tAi9DRENvd9RvD0YDZw98Bsrvsd6fTwf2tm9SdNKvI8Ecj7dN3A9Xo6APb53r7x9PNA8noHVvURkdTufUHO91mQCvqchxz0E4zs9lgCqPP3pFr0DWCA9ebLqvIkwbDymZ089whAOvUDNlbwvcym7EUn5PHnOGD0ccr48epNyvV8c3T31iuk9l6U5vMMLSL5sLJk8/vYKvn/p1TwGmGK93OUvvRCNHL0u7ws+Hpa+vbeTbj7PD549okI2vWX8SL2NB/Y9pkrSPML5Nz2yELy8knNFPs83Z718No096dS4Pdnxm70wdPa9S0tlvWs7n735XoK9PMmgvODd7r3cUtw9RzGqur5yq71ORIE9FiAZPl5RZT0Z1TE9vZAzPBEfAD6C+uu8unZ3PbboJL3mAoM9QJL7vAP0A71hyJO8gts0PpONID0XW6K80hp6PJFROD17wDO9EVaNOWBOV71HyAA+30qjvJue4T0XB4u9UisHvWaSYD0yzJy8+H2PvP51vD040BQ+CMTVPDotcT06yg++D8WoPKxLCb0J08I9Fh/AvRdpsr37Y5A9SIQIvW5CNrtGHrC8M8EovXHhhr0DbQG+J5uBPYyitrwrnPg9ebVxvfPec72zrxq9MxhFvHDcPL5Bl5S9LxTfvaqoFL0i7YO9/6rKPZhwgr0oJ9k9kDsePvdu77tve7M8iGtMvR46iL2XhlI88/J1vY40MD63gMI9VZgwPlrGBj7PBhU9OOyKPNOOAj5ImtG80LgzPt/LJ7682Zs9Gvi1PTBRoL0oNRm7Hv5Ju/tAoD3f8qm79KuuPD5H6zqzee88sC+gPSew/7tN3oU62sqQvZHQxj2LSTY+MKmBPdUnDT3Kc4C9+e0TvMzcVjz/cMa7WoN1vmnPZzwJEt49Kz+RPmYMrzxWux++NciBvR7pjTwBh7s9B8PXPVb2LDyyXcq9QBkKPjyQyL1LFYE9imrIPdqMGT65FW09GjCDPH1pID5Kv+67Dw16vSHolD0r9kS8","YE5WPahb+jxPGRS9k5N2PQBgDDx/uqi9DCmnvb1+AT7I6oa8h9BBvYLLC738oaE9R/8evmAkKL0xsZO98g/yPIEGzTwZiqQ6o4j7vcOyeDywA8y9UaZoPR078by7D9w6p8APva3yX76enVo9hXjtvBzmgb1qshG9cNtaPfq5jj2Rfsq7peO2PbyNgb2Yeyk+PvOEPU73Eb2lwBi7Sw+2PUc36b0l8SO+SPyIPf3Y27uHUE6+TSO1PXLOXT1qcmA9l0k5PGpBZz0iWGc+NHeKvZsuj71L36I95MYbPUeI9j2pma29FtqyvR+/G77A1PE8qAq8vJVjHzzNR429laOavYFSrzun8iC+l9SHvJ5r4r0fz9o93FWyvRn5DjsQnNS9yiDvPU6OT7toBrM9DxWkvZd0IT6rKYC9YdgCPnBetby6HJ08hkVVvhzMw71qfGG8aFkoPcn2kjv119g9Es3yPMKmiz03Ufi9YCutPI6+sD1llq69WRyfvYpesr0spHU95p5YvdhhS71GTA09Tt4Mu2SB0D2LRMq5nwQ9PClUdr3+yfy8EV2Cvd6ul7306EA+lDn1vXXvHL5V8rE9sHigPZy/Gr5teks9g/i8PaQyJT4H2WY96DZlPWJtS71uDd07ClIXvYcwGL5L8Rq+HWUiPYa0Kb03+hM94JgzvXazDTsh8LO9wQLZvWPYJT0L+nE9ljUMvhqtLz37CQA8nwxOPMAT2zyR9428D1+mvZuh5zh3gJI99yggvT6bdjxJWIC+hFOnPQ2lvL1c9mA7epSsva/hRL3h4Vs9yRgyOxLj3zwyB7M8wcoVPhmUQr0Ag6I93B7Lu8j887wwpj897D5XvbDaG77P+gk8HgOuvV0TJr4dPqS96nW4u2maVz3Xdtq9TMIyvRBJarvwhN69CBkFvJy4Bb2yi3G9W5SBvVnxXzyasva9Z4pjPfS+1Lji5Q6+wpGZvUcHQT78zZG8jjuJOj567Dt9Cb89t4g8Pi6Hv725s+G8rpwlvopeAz4gC0Y9","wh9hPeSkOb6C6SQ9JEhCPTcHQT3WIr09uW9APSOTuL3iaEY9oo9gvA01fL1hFfw8bCwaPQm1wr0f2E89aIH0vXm5rL2JCZk9fAFzPTKalb0jIMe9jUf8PRV9Wr7TKVO9K+QQPeEpTz1UPCq833V7u17tqDwQzNw8/DSfveXa2zpZq9K9f7qxPSCfhDyDOuQ8Ggn1PeOEY73qne2684hQPZq4Fb5lgt08xZervXgf5D0YwyQ9v0vLvPkAW71F3sQ6TA/AuKNVSb0ErN+9HYbSvfOCDL08q5M9nFKJPcn07zwZ50w9LJ97O3bgnzx3hdi8z7F8OoP0JD5dls68IZd2vOF9qb2bHGk+UU2Mvc6vWj6hkrA+HHfcPRqX57zx3uq9SFfhvEZmCL7kT5g90r95vR1qgDs56WQ+Gzi/O5wqLr21qbQ93FBOvd7GEr1PGm8+HnXAPVAulT65gdo9NhRrPmG6JLsoXM098CTPPQsIOL10DU++MhX8vRWREL6MaDc93dHIPM9JYb4436C9nSXvvCC3Aj0k2qa9VqepvbOscr5dWiK+JVedPRpK2z3etXM9IyTSvbZuRbqutru9KC+LPRWAvLwi2/69oQ/DPqrbvz3gzAY+1QKXvL3/8r1qAEy8oEvZvX7f7bzBBg0+HPZBPlXBVDzHxP286csYPpuNiDycK6+9jHk5Pl91ib3Bdji9Wo3DPR5Gvrq0V6E8pxkNPhUXCL7wr0W9a+XfvTfi8jzUsiq+2Js6vNKw7LsAdHi91mpOvX/sgD2N6Nw9ZOhfvrekIz09ENG9Ro6BvHLaKL2zogC+NSdqPLiVZT02jwc9j2wHvorsrz2nSxw9ZREKvZOiXj1Bx1q+HxQTO3+Gw72HjL+9M/yyPWBqND4pa9C9SD8LPTFMPj3iKHU9gAMePkqQrj1uf1G++rqsvVNqSD5/aAU+j0YHvkspZ72D4tm9GBtrPnucRD0uo+E+8VlbPcFipLtCtEE9STs1PjFjjL1pkrc9eLd4vpjPxj0jlTY+","sUF2vU8oDT2POBK9csQYvgy1Sr3onGA9KqECvY6Ulj2aljq94R8bvYb7Qr0ZHsi9kAYOPqtmij1LE8Q9vBMQvsaQej3joXC9jLhTPShUNr3Qe6S9JjaRvZ/Htr17hJI9H2TwPfZLAb2FEya8E7oqPYcN4j1/91W8cjmHPZ7ARjzhW6y7O3n8PTPVBr7q/MY8hThwPmiBcT4D9BQ+OTTDO7d5lzzs36g9ZY+KPTVfx7uPkUU+3eeBPDI3Sr4UXDM+dKFUvamprD0eWcs9xGsnPifrML1RuBg+07XDPCQDOLwgP18+90MpPEMjYT6PSCe+vEyYuykM4700ifa8T31yvFTogD0klly+slp5PZv5rbzbc0M97LbHva0tabwoWZo9AmyzPZdaOD7ogUq8bPJXvVQZFT54sco96EiEPrbEE71nAQM+HKjRPVZ2xLyMDVw9l4iZO6OYG735jjU+t9J7PnF6UD2Gb1S+BNIkPXyayD0/Ggk9M88bvsE8kr3j94e9flcIPuDSkj2J+MA9HR04PsNarT4DmMC8l5bKPC5WwT0Cvos9ceUdPO4ldr3iFmi9+p6lPeM9Cj1CnBs+rpL9POyKfL2tzdy9HOOvvbkG0D2h2969iuTRvQHK4jwwoNi93q66vS7zPj1FOza9rjeOPb1Ldb3C45I9lCuSvRJcfj00UG69+vwAPj0v6j1TcaO95XkfvsI0sT0gjWs9LdskvrhY8b0s/Bq+Gx+dPajI57wJ0qk94Q29vPx7vj0bY968rh8fvi7por1bcJo8IlrGvcuwJTyGzAC96QuTPKaYJ71lWA49nYhgPcn1Rz1XBSm9kX55PuxepD3SwrC9OoVTvfIRaT3L3rK9IKASPkEejTq7wsu9CVBGPew73T3WTiq9yncwPddT3z3aaAO+fm5IPZo4+zymfcG9/o3nPd6IZD3fVLK8dVnSPIggH7znILS94YVWPcGWKD3fKAs+kXZhPRvGWT35mJo78Pa7PT640b1M4Ue9tS9evc7fqLyiGAW9","T6LUPfhwH76xRPK8RSswPIJrJD3AS5i9Pfk8vhTmOL2FesC9kLG8vb0lq72xHj+9B5+TPX5TVbzmsa29R3g9PdcWgTwlw4O9lDLdPS+CUjz7avG70E1hvUAvsT0GE049J9WavGlIADx9csa8Wn/EvDWUqz095zq9TW3ivKoxLLq4KuG9N7URvXN9O773Uwo8RdMyvB+zET6ft+S8dp+wPZgfrT3u3yG+V9wIvmtRX760ldU9cWAEPhftmDymyo49VWS5vfKPOr01cWa9+i5VvsG/r7wKFIc7jkmTvNdzqb1XzPk7qIAFvgV5jLw99lM9/42+vdBoEL6dBgQ97+jwvMnhnr2IjRy+VuICvKc+s73dWLK9F2nDPRBoBb1v54a9o4qGvaHFnj2F/OU9PIVePoRqnz351ai9/g8EvQqsAz2bZg89XLSuu34qnD1UDhS8T+Q9vI30pL1Fl5K8asD+vdvf3rwntB0+sa6VPeLDbb3KdSc9cRUtvUnVJD1/ySg9si1uvEoSyD02j/W8vcuwPXD0u708DfU8yBAivXAkIz1SuKm9A5txvYpUMr0P1dI9Uf3+vWXvp71NOy69UoRrvZGxCj1qDgG9sFIgPWaykL2ORbw8yTIxPWlxdD3G+829HDzLvRXyAD6E16M9J6GBvWnJ6rzovNk9Ooh2Pc/pFD7D8RC9XI9DPtC4E754gIy8W8WbvXOiKT2vx3a9QPv5vMJ+Kz2euQW+2eHwvYBruD2yp1i9bBdCPdhWRbwUBb+86PjMvSuOrr2LRTu9qEh8PP/7vL36lQC+G/GzPDmqlzw65GU87EnavbNCZL1sn6i8N6eVvW4zYb3868O9CcbMvDPCMz1Pr3498CcMPR6Bt70upAA+umxWvbJkiL1O+VG94W+2PTP6dj0QO+I9pAO9PdSoljxTSrG9iKVmPYv6kD2Wb8G9T+hKvQCkrr1YMQO+W0D9vZPVZT5oy5W9jhnRveuLjb3DOK29+ULnvMcYJr6p1489m/eFPQ3jqr3SzyA5","Y90zvpp18T0iPse7PmuXux86Uj7/4Y291uQzvsyR2j1whfc8hTobPi+PVD1fJQ6+nWv2PMPm9T0aPSW9kV2hvVoeqz0o36e9Ii57PXE8jb0qTeS9y+R1vBMKAr31jhM9wUisvHLFgbwvZPk98CAtvDHf0b1LPN49Dln5PRTlxD0Q5Fi8+V5cvQNXc71IgRa9fgCtvHHO670K8R2+mNYhPmhnBT7Ti0e8nOAhPSpElL53r4A9CPTXPQsQ/72tlkG+LH8WPvMGb7t5YFC9d2povh5/eT1z02K9ZI3vvR7S3r0Yinq88qf1vQlfjbzEfGo+SVupPSSTJb4Qj7m9gYiuvapksT2nw3k8sMwcvo5CaD0cbAa+fWcOvnrOKD2m77W9IBlLvhzm8DzXF5m+3U8ZvaRZ7Lmoy3G91jw+vbsUr70VuUs9VHjZvdL9hL4gS969/TdAvsuxB74/Aa071VCMOgnG7b0BzBu8HfIjPaN9Nr3oAVq9ScHsPApPhz1VkEG9VSVlvTVZ1L0AEnG9xnn3O5D6kj28rIW8biDVvbGA6zxY4UQ9wQcdPigtxT1qvQW6gkbOPb4Mer1hKs+8uVmCvDD5/T2ZFby8k8ROPR8yVT5xaD89t/1rPlMEeL1FrEY9KcBTPfkyiz3Dwau8mrpWvMj9lzyRn1e8pj4YvqF2Rb77q9e8vtN6veZyd7yKWrc9IEvWPagnn72c0hG8d5wRPuBP971SY069vdabvqnd0r5Vjl4+W3yQPfHbOr6L2Wm9suOqPRQHiDytZ7Y7blgfvqHc7LtWVKm9SJ6HPaBlWT12EUO9uG4IvlZzjb0Oqx++xztHPiRLlj15vLs9yCJ7PTU5Gz1ew+A8l00BPkls4b3LZ5Y8o9XaPWDm9DyhKSI+OrnuPNQzr70tiQ4+uUnZvem2vD5ndC28rDuQPgV1Pb3iqa27kzvyvPO6SL7Nva884CTAPGpTNT3Pi5u+10jTPc1J0T3ICiy+ga9HPqYe6T0a2Za8EWY6PqVI8z3ZKD2+","268XvRGQWL5T1UK9KMzfvV4wjb18XQG8JUN7vPAbgTwwsSK9CgiDPTHmWj3rpw++lFzrO1jgMj5Iq3y+6uTSvSnrPT0GBYo8N+ySvu4Omb0yXpE9tHwSvrtKmb5s5PY7aFzEvYLr8r2PHac9jTJ4vp2aDz4Ef1W9xtN1vtlrVj71CIE8eAQtvFRJAT6RgQ49g8iBPUA2FjwMV/u8FYFsvD2zAL7+tg29e4eUvQjtgD7/qsg9eB+APIACuj1Sd/89NxYPPUdtir4DMSW+vW4yvKgRj75rzbA9cPOjPRB4urtNTV08XmeBPg85QTyQGPG7JN9FvlO25L2IMsg9/q+gPYNP3zzekuW9FQITvklATbynXWg9GcW0vAw+qbxI4Yg+gOFDPNl97z1Ysmk9ursTPYXQLD2UEjO9FN4zOyM5gb1xk+c9lt3bvANqDj1lpk+9PFUAPC/tizu5HvA9BQhhPS0JBb6/ziY+97qHPh3GOb24FP49qkHwPc/J7bqYsI48XJaLvW5zPDuopUM93ZZaPmfV8z0z/qO9KznEPVLC+z3X2ae8ZYe7vX3tMLxEwjU9lGlKvO1Hqbvyuws+K1/ZvfJpijqZY6A9h+JHvZEs0j1e4fU9e7S/PK6477xuYgc+Zw9OvSy39z36MWQ+tD+4PRDHUT3stey6GEQdPUO/yb2tMrU9+oANPYphwb22fou77aEcPOzKPb2+sD4+PgJCPahqAj1RHbS8Px4ZPXCEmD0IgoC92TgaPWJKqj0/goo8XmNGPRNrEj2KLok9GEwSPr5plD3/KRE8UdVUvFxvKL2WnME8qWYyvYEl2bynrQW8eJ8ZPBmoKD2K69m6LwIlvT3tEbyd2WA9U9oCPZ6a+ztusuK9AcrqPUBDmTzrZ6U9CUUKPSw7Kj0Riv06OYYCPqkXSz3Ny1K89PoBPf8D+D2v9r085R+nvYV7kD1v7A89gf8XPJk1a7zN4Es8oHAGvGa2DD1FUK492I+PPeDuLr2Jks+87/OyO7XkW73y1lw9","3RG4PfMUCD61Y7k9/4gTPps8+j3KBQA93hFuPcuP+7yfTUM8/nIKPrVFBL5lCOw8XqcFPhJeTDt+GzM8ERgrPSHjTLyXWC48iYK1vZBYLrpVUjI860RsPTRKiL3UbYK8H8yAPexAxDx0c5Q9EMpuPbbE0z3Olhm8judfvfjmUz1PZDy72w2lPT0ovj0JMAC8S2CFu1MRnj3Jl+68kleTvFTQuT3iVGo8yfgRPCjJP7zwkWE9ASQsPXML/z06jmq8D2MNvfC20D3KSio9i/zWPXqysLzdH2m9DhmbPMLdDr3i9aw9Bi2ZPTBuS7ywzGY9+9Upvvc4+D2unAU9aUIPvCpqLL2833C8AG1qvZ9wjz1+1Ka8BoWFPZhSML0JBpM9/lP5PB7MybwkI169rpBEvEitBDx0YDy823uWvW3umL2jzkw9VHnJukoHDb64p5o8qrKfPFeKxT2rbWM9+V9QvdY9gbzrz929fsamPdVh3j2a/5m7KW6ZvTpjhbzFsmQ9NzoVveOsJz4qJpg9EzWIvGDPDzyYHJg9t++jvBkHFr3ThTS9c+awPdPjBD12uUo+7sDXPQ+gIr3Eq+I9iRO0vc33kzrugSE9QkJxvtRY7T3N5TU9xMP4POWsuLv5aIe89T+VPSacD71lxzg93yOkvQjNHj5Gnes8pP5PvK5snz37jT09BQaVPQLKhj4AtES+ws7+vBJJBz6t82u9LJVMvj6SlL0gaAK8mvqoPejEo70/Qqc9JliGPWP29DwhWUc9ew8MvjfDDj33wUM9vMP3PRtO6TxMCX89XlkOuxnlxjy6QBa9atFvPNf9vbsLrVM9yALgvBu0wjzLQjq+JvGsvX605r2O1dQ87GERPUqBqbxtNfU6BWCLve+JpLx3qa+8MlnzvafRUz2TQ6G9kIeWO8HAqTr+D3q+lLInvtVycr04yCc9sucAvvUNuD2uk5W9rPKTuPPxZD3kxjK9q+GmPf43E749f8s9VRNBvvCgLzycl+68NupbvHYBN72X0/09","4LSBvbCk9Lw2Ia49LlR2O6xH2r3aYFK96jCMPUnmI72ZpGc8Cq1rPM4efr36kuA7iingvf3rq7395Hq9nH2rPGhRUD1Z/T29T/6dvqPsmz111LY6wQENvrwCAb2SjRu9CtCsvDRkZjvQnFG3dfDzPHtsYr38s/G6Go8kvk54jD0vEz+9g9rbPKDxIr2DpOM896H6PVGraz0mR1W+I7avvXdrnDx16529WfmXvTLWGb3QHNg85vAivQte1T3tx5G9T30DvcbZ5bxi9ne9pgAHPh/ClD3aLZi8poX4uxkkzLriY+49D859vSJ6+71l6Fu+plAVPgLvBL3cW5W99eGMvZhWib3gFew9nFkdPWd9d7y/6k28bax6vYRFtLz/U5o91nU9Pcpwmj0L6em9hPG+PPS41jyu1I88OmjOvbnC4btn6E89p6y5vZ8Kgr3n0yM+F20ZPFHBLz1YyRq+xYSEvVSMLj3i0fe9Ntjaur/ZyzzgVX6+T1VnvWcmeLxRI+M8ZFsQvZ+1Pb08XHg8DOqevS0LTr25a/u9k3g/PQr8lj0g8zG9InbIugkJIT1GYL88Ju4bvYLCzzs0VUa+l+bcvC/d5r1A9KO8RsnlvdMwR72Ibie9fPIevUINz72xbYE9fU8UPaSI/72HUNc9JUvOPblTCT77fWS9YjFgvb0Tdj5KwwO7HZzvvUHvWr1pNCu9pm7uvYdezr1JWZM98101vEiCTT3ThRk93Ze4vC3ZbD3jqf88PTrIPJPKBL6PqIE9Xu21PWY/cj2RohU+8JcGvX7IBj4r5z++OzzZvOqDwbsRD9s8V0biPBAtZTyGe2g9D949PZcewzwRYKi9u5iHORm1472iHoW9lfYZviS87L21JT++UWsJPp5miDyYMUG9smMHve0zaTy4DTO+08VAvsHypb1Lc5k9beTvO5w3wL3JANw9yUZTvYkr6T1HKrY91JhjvnL1IL4ZwV69IVfRPXD4O76luqo9nKg+vd3ETb1skx+9Y7jrvZgw87xiyQs+","Rs3zPMiYeb10C5E9dnrJveQklb2/GSM9ubLAvbIXaz02WBU8rZ2OPUScvb26AIi9YSSVPdRmCz6w1BI9OjFiPfS3DD0BkEM8RdBkvHQ7vT064ya80Nq5vR1AXT0HeAg+o1dfPeqSOb2VUD68XOAovoXLuz0yXnY8UPUBPYDydDzKYwQ++HIvvfEuj7zwFgO+LyWnPN5Zqj0fbwQ9jOmevbsfiL5s1oU9ovTyu8CFMD4IVa69fMeAvemDBr73ABA+MEMAPvF+8z1Qwz29vJbHPFNTNTxLC+U8pKSnPcHCqb3khpO9ES3CPWNnt7xyxle+LhcwPeaoyr0FuTe+k5jlvUTiujxdSrc96eQhvoxtE71drjI9heNTvNs/876vRbg8aofGvWSVDz0ZkqW8j5CKvaJxFL6YU+u9WmBVvZBKOT5ulb88EWUpPVoLUD1YnRw+LMnmvUxdUj2biyC+m+WHvWVUTb4xI6C9P/KmvKIVrrxQtOI8ThyovF46TD3Z7am9FemJPSCwOL4q6CI+erMnvF1uoL2PjhI8zGgAPLG9yb1KhC29QDMmPoS5Db2m1ee9IUVuPKGYlb00X34+FY7Gu8Uybr2U6eS7Jnb0PTcV/z201Ga94jSBPbqihD00DYy9KJSPPMbBLT0cNhs9wACVPUyXNT4OxK69vTdqPbctGL0pNwq+4SFOPuoz7b2xtnA8z/8FPg7XfLycPgc+ADp6PQDGLL2t7de9mfgpvYHSUj2z1q88ykDdvT4BKb4bt9w99YlVvUOjGb751Gu7uoKvPWaReT2Kkoc8FuSxPF6/ZL0D0X69z8DDPJl7Mzz4SoY9edVOPjbEdL1Mo8g8aupgPXAWAzw7XMS9r+1bvcpYI772pvC7HR05PvWXZT2OL4I9nifwPbL4Fzq0qA4+DeuSPgtIp72cMio940ATvtrGDj15SGk+kc5cPb4dwT0sfxG+u38HO70fH72m6ne+KFjMPc1K1r3JP2e+iL2cvQzvCD3a6yW8NcCovU/zWb76WM09","gfIavrj/Mr0Q/fq9dXmRvXESGb3lCLK9od1wvdCvPr05TwA9GXNDPL1vF704HHa9Ep3BvR7sxz3FTCc+wt4FvOKNEj4Fv3W9eBenvB3mwjtQu+K87lROvkctYj7Umbi9WpEAPVW/Hr47kaW8VfX+vTN1EzyTf3o9eGIzvgg2PL1nrz++fm5vPXsrLj7c9jc+dcGPvU/nFz27CS27W15yPauvi72IiJO9EwD8PVtjdj0PCNg9XUenOjClzbvtNLi9or5hvfk3Wj6GMpc+vAKjvDvrEL2PkkA8AHbgvQa/rT23W2C9m/ucOkIKqryIdJo9Mu+CO2Rxiruhuj09r7OrvDehoTyngIc91XSBPRn2Ub6awJg9N4OvvSpi3rsxOx6757WWvVNh5j0HTwE+QvqgPbbdmDwu5wS+nNq7PYE3g72ZkLe8g0c8PZfxOT346wu9HFQZPf9Laj1xizY9mvl+PG5VTz4HV409iNW3Pdo6yTzFQ549LA7WPcWoSry5pKC9PBrKPFEFwbwMeMo9UylBPs1KuT1WIku9Stp8vlzJYz5eB4o7ds+JPW1ob7zH+K88Et9avUo0Db4UpKe9BsDbvf/ozbzuZYC8NqtKPtL8Er7BQi28oGGhvNk31b31RNW9dlRdvVCy0L2dnri8Q5MEvlt4rr0K3RU9csTLPEZ1kb3f2z4+VUkePklO3D2CiJY9eHdkPdfTMb210QS+lpAevkTysD2Zcn09bZoSPOTfR72j5uu9eqXcPJtLjj3k9Me9Qgdavdv/izzKcEI+N4kLPeo2Tz1ioNE9+4/pOuaNF72xDC07TibrPOLUFjzIq6G8w+Qevauj1bwK5Z49Bw6nPFz+h73Hch4+8871vJQ30j0WrLs8xnP6ulKXRj11qFi94i4+Pars1r1IoGk97H0pvhGUKjwRsWG9ItU7PkFQgD4u8OO9y5cwvoXHQj6yYC4+cfi6vQ6sAT5z7dU9szP6POtBnTxRcRQ9iPVAPuWMFL05wV+9B1PXPMFTDT2LeQ8+","xUTxPes3kzvzNUK74d8jPj+kMbxPwE8+hfjOvUqtkr3eklg+uQENvkIgFj11956+FwM0PXX3hj0iuT69uCcbvTSxJr16ngg8ec9cPaC6zjwPTB2+9xQOvuONg736p0C9KH6YPE8srb2TBia9DxAoPIs9570xt9K8xS6wPAXsqL1ZVj49iwSZPcoIw7sXNzM9Sp7PvHFqhT5/o9E8B7vYvXfk+L3XVBk+S0S9veu9t73XJTq+sf4DvYBL5LxvG4C81CDMPCT1jjviMl68VQpDPVDCND0ZRtu7k3/gPTqWHj5ZhUA+mFdCPueMB775WY09uAyuvfQ4GT0N4849PfZyvUH7jj1vgfE9dQqQPrbkej3l1qQ9qRofvkQsMz7tR+W987DUvQr3RD2b2KO+HKW5PYniwTxnvVs8DepdPSqACz01FpI9B+FLPscqJD7u38k9TzAOPd/zzL23urC91AsqvqZkOzymbjq+ug0FPg9/uL3H/Io9FlORvavHwzzohKM9/MMWOfD44zzmE6A9XIihPfkvMT3pYjo9eHc4vGKK/rqvzho+rZzbPcEieL1I6S8+rgumvZSyTL1uKkk9CFakPIj/Dz3uBdy9c/4FO4Iy7rxN4pC+22OivUPOGb6vXSa9bevbvPYhDj6zayM9x5o6PpeDBT4OcQI9M/Q8PRocxL3VgVQ8mORtPXki1D0xMi49xKIKPh3/cr5sFnG90MT0PSHumj0uU9M8D33TPXf3sD0QQ/C9IgLcuxAW1rtk/hW85bvHPXWZwb2Ra9e9T/1pPfhTpz2iHI29LjpevgNFPb5XesO9PfcNvqzScr5ACfW9TD5APS/jGr5CRN87rxEBPsJFO7x1zvi9Eoh6PQR/C76WZHk+U7USPXcyrD3Qa0s92p5cPScOir0TDCW+qiQAvlFeLT1ODXE9MZm/PScKhj28U8I8m/iEPdD64b2/veA9mfinvZC17TzdMbq9F3wcvoT71ryWYhi++0cPvoSlO73INYW9EAHWPavzFj0sHy69","PSZrvEJhlT2Iav09gTxHPQE88TzoFJ08E5osvm/Bb71+HvE95JNkvTNIOr5wlNu8Jf19Pe0lH7xI87E9moSxvOb/Ar20m289a5JMPs8o0z1p4oq9C53uPQ+/qLyjFcQ8lWqePSxDAz1JDo+9LDzOPer5Cz4zC589HlDHvIvr47xbz5Q9tHq7PbuRJT6BDlu9qIHiPIdb0D2CMGk8oaQ3Pfq/rDyMyum9S0brPd15dj7ui9C98GXeupa5iz0Q0Ig9itiFPdSKfD3GZVI9NGPhuwg7oDxjd5m8cmtbPianyr1bGjy97N0Mvp9F0T3IQsg9GWsevnw02r7N3bU9JuqwPS++lD3cqpy9vMnxvAFfTT6ltFq+CrUXPah3vzuXKRK+MO+sPRRYUL0S4Xy9J+yxvFkSYr6G98S9RIgrvaBhOT2/HOu9Gz5Mvi6P+r0E7BK8TVqYPLcScD3OrFk9N2qovfCKlrrxysO9E+XBvMwtBz4qIhk9eILhPFHa0r0t7J09Q0oMPC9NmTw8p2Q+QrhLPl6RYb3E8gg+prToveO9y7otAHw9qWGdvO4zqb39gQU83rbjvtG9yzqnTWW+lR4gvUMcMb4vWBe+tYmevW3BCD3x86O9CisivElygT0HT9K9gs/UvXRvaj7wkxo+xQHkvP1JRb2hGI49vxw3vXiPhD6hbcE8ppoGvinyjT2W3Um8xiUtvc9yiL0EgP+85i/wvPwpj73jPY68v3CcPQHX4TtsWkc9DG2hvWTE8z30Yg89C1rAvX27ML2BnNc9Q/YovYXrWTxWtUO9LbD4PGSayj19JHS9ezAovMuMhr3ZZCK9xkGzvYgDeb1HZwI+CLL3vnKAJL7KpmG+tI6EPVr7Or1B8eS9bNdHvbo6FD3VtpW8uL8CPQdf9TzASXO97wyKPQknVL7PVfu9WH+evdMHqT2S0Sk9/794vZPafzxlRKG9h4hevu1o7b0U2hu9QKWFOwlfgr2Wd0Y6wl3WPdsc0r22arc9o8EyvXtHDr0tPM67","N5YkPkGRLT3D5wC9MZHRPQvzsL3Y8Ua++bq8vaJKdb6D6FA9h36MPR8Qobxe30A+OM/vvV/6H71tta68tWdqPFgiRD6yEv49qHzFvWULIT3iBji9BmBSPTgXlj17E2O8m/uwvaW6E73j3vy9bHUCPloSVL5IMxA+KVK6vaYpjD6L2Zg9Eq7OPdEMnL2H7lU+bhH4vbdvObzkMQG+85pcPdLMDL4Gt9097IoqvTqPa706r2M9Xq6UPdCIL77Zxdc9m92XPeW3cL51W1M9yodWPDxH1D3QXtS99QU2PdX9FD6h3Qg+WOOZvf9N3j0bsQE+km8EPlEMCD39L4c+yZHVPS2Qo73OOBA+X28GPlPDDj02/Ca9dskIvhWJJz7czz297usuPnKEmL1pTQg+/agcPeQher3J4g0+xpuPPW/KxDzOE3g8kwMku9eQDz0tag49bTjovXe3jz0GchU+XiyBvNBTR7zrsxy9FBqlPV9QHT3icIG9GjWmvFbPxTwbjTI+65kCPkqhWz6s1Ei9RM/VvBi6/Ltdg+87uIu0vZkjAD5ZFN+8zzBVPGCu+LyBCA2+YrUtvBUl77mjZ7g8nJFjPZB/p7xSC669iXaqPZPbnb0QSTS+0iYHvqecvbwW+RE9dtpTvbKB0jzsE0a8ENl9PdGxLDxqsha9FzbuPQESmL3rUuK7dAzDvZqkgbuqV+A9Dd6kPce9QD6AxNo8IzSUPSBCVz7iOIE8P4nmPax46jxymZS+qkKyPVsm7D0Jjwo+mUB7PdvWqryHJCQ7tY7HvUzTSLxOdus9aUpmvm6B2Dv1FxC+TPJmvqcURzxIUyM+nDRqPkiz5DyGyja79yQPPuvroTwMrwU+JOr/vEjUk7yeqQs+73IZPAUywD0WS4e9/SUPvt/waD7EhJ48SPdqvsAiKL2CZx49yHeJvjKVZr3m/Me9Tk9uve3uFj7x3Q89VBf6PNDnDT7Q85s96jSAvb+TED3fjEo+4SFHvv1EXb1GE5Q96+k6PhznUz2C/Sa+","hCM+PiQ34T1J7Sc+FnQHPtQJiz1vOD29li0EPYUVmj3Z4wY93KdNvCUwAT6ATvG4e80FvOoeXDx2rm49VGEDvF2XIb1Q/aQ9zM9+PYtX2D0tQ948cvyiPV1e/bzQopA9ltIevcoRZLw/dLg9Eb4iO+HPM71TNGm9V4yZvZy92r2hZ3i9zHUVvT1oFz6MmRQ9VxSPvjqoijpttFU9bNdUvH2rAztCQoE7nv52PBNTkr7tF268Y76ou2XyKT127qS8HkQWvVxwA7yDbZe9L9QJPbfu8D0iDGa+Ouq0PSCvmz1khAO9uNCjvCFtkr1jxQA+IhjRPUnVOL2wUkK8zlZUPRZqnTzsDY+9zCvVvLRm3D1m8aU9Dl7oPZZLF77JDhc8rsjLO2eMMr1nzxS+aUR1vbUYMz6OLb+80LL0PSbu4bxtQbW8KeqPvRN90D2tmdQ8WOxDvpzorTyYCQo+AnhqvQb0v73/eV6+/88Ovdg8lj1Hlx69KeZJvZ6V270tpxM+SvHSvZ6a5rwEtiQ+b/73vJETpjyLPJg8ga0GuyqATL242Kc9Y4ftPZCWEL7Mx1W8iE6Ova1icr4lAQI7UksHvrvJPr1nzo89uyaXu/9Vtr01ZHM75TDrPVQbVz0ARkC+q7p0vWWsRD2MmiE8pe/DvCLGiLxUh4c9I7r5PYumyjl+LqK891FGPbODmL2FSkQ9ENVcPeVsB7yx+2q9qfAWPrLepL1gG+29hhhcPZNSuT3BBhW9l7S/vAb/ibyCqB0+M4GgOuSGp71teGS+v0P5O+zPy7xwOEm+SD9TPdRZ87zRIFo955lmvemGN70mvQS+cYA2veeDmD1HaTi94xCKvWM/jr3a+BK+M0AEvm87I72Q/Ay931SsPN/T0j1UQqe9XfJBPSpuHL2Qtb09cE5kvj2Opb0s5Dc9HqD8uwNKEz5GgR49/AZWvDbDPr46dKi9YcgYPoaW1j1wEp49n5+uvMJrt70z8f69HKrkvYBVjb3zwj4+V1EKPnhZP7xX4Cg+","tG6EPQswJz1j2by9ZLC1PJ6WhT3nREu9DbPVPI/9ob3nIoy9syw3vKTjlj0aKnO+r8AePtwEn73JGxC+mOrmvL+FK74z2989as8jvenjLD7jTT89Yxu4vbxk0T278ug8Gj4ivd+Qoz135Su+RsBmvLII8rw4dhW8jlOdvLSa6rwFU8M9X567PQEAsz0wfkq909S3vX7W5T2IPok95ybdPZNosD2BX4+97+ySPRHJGz7Gl5Q9h5Fsu+BJrT40lzE91BR7PROsMb4vuYq7zO4yPfPWeT2hoxa9Z3PPvYvMPL3vBZ09xahVvaqL0L2gFU093+2evIWtfDyvt5o7qDPivZcv3r1K+S0+x9HtvfX14jyEMxq+VhREPmxr+D3ZSbW8t125vZAsvT21AQ09P0DuvR20sT3Ai2Q+z5M4vqIAgT76jDo9wdXDPTxpir0T7py8I5OkvcdsFLxB1qU8AezWvHNkDL2GQF29SIgTvr5etb3qtTk+s63rvZiekT1aQUo+tKYgPeNUBT1nGQu+Pwhjvd2ZOT6ZxuK9AuFgPJ64Zb28AkA8Kzgavg0mKL7Gs4U9KWWtvBEnL75fQYg7ycSZvZm7nL1wBSY94YMmPk4EkLo4q0Y+BvMePUyyuD2aEo+9GIQHPuZF7z3ag5g9wp4PvPaKNr4rQqo8RvNUvVZMpTwhfou8s2VovSiyET1N8co96sGsvZznRj26ylC+Vcc4PRGRr70ERy690QszPb7nzDpLAOW7bgXyvHX3zbyD/Hu9u3bwPAXqJL3iowW+3IBXvaaSjb2Z+Y29LblSvPxRFr3o6BK9FKLpvSSqDL4oQQe+rA38vZevfbx2VNq9O2QoPSkOaT1s9BM7JmlsveShRb4jQRu+B84IvnglDr667vS8ge+VvfXf0b0lZxi8fbYDvXcWerwiHwY8MzSBu9YTMr6Tqac8Lk6fvUwpET1ydOy9sP27vQoIYD3MApE8eRYJPg+tWz1j+Yo9CIo9vkRTaL31Y+O9t3EfvZCwwL3gg9C9","8IC6vd3bmb2WGek93BCmvXXIaTuiLBa9jXsHPnWU+D27o9c7BCjAPAplRL7yOZY9y608vRornj0RSyu+FTREPcrRjLw9WgW7pZrqvbE5Rb2lxkG7nHp1PZtiH76BRaa8tX5fvuq0lL0BJhK9LrRWvcGo4b3GWvS8qBBnPQqICzvlmDy97FAMPF8ZTj0JjdW9wSUtPlmtXb7Fudy9trVfvYT+iTs9Rss9jIQEvjmZJb5GaEo9tNQ1PUfpBL6DSy6+ZhmCPQ0mhb2ofoc7YqT4PSSG0r34dHi8ZRGuPeeqg7wa4PA8KPHXve0LsLztL6Y9TWAFujzK0L2j9xw9+y0PvmHyl73nBVW9hVEGvtDbij3uSHe9ScrLvBwcPLwubNA8xjH+PZ53kb13kFe+4wojvi00Gz6Fhjc8nMlau6IdTD2Gx1c9un5YviRnCL38mx69/8MCvgq88b1VLKs9+FEEOlnKQr1Gnra9crbiPXp3/Ty8SAy+4CetPHDWVL3FswA+QHXuPWM/9Lw85Bu+WtY1PvGDlDrtHyU+VOqgPG5lKj3JR2O+TaKfPUXFaD2dC0I9CAabPY1kGb7Ieyq9CiuTPUr0jr0l1YC8ntTPvUg4KDyAf9q9bQgWPcSCqT0Jhpy9NJMWvjFbeb3k3549CoU2Peyt6rwzIAk9M9c3PUboLj2K0Ha9P7njvUtF4r3Mpj+9PwgMvvNd1b12RrC9l63WvSWRrz3w4rS97H3OvDj+bTywMMI83MyBvIeagj1ltxu+dJ/MvMA3ib2C5F89Bg/4PH0uIL41CjM9OC2SvnQa87xRbiQ8cLUIvjWox70MhhC9yRa0PDM0Db5jO3y8Q3TWPRBoi70EmV49fuppvlUT7ztBiMO8A0zuPV84s71FUgW80vqRPBYkaz65jIK9cpGrviZER77nevS87mPbvG4Enr1PjIY90jWXvZi0eD0qE1a8FbI6PTlDX74NyJE8EBTEvVPYlb11B3U7Sw4fvoEdJj1PZSq9eWqTvdlohrvH5ea9","CJKTvQa3lj0Pghu+pKYgPnYzGL8S89y9mVtYvjjyCb42I8K+VSVmPr8x1733Z5w+L3+hvSqTTr26vCY9tky5PPf3Eb5aB/O9rOA1PSog8T10s4G+792ivdwD6z6Gula94zQCPpmMwDzdNdu85g84Pq0jvjubVI+7Cgr7vCiDO7xvMaY7lTohvlov+7uZD2m+7d8JvbFSBD6JoYi+eg62PGZA1j11X9Q9KBDtvdFuUr7VSaW8MJV7Pgc7/b3Md009V7HDPBTqJb0VX6a95zOJvgRsur13hFu8k+TKPGdwCT2O2KE9Qftdvqeqfj5VYrq9nJ3iOwSBBT0/Sr6+Ro0kPdQnLD7kAku9UgQovmMmpTxJ3qC+HMxkPr6I8z3XuIg9mjLyvKqpKD6SUbS9kmGXvQs0xzz+zVK9k2obPokyfD4iYo+9TgUlvl5hnj5iHMW9qyK9ve0oPT30fUq+sFepvTqTdL6GIIo9BxXNvSq4Cj417I+9T4OHvSPBwD6+6Y68ICHUPECNiT2vRiS/rxboOxu51723EYm+Ypn2vaG4D73l8Cy+vU44Pk3YCr5t85g+cNgKPtzHAT0EsqA9kiBVvcr+XL2MNQe+PZQZPu+15b2tFBo++esJPVslhT67sFw+H0kBvh2wX71IYUY+NAmivQSi67wwDTk9P6ZEvnDA0L2tcoO+PUnlOy0euTs0Hfu9B1zvvdMJDj4Liqq8E9WXPhzi1TuQj7695XJCvptbDb657I68tXf1PiHZ8bz0ULU9DRNMPgCgvjxkesU9FoSkvY8QjL3OABA+DzOFPbVqcz6U9uu9oWVevpIOSj4Fi8W+KWv+vermfb2AzuA9eTXMvR9otj4dsA0/AfAdPguBF71beKQ9BF/DvkLvBjylLAg+li9Zvq6hbLwCss6678CFvnqI3r1iDk6+g0ypPQ2evT0KMJ299GERPnpQm7xcVyI9SgBEPdBVnL1HEWA+bqmcvvhjUD609uc7lEGJPZXr9D3l3cG9bL9YPQ6BOL1OHws+","/js3Pkx8n727qVI+qc5MPcjusb4Zu4++TkMVPp/UgL6gZdI8h+unPWMl4L1SzL69dll3vekiSL7kRRm+IWpCPQEZQr7f+dy8Mbh1vUwBlz6aVz++bDpmvjRpuL2yAeU8J02vPTyGLj4Nihm+7VxgvSwDGT9OI4m+L5oXvgJMTj4qGLu9/WHqPItXoz08t4e8BVwsPtmh1j3l20o9JSyHPbHRpj6QGLS9ymQ+PohVZj71/Y894gf5vfVFmD7hrgi+Pd4uvmikdz1S8Yu+kWCAvhJQlr5XEYs5f4buvL75xDzz9o4+QyxWP8oldz2aJk++qAW+PY0AKj56twK+9ABMvr09wzw9ERM+He7Wu1TkMTyZ/Pw9RWlpPWM6u709Aqc9ShGHvaljQT4KS5k9E72GvUiiGru+VBY8WKdOPB/SrL2sYQo+rDnePV02Lj2XAU89VIRyPSYjrL2e2mq7P0O8Pesznj1oSEO+17UTvk+TAb74fwi+/4d7vQwmCL3RzyG9qDG+O5y1nT2DppG9WKEyuvMu/71J58S8ZNu4PceR+rzwdy2+YO3duxfeNTw54RC+7Q3AvTXlRrx3X0s82uMcvYmA7r0zxSK7dxe7PaUk8zyf5hQ8HwbEPIJyNj1s44E9Wr4VvNUl7D1vbuM9mU3xPRysDr0uv1W9hFDcvNajjL2RcJu8GxrQvPjc2DzAgK88iyxzvEYd5bzlTxI+Q7u6Pa/JEr7il1Q83nvDPOu2YT3gkWq9wKOCvV6GCz3vMpS8iItlvAeAZj3pVpc94NBnvMvDdj0hvu68881PvTLTkz1R3pi9KEi1ve+bcr0sGaY9EqEaPgy0Br4qd8q89DRCvcfuZL3RfPk90zvcvJ2bDr14NBm6cTyfPbuSSr2jnGG9yk5pO5zc+zw1XEc9kDbYvNuwqD0UwsY9zwu/PX+GtT3/Wxg90Ky1vZdG2b2z3yg+h1DhveAejL3VorE9tWirvQ+yN72IbvM7iSMgvoY6n7xHvR271BMDvf5Jwb04xcs9","HpMqPSJlCL48pou9LyqIvK5D6r0PAri9mPhdPetcnryR2R49RtYSPgfCc736ow88dA1Zvtx19DuP0Bq9sVMYvdDr+b1oAj+95nLIvXhNTL1mX4Y9OoEOvpk4lDuIQJC88bW9vcmCk7ynhOm8Qh/MPecuub0n4by97x/LvfRBJT3z28i8a1TMvXODGjyDQk2+mh5fO6XYgr4bx1K9BHY6PVi+l7xrOT29sfvSvSw7sT0PTMy9mKkiPQmFsLudFeC8a6cgPeR5O72jEf090a/RPS7IJT0h2cE8EpQdvqXEOT3iNnI9v63pPfL4sz0fHyg9tQ7ivG78WDwV+xW9zXjfPV2X0b35Bxq+uKpWvaFVOT1bDFG9nsF4PUKieT23NP08jKcVvnhx7r1bHqo9PUugvRLzab33bLC8KdCTPKM7Lr26n9I9A99oPBFZC7063uy7DtChPYUvlr2ujqc7go3OvYQbr72FcSc+BuNRPS5YPrzqaB+96Faqvb/PEb3A7d69N6YNPMf0JT2LiIy9skqjvQaSD72gM9m9AYsYPCrMR73iE6o9VGW0vd7PMD7QLk+9ErwvPZwuND3k+2m9GgJ7vCXhG7zXZmO9VOagPTUXHr7I1lU9474PPdePqL1vhmi9w28yu3G/Hb5HDw09fsLkvRXMN73oZDI7qKafvQZnCz0="],"bias":["3mrDvuq8tr6MYVM9r2oBvnojhr4Ieik+mqrLvtAqUb5yFYu+AHoPumZ4Er7ySqK+bPSmvyxumr0AKKk9AGd0PIDxb7sQl7M8AExFu/D/ML3027E9/6+KvvsAVb5A/dm7DbZcvljIqb6o28Y92ISMvYDSPjydzaC+pkYfPhjJGj2Ab2c94FySPCgeoT1DP9m+CA2FvsY5xD6UwQO+uGYlPhoeSj4oW8C9hKrmvYCBDj6ehk6+isX/vvBRwz1oHdK98Ew9vex4GD5UkFI+uKxRPW44Jr7AMwA8QKyxu1fQk75ghBi84H0rPibTlb4Iws09nEQ/vuDefD08mOE9DxFlPoD+drxAvh6/LpnoPeJI3r2Y2YQ9yND8ve2p5b7IUB29wvOgvvqtFb6aae09gX2xvSRUX72KdCS+LdAtvvK7270KupA9xzECvqBQhLyAKt09poqbPaDon7zh506+8FAhvWDPNLyAolq7hLCGvskqwL+EpCy+zmW6vlhUjb0IOxC9xiqpvc3+a78o7KC9Tn3evRXTHL4AGlc9iMZzPj5Vwr0E24W+90fAvoCecT3Q+oQ9oENfvcqmrr1ooZK9yGvePZIwYL44Sm89WhmxvvDDo7zEoJ2+gsbXvdTlkb0j85S+CSOGvTgOIr1gIHU9lRktvrD3jz3yPjG/8KYvvRh7Cz042W29Tpp0PrhTNL147QO+cMKjPEzWoj12YAY+MAO1vUgRd76TkqO9xoipPiAiXLwg+W694EvkvRKKar5s84o9com7vsCyfj3kYxK+OLKxvQC/sDq24cE9yJQFvsitwb5sqJ69rFuovRBGj75+Qns+Gy4xvoixGT6g8bQ9l9SZvoIYwr1Uiye/BmTCPdzoFD5N7Fu/8Ff1PTgH37zhADa+oCFWPXzMl714W9C9yC/zPUoMab7MTQ89KK1hPS9Xi74yqRS+3FQ1vmDjML3QfRy90AWrvMHRFb+JJEA+AJ/5uxyfsb6Iyky95KgZPtQusr+4Xwi+IH0wPNTwur2p4TG+","ZC5Vvvb6sD6EEri9gIe+v2gWZj3ud1A+aPafvFD70TyT8ny+nh8QvxoYBr6xzzm+0H+6PQAskTwnym2+ynaivdAdaL3ieQa/MIm4PJ57Qr4A0n+6dFKpvXDYjD4AKJO+DjrcvgAwAbk5ZAQ+gP8PvBsVm75ARJ08AJSmuVgRGj5wPtg8YMUAPcAPUj0gzAy8Gusgvjpmxr0oBli+6ilEvvxRuL2EgV2+AH4cunBeFL2tIlu+1uMMPvV8Qb4crVa+QIzxO0D8szy0eoc9KmUxPtyIej6eeKQ+aD+HvZr1cL4AqZ47OsMnv4jjGb00cVa+qtSPvjT5pr0A7P85BePzvQ=="]},"dense_4":{"weights":["55SdvKoPXz3ya7I6IEKFvPOBHT3O5m69+HIRPtMidj3/Jpq5InujvRaejb3lH5C+o4OXvZGeozw026O7ODJ7Pds8aj2iQ/09D/givrGKSb0QEnA8BCLRvd0gbrvrqAm+5L2SvulzmT3BjFu78lI4vEFODjzlAMg9Eygqvie7vL1/8Yg9gFJSPILtIr0wcCQ9WfQ5vA/UC70pcoC9kReWvBdCQ70dtm+8Yr0iPSweNLulwjE9ys1EPAflAz3F8L682M2iPZDrIL3XOBs9YULCPMtAEb5wpHm8nYUWPOjM/jq9o+8862lCvpXw/j1F5uQ8QQmPvpsq2rwo4OK8xIpRvSJTITu7aT086j6DvKcKK7zVT749hxEMu4FPTjthyyW7dKlZPPpXwr6+eQg9kOCmvuSKF7vxb5y6Ns6jO6qifjsEDkW8xJhJPD49er597BY8Bo4nu59bqr6ajkY7r3Ervp3J7DtbA0y9Vxozv4shDLxE74S8rg4CPgmicz5kkgi/HtJHPcRuObyoNe07GRq5uts+YD39+JW8BCvoO9Tbh7x3Gk87Co+GvGnNQb15ROq7p6qNujqiRzwNvyg8pe40vOrYP7tBzx488vQHOy/JVTuMIAE8aim9PXxdtjsq1nq7GkyqvD/G+L4iFdy6Ce1TvHOw6L7Szhm8ElKNPCHG9Tyy5GM6LaQDv9sZBribfq27vRBrPanpQDz/N4E8/OWMvKViqjvaDuS9PIDGO0VeITx3v6y76SwNPDNxgztLzp46ZXdmO5RdrLzRLmy9YoIRu0ISiTg0YBS9qX42u0hGCD0xz4Y8W/WyPJabKL3vONs7DvOAu87QQDt4bWK8TNm8PVFzPDx3PRK7a44/POPxorwSE9M5TPTKugPI77x4IgI61G8xPGPcwL7Xmio8hZu9ukcgOLxD+8M7plUuv+WkljoonDI7dfcePIMbz7ugJc67a3fKO8uQ4TzZkoa7ShyFu/wsKry0iAc8OYdou9djf7u/Ncg9H6wRPU5dETsZBQG/","WYGSvEAviL2sQh28/tSJvQwombznd647x006vKyTBrxYkY28JQPCOgjKAb8c0uW85eVnPPhoj7wTG0u/BgpfPBCwIL92dIC5jAgtvglvIzx/pS88d1gVO+p97jv1XLW8/SuSvDEfZjsPJXW8Q3UdOzsaaTzN8H09baeAviVk17zH7se7uentO/+ZAb2W7wm7m2wQPVJOBTxs3Ci8bnxDO6YrbTv2zgc9jdM6vx9WZDyFqYk84gj+OxYylTv63wC7MAyCvM0YlrxeQou7BcBIPEBpS7twe686cigivBF0E7xd75W7enqqvdTj3DySqxa89Ql5O0+5gLyVrie7Yo1CO03uirz7MX28QNrsPEGzAz72Wlm9QZqwPdt+ET0GV048I1qJvLZGrbyvPd29rlY+PUy+hDw43pa9D6GnPFkVhzt6ehq96uNgvK1LJD2YfSi92/9+PTwEtrwBY2u8yD08PIqwGj6HV1E6fRqqvPsQcLwlOwc9JHMWPWiOhrxBDlq7MyXkvI9VCj2Dgjm9d2rWPQfDKjxnkaA9ODAbPQnOJz2elZI9zk5CvTfH/DxLAKo8V9MJPnBJTT3TA6u8qrtYvVczfb1FRmi9ZoeZvboKfr0jZji9zYZjPbYCDz3dXT0+TAwNPGmdCLyE2tO8wp+fPItzhD19gmm9nwXjPQcTtbzAPK89pQxtPVY0UzxUvW+8MkTfvcjVIjvO4qi+BNSUOyWeYLx59qU7diXUvSI+yzyJzb+9PDKPvXT65bw08cK9IVWmvnAgf7sKIRQ9ms+jPWhigz3YVbe8DFO1OysAiTy+BUO9mPnEu0rST70YTow8/IUHPLT1SzzWFI6+r4vgO3MdqTtUvVG9GblfvqriVDtg2Ne8bcGePSAD/DqdwoQ8rl9dvCP6WD0Myha9VLypPAnyRzsB4To9mhd8vHh8QDzEz1U8zQ+cvTH5MD0CMle9BWJjvfu3w71ALDS9YbyevI7hEz1tlT+87cAUPWi/Xz0lLrk7AsgpPKtwmr2KtAe9","ImUAv7IO8jpA7oa+0VfzvOtVYrsKoRU54Ju1PIsUY7s2NeO6LPo8PH3KiLs9yqE72U5HvDrdxb7OAX27iNKou1R0hzydFzA8Syb/u+GTFj5EFhe/6LCKukujVb679mw7nR9dvXx9gbv2xhi8kKw/v5qLwL6lXpK8k2gGu6XNW7phsJ07qRwBvFO/2juwwk47Khv5vsKAeD4mn0A8NESCvUI7vb6POti7lfZavGxo1zzrAD48zFJ+Po7RJrwQEs29W6ITvR+Lqr5w9ou+x1vMOzbajTu2bwq5UXLyOyYKKz7/bRa/ejKnPCh6Izw+YB27W5lwPIJilzweeDu+oa4CPKXcNjwawwy94ueYPS8Xrj2WmSk9oYd9vGQcI76Ofjc84eo0PSPHCTzzGUW8xyGsPKvi37y/vOm7z1+mvb6kzryXSoO90jwuPGOgGrwlHxK+M2vqvGHbaLwe9Yu9YEqYvCp5ij3+bBS8z8S3PHSRsbyoRBk9WbEkPgGfWL2nxEg85H+mPL19qT3sbC08XP1kPaYW9z1PuqO9gKvbPESclbwOEr07IyfruU6g8byph4+9apMtvUWOfzvZf+S8Qdchvd0FqbyPqMw5ng6bvN0LFjzw3lm7ps0VPY83kjyf7wk8j7OBPRU4tTxI6cC7BITwPEdDrDql75q8Q+/zvH2o0Dt2hyM9g0meO/jPQz3KLTq+Ht+nPXdRCr2Hkcs96nQgPcoy2Lyw5AS9VNr4vGsAcL3qf4Y79L3RvHnnkT3Xx568hZcZPlmjz7xeGeA9vVcZvE6toT3RETK8d8MqPbxn6b3peJK9uPSBPekUkb3Ib+49XDC0PeOKo7xlSAU8OpSpvCvAc7x0Wlc91ASsPejeET0h32o8fA7mvbSSGL2aa8i88rI0PDdrF70Ne0A9/WH8Ond8Rj36L5C8nU4XvY6sxbwrGz69vO4VPNW6mLxBiSc+FI7sPMNZurwz+oy+xnv0vPkqH726FMq7HzazOq7LC71h+o2+IKe1OoVhwLsESWi6","4AwpPD8kMb5LBQq9SyJ9vCQfv73IVMG9/Mhmvs8zqDwvP6K8K5sEuou8JD60ZsI9gpv+vJjOdr27oVO9J+Y6vU/kBT7DdYK+pykVvu8dsj0xQLM8VqitO9GkGz3N4os9RHQZvZaTgjzjdnA96ivHvOvmOb3UWEa9iV/kvW4Sx70FtAM9Db6mPfVr0r3TTaQ80hsvvRLGyL2ZPxm92+X6vALegD1tuDQ9mZ1NvoNdv7zniJK9dWr8u0jfiL4go6k83tHbvH+T0LuX5zw9jjgHvm4t4T2uRw2+FXNivZDd/r2idzy8fJ7sPZLjtjx72ve9115fvEyE2byF7Se92hsWPdSyJDuTQQk7gA/EPf2ivb3MvvM91gbOvHVJmr7BE2e9U5j8OyhUB70JaBY++nrqvPk3Cj3OPxq9LA37vEWI/T3XiS87KKD+vQBiCD3lbxo8Kr8tPKAHlj0WWkG9wowUPAw1+zw0I3K+FBnSO7Es+7xKXos9gYCVvCPpAz15sT8+6gz1PZsqCjx8MfI91JAPveOyKD2U1yE+DgIKvRkF6Tuk4p09EPpaPu3RBj3NM5S95sCePc7lJ73tEBU8AsOqvfJUtDxZ8me8BTmaPRnZ1boX/Uo8IYW4PGdZUrxPGDy8/4ePPUG7Hb4Mpts9zapnPG0nZb0p6gm9XoYju57Ij70r3bK8yvKhPeaOlz0k1cK7jv6IPcUKxzxxb5E+ki7lPZiqPTzAHaC9DjfyvQYNnjwPeJm9Iy/TvMT/IT7x+rA940lkPejcyj3hyi29SpquvFSqJz1tHVE8IyA0vTD47r0xF5w8rHqnvO3CtDylhec9zrMevcK54rygPdM8FJmEvIYbgr5ueMc9qhz5vRFrxr0WpR89TFpCvTTLsr0qMKy9JC+RPcbWB748B/u9hbb+vZL9rT3gct49I/b6vLooeb2UahI95sf+vMAhLL0qsAU99FGlvc05zz17fKS98tQWPDZtUr2z3LS9og+IPd6xu705zhY8kXYYvWMb9LxP1r+9","YgjMvB8gUjwLUsM7o7KePKK1sj3kN0098D0RPkgwwrtDM+29RWGGvdaiWb2Xy4O9FjdHO7xKVryj4M89q9SWPAtdBb2nSXs8iDH0OmFcYruX20O+ZG+FPDPUVLy1f6w7WLPYPWb+PTwWUQA+/bHlvLVrnbxIh4o9CnHJO8q80b1r1bK9BA7qu+q7cLxkPXu84SjOPL8Jhb09dJy9Wpo8PKuK3btzbAa9zZmbPeSq4z2hYa27Ax6uPNBolzyyH4W8851iu8jHD71bP6s9YAoRPTwz3z3ziZO9lNAqvkvJbrw/8p+81fD+PSXXjTtiVK299/AtPku9FT1zgSA9r/ZjvDmLhzyBYpu9V4KUPUvUrrtlnJU+MLYpPmG2h72Yiws9OoqUvfXkDb9kyAk9VVdnv+jrWbwqgKW8UIfIO1ostjxP+aO9CrLkOXgkTr6g/w+8zXw9vVP4ML7vO4G8MB0xvhK+Njx3iz09pv6UvqI8PTzXbwe8PGeKvb0WUj9tDtW+1lHtvlck5bsnfYa9dxIRPUT9wT3wpje8GxaLOpuiabzLNNg7sL33u6EAFz33Eby8UWXLPPl6RTwUaRY9nji2O1H/6rzi0sm8kJbsPEM+oT0zm4I9RVVsvvKlBL0Uy1c8PW2UvFDB27+eod689R9/vHLppD8wx588My88vFATPDxoRpa99l6LuhWIWbzoS648tzSLvug1W73m+yi/dZblvFbPAzzOxxw7mAMivJjD2zuGjRw9Y7YDPWrzNrw8JD49q2XJvaigKb7jJu89KZVLPmhqZjsAmuS8Vbz5vA0glj3heHo8QKBKPfJMtLyVoKw9jONZvRFUL8DOGBs/n7CXPZGpJb00wdG8ZHlYPONYjD3a9IG9HhKBPbzC4LsA8zk9TsChPEWYBb6dswQ+d41rPCtZkD1XKso7WvkEvYo4jz2MzSc6WXolvGJvNr11n0M+VnhQvafwhL1qZRY8UecTPnt/uDudXXW+e+sgvcvSFbvoeU49F/s0PQ/nMz0HoMe9","UENNvLBM8LxP31A90rnuvX89UL1XDUu8TjNIvmOOZT129Ru8dypPvWSd3bzqc349SodzPMb8fT2MYlc9tinsvYrwXL0v/lO9iusaOzfsEj2bAK69esSTveJkoTsv5Fw9cDuavanTBz3ezQY8pK0qvdnv2zxyR2e8niYNPpq4Ib6bWWc8AStLPcOzhb4B1Eq8qWGCvAdLJLySBeK8rHgWPS/3hzwDxxY9/7sKvNaPlbz2ZhU+dFQtPMZmg71VR2Q89gYGPRflDT0Z+Cw9TcSqvZvdSr1g+us9sJZbvRkt4LxPozo7fdepvbCklj1/sPk8lqZVPZ2CZ73daUW7gni9vA4uDLsgYwy9DU33PC8LJDxjnqs9IRYlPJ8qkbzevGg86kIKv+IvqT3xTkg98U2KvZY+i7tq+TI8GCpqPEyVCrwhwSm7hNmSvDssUb7VPM684h91vujnkr14klA8xFygvbfpEL89oVG9+57LPbzUmDoRrGG8ej2FO28B7D3bYrW99ps5PCc2tTuGjCo9V3A2vOVsvT31sIw95A9+PK9JiTz8zg48NiWuvFsYDT1f2gm9UkG7PMR0vbvt5iU8692ePKhPJTyvqko8JdTCu7Epb72DyNQ87BB+vmD7qL3Dal+77e1TvhAFvz4+f8g8eV2JPXS1RT1Cww88aVU7PF6G9Txnztw8CWo9PUb9CL3nUne81x3HPQDyK70oB5c7KP8MPbYNljvsjzu9qjicPUnRUr5U+V29Ikp4uwVeJj2bHYI9sCdjPbSd+D1G+129CGocPbCq/z22Coe8Yry0vC5BQbwSIy2+GEfbOrTGrrytZa89XHn0PDF/ljpbICS9ieMNPkD/Jj2Q4AG8wcwMvNPw4zwYV0A+VuUWPd7VR7wFKjK9D1zUPDd0OT3PMam9hzAMPS6MDTxKroc9Xp4YveUngD1hI0W8z3JKPCnKDj1NXng9hzJIvpVF/j0guQk97GBKPG0jbD3hHmA+wNh1vKMAWz3JWfa9g1zCuwf0Cjxt1X08","Nn88vR2ELrtnlh09Mf3aPD3Li70TWQ+9sJ8RvSq02zpaNwU8lxsKPZ50lDzHsnC8NV7APGPt8L3+OV89zqIEvSolszwKLL+7ucncPLFMwz3+EQU8buLBvC/NbLzzbyY7X5K3vKsUAz0KWVQ7ZNtWPZ5YzD1odhQ8UoaFvPuBUbwsYZg8pH2jOv4NB7y2nqU8qfZEPRZS/jxiaLE7r/FSvT/ckr0nVaw9gE5NvF8nbb0EAXA8GEayvNKmNL2EoXY6FO6WO2kjrr04+A48RkSTPNyn8js4CkI9TZyYvFOMH76OJzA9iUGGvXn60bxldxw9jqCWPZXTTbwJHoU93T+MvKU9m70fie29dysDPu+END6930+8yXvePg6CVr7305S9htlQvLVWu74ZgDG9Ah9dvRHPNT2Aplq8T/7rvZsuOL1+Lki+/IM3PdgCU7xp8ZC9IGbpPIaIyb2sTYu8EKjuvVuBOD2PzjU9fa+YPcO7lT1dxae7/jZHPq0yRT/AuBS+ifHbvcGNkT1BFyS+7lvNu/yaNLwOh5U7vA0fvYAU6TwpZRo9W0OKPThZ+7yMeEK9X4zWvadRzL2o0OW8M7q+vK/cBb5AOGM9XT2NvAG//TwtPyc+byvEO79DTT4PWlq9etKcO3duCb96RIi9W3QqvPtb2z7aCfO5SU6hPVHLPr5TLIm7nEjvO4pALr45UiU8291GO54cgzwG1TG+CpaEPF0HMjrtVNQ81IGHOtBvKTsE83c9JlScvstZ4Ltphb+7uu3pvCHeUzzwTea8cj+UvhKCGbx3wAq8ykievrn0NbojiFO9xru4PIfI5Lv2xPm9eGkFvib1T7x+IxU9zVnYurs1rTtdCSK9fR2GPaGobrvjj5m8QYyVvgnLurzJsHA8lNfAvcpkijwGgig9ab2tvB4LNTymFfu9onAVO/LLJr5iGbu+R8SMu98hnjxeuwW87GywPGo0gLu4dmK8iVQzPtQMRrxd+08839XMOnpezr2WTKK8UFUhviX0Frw5uXg7","r/29vTNR2ju09zC9Z7+7uxLTibxJaJE8ZFu4OwgdhLzr5pc8QGiMPBZtv7wObZ+8FHe3u9Cb5TzO5o+8Vy8bvJa9szw/auw5dBv1OvZpkrxTwjS/4C/iO0FdB70qPa87/KnOvHMZfjw+WIW8GBEvvgAMv7zHUQc8YPWtu/dd+7s31Y67HwsFvb+REL12SJI85Z1wvgf5ej3v+8I8f8k7PXq5AT3BH5I8KPugO9QtsL2GxaC8kJErvVtX37zcxo69hZaavFxsf71zmsg8nY2AvP93zzwu3By8RiCIPZoswLwtabe+gfwcPY/J7DuzMEW8Jb4XvUd9azxlB0O+3KpOOp9lsTxp6le811ELvU/cwb2xCQ6+heM1vKkLuTxaHUs8EyaGvcqsET07WhC+a4eEPARIZT1/tD48UoEyvi3el7st3Yy+Ww2tPNqKk72OuOe97UGiPUbVajx22WA8spPNuw18wj20NZW9gXlaO6J+RDxaKU492St8PSMITr+y6tu+6nkjPRRLvTugNT4+M1zPPNe2/DkjY149taWPvdI7hzxVzzi8rxw/PLFRXT2/3EE8MJesvQ4omTwk5ce947TAPAgqhbzuzbA8tohYvW8MAj1P0LW8RghZPVfkjb1Hw+K8oIi8uxbOTD4ZsTK9xa+QvEVreT1ISMO7A32dO+0YIj38W7I8W6wevVOfgz38IaQ8/SFQv/deDD0ZgrC+h9qGPEBiyLyTiLm9udoivSHuK7927TQ87tJIvL2XlL3BLGS8vrQAvoagiTyl862+KAuFPFQ0tLt6qYG+EpflupUejLwTIDG90L41PkCOXL7CdmM9zqZkPP6DAr+AqFw/Huy0vupoNr3CgA+8lWX6vXYShDupQse81a+tvLSwsL0E0aO71BVxvA0UHD39xZC9gqjDPJjsSLxsLiq8o86wPM+347urQ6E8VKstu0OuCr2y6Ji8Xde5u4guCL4frAq8BAgevErXtztSz7a/jByluudehjvJ8hg/PKG7PCSuGT1H5xW8","Lcxku/zrNTwA1UQ6LQZjO0FJQ769c5G7ovWhO/uJLztQKRC8orQAv5Y2gbsy8z6+ShcfvJsCkLuPBO67T8knu7/sn7zxsWw6l8+nvseb7LtCiOq63IcSv1ksETnnFhq/STZ7vKSKwLzs4K++OP2jOgqsyrsSipu7Xuf6vjNWz7wTtpE8z5fsukzWDLu1itu6vf/UPFGRJjzcJtC6SDqku3UZlLqZ09o6mojOO0f8FTz1yTS8/vOoOSDfMbw34Fg8aN/MupI1CzxdCAM7hhwfvFjx7bqLkRy/cHYLvAF2mrsJay67bXgzu0toHbyZYRq8qEOdvo+N67qEipC76ZNku766JD6c9Ki7QSGGvYLmjr0F8De9yGIzvVyJNj4v9dK8rcgMvmSWlz2c1D48kSG0vuXaiz3ta7K9NcOZvVRDtr2bt6+5pjWCPBrDXL4DzrW9f1civbxrvTsQFcy9wkgqPTuDnDz6qh49ADYkPTiW3D1Nsw09N8hvvSQEljxxcHI+nmQVPn0/lD0asWm98pNFvPu4Aj0SazK7f0W0vdCEGDzlLra8+MLkvYg0bb6rkG28G2qrvW7ocL1cYwA9mWhQvRB7MLx6lo69xlAHPA8J47zbxNQ9d9rBu9L6fr0mtoq7VTSuvesejj6LJ6c9uiqovV/RUj7amJ891EYePDKXlT00ZlG7cGDXOqigUb3kdnI9rn2yvNtMJ73VW7q9+FgbveDOpD1pWty80ZAxvcIIpD24HAE+zOgBPaU1QDtueaO9rZj7PHfwWbuubjy9iwoAPU/CAD4bAAw7lGQzPZwfg7zBUI+9/27mPKvKLb23pBu+zb1bve+cd71D9su9Q4KJvBMFib3Rpa+8gHeDPIQsxr140xc9NZtNOhDpaz3tBUY9vQrTPOSrfb3x6rI7r5lFO3b9KD4bcGu9fIJRvU+ZmLzVVX+8IGLbPTU8xj28Q6O8hkgGPP/xxjxwzY+8x7usvdAfWTy7FG48GpzTOyfW7r2RvM28jlFavcI6CD7XqaG7","WQagPKdObL2+b+a9rmlzvT+rwT0FRUo91gnQvFJDpbwhoEw+Zy5dPZAOPL3/C/69TOYtvnGkgD2pzV08LGqqvZCXwbtzQ9w9GS7hvdXekzwqCtm8swhivZZCyz2EH0Y9aj9uvdd5mz1q4jk8mxMAPjXGsj3dhxA9v0LJPSHe5D0deS69jySdPfAOoj04Qrc9RcwRPf6fir1B+NA9Vih5vWJQIL2uJ748jgGdPQ5mgT2jyf+8pCnFPehQEz1kz7W8cFs/PKuhQb3toaK8rPm6PURlQT1kKjm9rJBcPYoM+btDO4Y9KbhtvSJjnjyJLsM9UN4KPbFPVT0AvCM+OOAhPa3qcLs6HfC8tZVxPVkZjb27hBi9LOmxPItf5D2TWVc9sNYgPUbTEr1DN8a7zjUgPUjAXb3T49e95wqJPcQFZDw8HCu8btxkvpBMwzyVOmu+PeQPvDvGwj1QVAE9H0IPvZ9+TrwreSe9cB5APE4ECT1rKFY98VofPRDFwj3i30W9D3OzPPigYT0x+ua86eOBPEvS67wPztS9SORJvcUrZ73KOYA80+wHPcCDpz1z3DU8H8FPPkr+ND00qRs9Vq4xvSAXR714io889ghSvXkJMT56LLS918ffvDjYjTyt72m88q7IPWK/j721M6a9u5QbvvKP171sj5y7QLhlvZKnxjyDOeu6d5gbu0UrKjsU8iS8dzaKvAJjir/AEoq8iMEMPCK1sTtBSKm7tRZuPCM3ZDot6Qw71xCgO2Es2btXRQK8h8qcvLu5zzuYHpa8K2mVO7o69Dv6egW8bwR5O8JSBjtRYqI6rnWkPIul3Dp55um7rq6ROrw/vDv1rog82abXOgeBSDvKQwe8nXFyPNCaLrurP3Y6n0G6uRx0Uzs+qX07yt6Fu+NDAjxe9C87oBtIuy6fSrzzRhI8mF/dOnFSFrr5Wo07/VS4O/dDhztDYjy7RsWIuwNCWbvWDwE77SpdutF1IznzbCC8yu2uO/7lzjrg22m/aJGvupYBnzt+VYs7","kaECPXHwtzxIMd+9J3SKvm/+mDyfRhG8xo3IvqbEML3Anea7chwgvdyFjD3zIRa9eENDvcLOmTzZWQG8xtfPurF35D05YHE7s+h3Pe4+E76MbrC8O4ugPEd/8L2PD4c85PZuvY3117sU0xA9VsyFPZfhkz23KcO7irEFvqT+oDvmfBi85cyPuwMl470T3Pi68SwVvDIeC70ct9I8hX6kvP/7Cb0R18s7P3EcPu+U+DukDa08y25YPWySkLurRJU9VJPxvFkK5Lv5j/e9vBUbvuvZqjxxyLA8G+ZCvSLHDT39PAY9V6e/PEINQbz/z+M9lFGCPOWcPjwItQS+QO25PEVQLjk18527ns5hu8069DvTS4M9yxDwOUsU9bylRa+8yaASPJbax74t+TE9dc34PPjO6jxaQvu7WTOdu3GKxzw/MF88oXgqvLexUb4zBCs9Q/K9O+69Yr3qZYk8APtBvGAW4ru2DM48iXuyva71Dz0LM6y8qi0RvURDJL9Df749sf4OPK8io7qwWLu8SleNvJdGZzxMzuy82mFou0kkfDxwYZG8Q+CAPGKaq7yf+/G7chOMvGSHYDyd74q79OETvB8t6zsH7GE9EKyPvFcOM7yTuD48w5O2PaGXyjveorY7SWdPu5o4/LyO/Cm8wC1jOcIihLxSrpG8ENpVvB28Qb7jgk29In7cvKeslT3YHdq6g60KvPDXNDwy1qE72RHgvEK5A73jmV46Vaq1PPJlObyQlg09YbS3u4jvor5fkVm8EaeuvvbmTTzH5iA9I07JPHgiTT1ZRIG8gq3VvBztFbt5cWY8VinwPC1ROD23YRa9ePbGvO5xiD0a/u2+4ULOu6A2xzxHZKM8E0nSvmTmXr0hZ4a87umTvFyi+jzN3BS+uunEurad9bxvypC+n3kEvZw8Xjwt9m+9JDTPu/0Cn71bC8w8xL0cPaB/ODyaRbG9AxkYPcoxAz3+FJi8MvlNvdV9wr2bgB69vST1OzIQ4jvV5D68/qTiu0SqZL2zfRm8","1sG8PTuj1D2n4og9azQBPZkGjz10FzS7uVVkPgPQF728H9k9A+cSvYyZ8D2gTdk8ACwNvqd3dDuEdm69c5oevWkkar0D6R4914nzvV5aKLxaLrU7UDwQvU4/L73icZS87gTAvDBmv7s8jgM9XMkLPs3FKTwoshG9e7L9PPchmr2ZYIe9XNo0vYL/cr2NCNy93tNJvmmezjstD3K9dT4GPnf7Vbz0CB69+vgvvOpKib6+tT89DiDBPWjL8T3cOT89mZfxu1nqEb0dv0I9I06ovNtnGL3d6cc88jqhPXWIFL0VFhq+7KkTvoK1t7zT5zy5eOWlvPTMCzwFuhw9tCcpOYkQl70S0qM9eCkYPY2e3DzcYwe+1GHgvDsnk77NHi47CV2TPOqakDwWIiA96LC2vMa/vTu3r3U9C/fMPVi5Czy7Fke9ooX7PdzaEj5NpFw+gsb3PMlibrvrbwu98Se4PTlfHT0rt6Y8dCWEu2bp7j2zglQ9dUnLv9HIUT4GBuq8thwpPdy4jL0G6YQ+QdXzvDo+3L0XzSE+HOEmPusPtTy0ck08+Me0vvGT9rxrvma9AZQIPe3PCj1ixoK9x+NtPfiQADwAp4O87c1svRKgvb2o5gE7UGWBvfTLR7tVheg9mGfzu9wCCr0rvwA823GwumpOTjyLNbI8ZrHgPDP/Gr22vQy+IqWMPHb5Jj2l0tG8eNlOvSQ5t73LlMy8qTCIvOgaND3e1zC9QyUAves/Fr3Q6Rm9zmcpPSKh5jvw6x++Nf3YPZlELD0ZZu+9eFNPvWa6YL2Eszk9vCTwvVI8ML1wyni9thtePfZWojyQJMc9qrhovYYaSL0A9Je9NDu+vJvFlzw/NfM9yWNUPo5o2z1noUc9/HSrPfxriLwv4+68VFYKvRT2TL3H9fm9ahMIu+6SPbsjRQ6+uOdbPdlxcLyYjzE9h8j4O/6ll7yHQtc8BALvvcbLnj0wAYq72/IUPeXpQby0XKM95HjAvQ6m9z1Pxgm9/uwgvXDPtj3hX9i8","2aVKPK4vkz3bDk+984BRPqUYFjzXJsW8/mAOvOtH+TyqL8w8TCMyvfgidr7epHg9r6gNvYCTnrtu1QS+35TjvIyTwr1RFJG9H69qPdkeg73JWT88OUTuO9XFGbz4rpm6mMsQPaN0zTy3yxq8HhKOPKezwbwDkgU9zJYNv7fjHT2BQ3S71wXBu4sgtjzMxYA8JwS0vOcLkj0lax09iRwsvWAGzjy4Bb08CpE9v6HwFL3YiGI9X42cvMkY8L08j7k7xLCSvfStDT2+gKQ8OCxzPUvjYL3Ita27neUrvTK+DD2SvS49L34yvG+wsL14f4c9bcFVvaRS/jz3D6c5iTKRvRZefL39hOK9bterPc4wqTqlXSg81pSuPTG0jj0v8qA8zeL+vZCp3ryzh9u7Gj2fvJywX73440m8OONKPVWpIj6s15A83DqgPPKqFT5/pEA7G2kEvnYkqzwDtgs92R/2vB/Bqb1zpqw9/6psPXsZsbuHkUQ9hZkYPc78Aj3g/Z68RfQOPjrmi72wIKs61dgvO7zZJb0EGoa9QnfBPFR2Ir27D8k9VePmu15nqD1csvy9NLG4PY2hpLxLjR88yboGPNXMO7wj4Ve92MPbPLXN7r09kDe9JMb0vWEjzr3rAZg86BSOPVFHHzzarH48LYzwvbMXyr33+/y8bJYxvWclRjuXq+Y8+a1dPVZAfzyx2ji8gzRqvPW0oL0zFsG6WQU5PQxTdT19b4c9GjRXvMMXMr3THGY9S40Jva6ZP7yaJo+8fyfqPORR3zyZ604+wUr2vLvhzjzpbEQ8H3gYPJfNxrvjhhs8q0AdPTapJT1/Sem8emOAPJjo8Dyc3Qy72RKoPBA5F7/ycCw8KB6bPJATJL0GK/C7x0OwvJEgkb0Qz6k8Pd1sPDMMyjz+5Nw8/Sq2On5+sL2CdJc8PEjHPBsyhbxXcjq8hroSOrLoJrxCffK8yzMUPN4ntT28rw+9kXLBPH4eUbu5kjw9DQUdPewxljy0yu85CnZBuZLGuTzxLRa9","JIN/PCzRkrzfF5K9CZWRPG6ORr3Wa8w82LsMPZ0TYbz4hR49e5COO3TwqL3l07G8/HhwvFVuyrwZMW297B1gvX2zOb4Qwww9R9xePRDi9Tzvu9q8bSVrvEBJ/TytquG8EaCWvRH+Tj4zdV89C0gEvuv1tToJ3eU92V9OPC7yxj30DwS9hMSVPYVR2b4nQo09yCFOPlYDk70306q8aeadN34VW72utGM8aml2vcFTQzwiLYe9B5IHPnXpB719H2+77ufpvFwoKr0YNjk9awCYvIXYFb09Fw8+xyHGPMzI1ztZVvK8spKNPg63OT0AVzs9ycqxPdEL2LvK38G7FngfPRJYir3F9Zu9aJMTPbSP8r2dz5O8soR3vEEqTL1uPgY9RPLIu/9C4jzacfi9/bOQvabmF76CTeq8b7HTPZInOb3ALi2+pPJ2vdGp9DyjUAS+isQgPoMLKbwCPjg9BqD7vDEJqrzWthK85K8Xvaj4PT2d73g8fCEJPa3LtryqGlW+Je5BvUzFnbx/K4684/NYvTX/Fr37bL29Q3Q9PH4mPL1ptCi9ChBNPQ9f1DwFzrM9xxW6PRkP4L0xrB69ORZxPCkDBD312GQ9JCG/PIERqj32SgA9lhrUPXKRQDx38he9PQ+kPZ/NOr7WOXi9jzkjPd+RGjxIBTG72q5UPfZBkL1WU+K8kVZCPFSjsr2QLYe8zcdMvG/3nrxsWta7wCiDPfzfpr2n3Gi8OG8GPAVnfL1C0MM6UsJAPXSzdz0EXZm3XYu0veD5fr2ZjJ09auY0PVNWiLuJhIU8LzvgPcVpJTzdUqs88fWfOpWwNTqUghc9I6/4PLyriLzmaZ88JdMgPW4tKb1M1eC8tPMwvhgvszkZWOy5ohquvAvVIT3e+py8YHIfvhxngr3n0g6+OP8qvbnsTL254eY8K2K0vN4ulT0/68o8Qk2Mvnu0nTvJmdw8xewLPYNoPrzc4v27jP79vRTApLnXKis9VFMPu1XqXDzcttQ7lavgPJ+jRb2fciQ8","Oet3PTHAILk6rZa9JN1/vqv99jz42oG9JXS8PDRgqj29Rkw9zoaAPKDboL1hB8W82bNUPklXez0QZWY8QJOHvWY9q7uEiCy+MqGrvAYfozyE3oW8/mMFvUi9QbmHUoQ9a+87PSmMl7pn2zO9cySgvPz3jzww/K+9RUXLPZKT97sTuVq9yAzVPWKXbr7Hr7I9adtmPXGFIL1rYIE7tgTkO9JDwTxJ1wa+Qrq+vueQ1T0UvLg97lZ0vm5mCDyW8uq9QyADO39kjr3AN689FOkJvpyzFD00RnA8SkcIvlqzjr1faMW9Mz5Mvo6G8rxvVWe+ZUaAvNMFpbpqjkK6xI53vZnKjrwi7N47Ek0zPefJU70GZ1c8bImXvcFOzz2QL9k9j1qiu+F5Mb2dQ8i9oqUZvDZKFjyh5fy9nG+4PVgKID0zTs68rM3MvGF+9T2Tbim92raRvKe+Rb1CFkW9SAhyO7DyCT73uI09+neYvH1AnT0tDZe8aQF9PWefiz04dI68aUoIPrAmcT2F5tO9GYYyPYT5eLwunoO9R8AyPMaBAzxQhEe7/T8kvWm26L0NJl+82skxvNfzPT1mxYQ9VL1tPJALGj1XuMO84rq3uxVO1rz6tB87lgILPWGXoj1LwTw8q03cu3rWLD1y4h09wTYJvUBzDLy2I6O8QShUvJsxor3EaBO/8bYpvITrAz3OjYK8iAVmO3h9S7vQd7I8w7WaOgVEUjyl3qW6ZOR0PEi2F7wd2NQ7tOcVvudr/Lw1WYa8QKPyuxp8djz6RaU7JQ+BPOiOtb4mPY67KbvRvqIP8jvZa2c8BoueO+TI1buvkxm9YPwCvQftSzziQHo8qiHTu6qLCbwQdpi8PjwfvcA0lTvw2xy/6VsIvmXCk7s6O7C8NJDJvD9v1buEdoa8OBifPIy0Lbx1fMW9vv9JPGo2Bb1MnXe7FsqXvBu9yL0AKJC76cf/O/WHzzv3vrS8V28KPDjsH77vEQW7g7JpPHABwrxuuok6hqwjuhTyhr4LBYQ6","DNqGvAKiIL1IWze8R5Rbv2dg4j0uWPG7Nv+lO8ntajxNNlY8XWnqvPNPjb774EI+2vyKu5BZLzvF8Di9WARPvTZ9+Lz+4eY6ReVivkuiU7t2alw8/L3uvZPrLDrAzJ69JHqWOw0PHr35etc9hWicu104nLxePZQ8xtaQPCijTr3EWTg9iH4DvICQcD0L9Tg9DdqUvbarY71S7Yg8aQy1PCrtxjsAEYy8zsrNvYTygLxyCjk8uiCDPKbjTzrd06A7pAc6vcJcF7wLsBW9WIBYPB8lt7zJVls+2dMuvQgPmDwe0pw7jM/hvt25Ibxb0Qm8/vwavjCcLDxIgLw8pNlhOyfzGrq15gE93K5EvWLfc73zKwu9qmnovM2JGr4qfo08t4h9vCqFS7uqxke9ItrIvLfVQb1PEU08qhTuvZgjHzxn1Cs+aZ8evWX7BL2LT0++ZA+HPJBrpzxDnHK+TVNpvEJ+sb0ldLM7HO8gPIrY+zzinji9hAqsvTtCRb6NAzq8Y1AEPEoZi70sUL2+kGTLPMQekDzlyom+XE5FvYAPSTwICbI8rbJxPaypM77tDc+8bruCPeFnsL6WqpA89jWXvY2NOD5mZey89wA+vobdv77j5fs8OJggPXZfBL1cCIO8Gks5PNt3yzyk/ng8X2mLvXq2yDsJrwq+Bq3ovdC1bDuAlnC8DtMYvhToFLxPcnC9D0W3O+OVlb1RmWU++ps3PUuRSjyKoDy9nboAvgO7wL28XzY8b2PSO0UDG70It2M9GzSKu4C0hD0Vrna+/xVNPcBCsL3AO0c9gxiWu7cspT1lbzC9Hmfuu6ZWlr0e6GU8RntZPUccw72ILiQ973wkvaI+PL3zCX+8wKS+PehH3L1BI/i8Dd8dPt640jsvTD29vkwAPDAc+rzRbmU9diMhvvAYDD2Dije9XNxAPS4z+D1dn7M8OJzIvHGjVD0V2AW92UCFPTG07b15OHi9DTTvvPTFqDs24IA8AJNWPcjBSb1q2tA93F8kvZbY6jxxejc9","6PiVuw3AzDt5joU6SVzDu2Jg+bxMW4O/OISivEDrGLkqsXk6iP3uuDClgLzB5Vg7COgLuiMCtzq7V7g8+K8hvG9YxLxjKxq84gm7O+4QTzzPtCC8D5ngu0x/77s5KTc7nW5uu2XHgzzN54s7ffuQu/CIQLxAglm8YfxFPBx/Mrv2tf27DTUmO701rTzw/R88J3cFPBYY+DuevjO7vfPaO+Z9RDtDJPy7Qn8FveEqqjsH7jY6DJsgPFWlibuMdZI7MZFbPER9mru/mbm7EmijvBoWBjzp7WY6ajbsO/u0AjyBufI6PCx4vL/4cDusnFe8ESnqvc+0PDoubT87aYPRuwtbq71wa4K98bwbPaqg0bzGeX88aodtPQcjJz7OUrQ7I55NvDBuaL2tRYS9stivPZj3Ij2CZoU9EG+NvSaSW71X7oc+xG4BvG7gjb1Zv8a9lLOjPJxhGr0OSJC9grfIvUUXN75/XEg8mePEPWRgz7xbM++6JcUwPMmxPL2iFXW9vJgBPWjLdD7QvjS9lBbqvE6GALwBJPu9IXQyvQWzZr36TpY9YiO7vUsbAj2TqsG8upvCvfyftDyj6nm7hZKCPFXAqLu5KN28UVEaO67eUj0wv4C9v66hPZoI7r2mH4O9h0w+PSuT2j2bD7s8It74PbmWjL2ayMK8knhvPUFq1L3ytsc8EdR0vW03MjtnnCg8H4mxPdQyjjx5nAI9VWXKu7G3OD37C7E7T5tlu+jCCb1UGsS9ATkmvAn5qD0jOBY9uXQPvocbA7188f88Al9+PEueV7x42hW8QH1YPCQIUDyjYLu7UUGrPCpzB72v+aa7P+YivWjNmrzFhia8N+/IO74FAb0VAN+7PNKePf7RsrzDwdw8muqIvKD42jyqUWI8taHbPKKAyDsWnSm+pO8cPWRGHD2jQYM8IBzOuW3Fprxr8nU9Zb5tvDyd6LlfYQO+BmCWvKGoubz5RrY8jigUPVud8zxNAiQ8UkQ9PWDBGj0LXKM8EBWevMXwbzyd/Dq8","32JrPQSEADwoOz296vk/PLBQED3nxh68AkFLPsAAfryOcqg96LeavCd8RT1BHoK9xxgjva70WLtUY4y9Str/Pe9sQz3Sv4Y9kaKNOuXHpT1CWx09tkJ+u86razxOQJM8eYwHvmhEUL0uOvE8IBquvdmHub1btQ4+HbOSvL3u/Txk8Ca7gHSXPWS7XbwOzto7RjSoPYpZsTx4CQg90Np1PHxHd71Ctkc9l1u/PBJBEL2mmjq9dG9uvK6njD0aCMY8L6uKvKpypjzBrl893GLwPCy9GD00V4O9QvkkPZ28Rj2HEc49Xw/5PfWRCz2+6wg9c4UyvYx0bDsmYsG8YXMsPapJnbt9N8g7btCqvl9ErDtcrlQ8/Ni7vQsNEr79ByU9ZeOBPcdnPj3o4RQ8db9MPOf8iDz8kHm+GZk9vLqxprwovRq8SgQVvhixEDwOIXm+TgmSu7eVLL2qJIU976aLvMcsTrklRno8axCyuhGqSj2ftoK+fRMova5y9Tw56tI7P2b9utsynLvaCRU9GjyQOH2XtLs7n8S+APJtvVOhobvYr72+N1VaO/xw2721n5W8+0uNvUTklb4lZmE8yn0ePB+un74uIRM6LFtNPcfOg7wuNpo8K8AAvLtfXbyl/rm+v8x/uTLNlDyv3by7eq/1vlLopbvJm869PSbePDSCDT0jnZo8NQW+vV7gpb1dZRE+2o8FPlVigDzYeRQ+bOz+Pdf/qz3y6MQ9fbs0vcNJsrz6DMs903DjvIJT17nrN1i+CYLGOpOeHbxKSi09TLuBPgt+zz1KYAY+iCgcPrYtFbxNfGe8+8OYvTV1eb2l1Y09mrZePd/4HD2PP4G9FbK2vFHUIz5krci9pfSAveAkGT4gcCw9RKscPCU2CT00UqQ8KWmwPFE4hr6aNEy9H/lTvSjLG7yZoTU9PZccuztPeLy1ktI95FGWvbZWPL0Dc4s8t/j1PXprID792LA9ja/mvRH3nD0aWR0+O342PZvTgT2qYvW9zD6rvPPnLDz9QdI5","Dm0avF+cVrxSpeQ7EzUlvNZuTb4YGL25TMZiv8e7grwL0n27eTZRPHzoJT3pWza9WYYnPUpyOz2tERu9IIq0vDs99L2hXji8clERvafunz0ZQz28XiJdPEU4H7wJrzQ9NRACPbUkYDwAoRS837xsPFbH4Lxg+fK/aK6YPmsahzwt9aw8AGKUu0Avlr3tSI46/ZHDPbnEiT0gU/c7aZ4EuzU+U7z7vCy9/+OXvf7In7tGYx+84PCdvOsVIb01gwk9WiX4PFjdAzx8MSs9WLQ9PcXA4Lws/cI6N0BFPLS9gDyH5bg8Sb6ZvlVDlboWtOe7XIEZvaZPnzwW3bi8ybBTvCm+j7zn6Mc8pKBpPJXhBjv9WyS6q48Yv+Ro3jxvwBY9J+ZkPM9KxLzIXay9FESfPHgMgL1H83e8VaHbPdqD8zyVxFu73ygXuxlXKz2xVQa96pGyPOZ3Cj2QRPQ7XPIkvaGfVL37a6c83YH3Oxs+Ij2QtVw8eN5fPLJgnr1vsvk80LovOxoiMT28n9u9w1z6PAf3z7wbmIM9agZTO3nVbLwebrI8F5CQvWcv5zyPDT+9eBqrvEEiyTvgdjE83iiEvJXjQTuOlgg8KOUzPCFNAztPJhg9EsU7vJzJHj00dv45LoE6vNJqOjuNJtk8AgenPXYqk75WiTe8jsPdPEwUDTuAFh0+dsycve0TL74WaLK8tMm6PMmMgTwZY9W9ofOKPCIchjwJq3K7bRuzPFGDLjv/Zba7t8I6vT/C4zsuFAG+his+vmrkUz3mU/k8gaQrvWHVxD0YuSs9GdLXPb+NFzy5JHU8SwWPvBKjeLxwAGm+5fGTvWGTGL36ODQ9fDByPWyIZL7muHq8UwRrPn8ty70J7LG9TvAJvo3rSj3551s9GEPhPHkFx70yRO49UyKCPeHQsL0q2T29LP/svQwnHr30ieW88SfIPNvkwD2ol2692s6PvVW1lb4ZoC486cuIvKI3u7szPGk8TWZoveOHDLw8mZ28hW68PXTBLrziZP08","sB+PPYB2TL19qMC8m7eQOkz0rD2aTWw8B2O+veTqHjwS3u+9BjTouuQMQj3GGeU8boWmO0Kdqbtp7lG8LKShvP3Der3/AsS8YLBFvX0PTL1eb4A9EN1auwrBLLy2AK48kGQYO9mpgTocZYw93/AhPVE9x7oiboo9ahrDvFi9rT1aSqo9h5YIvYjd1bzv/q48vlanPKaAvzw3ZrU9rP/yOwEq87yeIwK+N2QFPj5karzXnUw9k8tXu+OD9rxAktO8ptPRvNsHKDxqFUy8dJDzvNhRcL2pkgw8sr0YuhpeRjtOyua8HUVgPqxxIb2xl2O8/jkNvZfSALy2LJS9G1N8vA55kr3FZg+9XmTfu04mlb0q/Ao8y+O6Pa6PRryRPke8TBY+vcTAjr0Z4bs9OCL2u7ODdTyfZow8Zq7TPZ2+5zxa1Bo+ZcsavhrrVr78AAm8/BQtPKD/+z1DOfW86XJFvcWsnjykTb+7qdVWvINk0D3QcMy94CNvvCL3T77H+HS9GWqlvCBJTz0Jun28KZwMvYicB75HWVC+7bttvYZZrT3DW9g8tgcoPbG+5rwhMME9Bp8ivgBa4T0beze+rzMHPb6IOrz9q7K8zjFGvXckpz0cF508+6Ncvo6OqT0H9Zg73X9OPbYidD58LKm840M1vYW1Gb3+bYa7zs4YvMp4Pz4A1SA87cOLPV128778gRi99T3DPIoMeL0LVow+oaaZvWVt7rx4OT09hO6SvHEHV7v6fmG+a9DVPXDerL300Qw+tEwFvREdMD4mqlg8YrjWPqSq/7f9L1S70me9PLiSuryGvou+/wA+vX/ZGL3yQFa9tfMavdB3yD4EC0K+Nn4EPPo997zAWE49ros9vYaURz0hoYY8nYAWPTiUsr2neMq8XzeVPNPn+T7hJZe9aL4yPYb2wT2qALa+4tTGuS4GCL+WZM0+gAgwOwLCc716ld++/cU4Pon4xzzhKuA6zUvRvpk9Z7soqQ+8SxjwPEj5Mz4VAS+9j0oPvmMPTr5VUqY8","TmcUvY6yKby/sz898G//vpuaIr704XE7izoRPMDeAb3qMJS+Llz5PFpCKr0PUv29SorxPOAIeTz9n1Y9taR4vdokLjx9gpO80HydvKZKBL7l2wY958/cvI6ziz1Rfy89NkHEvWt85bxmdIC+aXhxPOtQ3TzhJck9gzXSPGjcwzz4G2Q+sN+XvfFMPb7SNMQ8JrbvPT0Pyb3DjxS9JGI3PRFlHTzSgg89qp6MPUt4oD0EpKW8j2eTPbL7ET53NVE9RdBxPAfH9jzNMZW9euEVPSoAIz3lLcw9ld6Bva7S4Ltk6wO9sOZavStad726Ot28KC+rPRMyHzsNXoQ7UY62u3r8fjwi7Bk+nk+ZvRExqz1DX9q8/uAPPRe/yL0PmMI7XuxuPbc8aDtpEBa9+9boPJG6M71aXqs+y9wHPWVqsDwENLU8EStMvqp1yDw+6zI9J+2sunyjMrsH0Zw8riSpO1OKbTxmvYY7iEOsuzzUdjw5hw++NQkivcs0QrwwcPu7aJyOu4WxrLuVkAS9TF5ovOAHzLhYzju+Jf7BvatvO7wfblW9EVQmvjzXLb0+LJQ8OkhEvfxFUz2Bp488FPdpvXdFt74cAnu8NayzPHerF75MELg75FGHPOL2CTvPcr++Br+EOqnvOj1ozAy8bC0EPf1qjzrMJoa9OC8RvG76tTzCBYU8IZymvW53FTxu9t49mm8uvYPvR732ulS6CLlhvaI4i72vIXQ9MkIMPlep/b3fkN693ZIiPjsQtD2B+b89H/KQvVPXFr7xoe098s0FPvP/kDwpr8q8XjNkvUi7Qz0Soyc8nIA1O53ITD4a+Uu+upLKvaVzZT101W88jPoivaHSoj1A5ae9B17/PViRLr2A1ic+2HUTvWdoqT1Dpo49v8aTviWgXD60JSo9qyp6vfCsjb5d5CE+g41gPisdCz6QStw9eijIO4jzKD47C2s9Wh8lvmfUDjvp4sE9ncjmvLayZD2GIdU9a4D9u6Vq+j1YOQc+G7qcPSTzj7xx/bI9","l2Y7PHWVHb/AJfq7DP4evJHFgTteITE6apSGvDdGUbnWSS88eFHAPDZ0fTquNA09+tGKPJpg6Ttlc7A6as9cvTczFD2mx4+8M9ZZu1L6r7xIMdq6NHppPNSJITu0xP+7/OI9vfEDuTvoHcA7OzyuPPm46TvkTae6OfwqvG4bzDukncq84KvEvCEzxbr18ba+c0goPfw8mztXzWk834Ycvsr2G7wTggA9eUCNvGQGC7yamZE89Ifdu9Xh+bvjDxe8Pjl3PDJgdzzzKdi6BQCWu64Ibj0Zboc8DJdkPB8YmLs5TRY7GOjbO3LPhr4rvqA8zeB3vAAp3buP5ga81/IUvFC5ET36KXe+F0I5vBWfsj3vU6M9WwvJPIBRrT1jt6Q91fP2PaSXaD2yArC91X7MPdkBwzyLS6E9kpgFPvg5bL0q7GA9tJ2ivSpPvj15JDo7kkwYvTyssDz5Cr89SWbzvLuXMbm4fu48QjhAvaLuhb2rsDU9EghZPKehD71Ha7m9RRHzvAts8DxX14O98dxuPM/lX72h8eE9hGyAPQJQiL0rKP08n+euvqmAGD6q72A7HnHWPZNWsj2UEHm9vTk8PM0rFbzACa+9aez4PV+DXz5+wqc7CWlUPLRdTDx6IQ+96HCqvOXmrz4eCK+9zOrEPcqvzTxNGWY8vQYcvbs7Vz3JyiW+NCwUPU8PFT22OZ+8Fpo6PWqKcz1ByYc9THc/PSVQL71dAOO9oGjEPLBJ87tXf6K92DLEvTLorzy2MOa9mEaVvIcxpLwZv4G7epz3vn+axb0aZYU8EYi1PcvylzyVFPo9crRmPcOP3rx/17G9QGvtuzoejr20AwI+tZkaPQzMSDxHRzw9VIUuvHJhNr0G5x++AwjQPV1qqD0PPS2+iceZPS4BtbsQaYC9uqzDPcLKWT2LYDs+jcKXPEWipTziizm+DgfjvU7sUT2+NCc9EITQvZjgVr22gZ48CegAPfARob29cha9V1K4vFJn0L27KDE905LAPcPGFDzglhk8","8ZlKPbb0Ij6dgMa8NTTRvETtAz47C5g89BS1vIOb0zsiIL08jH7IvXZGCbz22Lu9aTU2vsDUzD05mKI8oR3KPSiM/73fdYi96x2svK2U0z1Zee49t4/HvJSEtL2oHjW+iNOFvQPkU75KE/k9WvZ2PbzJlb0I5eI72EHavTLVFr5kjBS9cK5SvdqZdr3icQi89hU9PWrbVz2jaJw9u7MEvDMV8jxTSBK8szdgvdq+ETs3AoA9BNIFvWzbuT1Uc7S9vCCMPIuelr2xW8y9X2GCvZFozTiiDA6+1lJmvMBnarxp66w8qpw7PWpcRz6b7fo9HY6yPQBo47xHTQg+wKVivVBaBrxAJK6+2QcrvMqRGbzTqHy7V3ehupYccTwzi9K78d4Yvrayljw0a806eLh6vAZhTrsvy4A7WUjUOssgNb+z2y08ocoCvMd5ir3HbfE9VEXPu5tZ9rqYHOK7Jk4qO//Pqz3Ky1k8+u3cue3aGr1MRQM7402du84AybxeIDi8lxqIvCJwYjzwGA08cITXvo7Y8LuyTga8V8JbOg9ul75aNzO7VozhPOCzULzk7vw7pB6VPNuBNLwDP6a71L77tyCiELynoOC7D88+O7wDa7yZheS+qHQbPN/zYLuhoNY7MxfiuxCUFDw0EMe+N9n4O91piLyfFpi7BkCIun8TQzzO5jM9w4vXvHlNk72WjQu9XuPdPD2x5jz4Qa+93KMJPNPzLr2ucAy84fQHPVl0g7xVhNw9BeADPSc0Er3bdXk9HloOPa3sGbw5Jlw8ZmQwPn27lT1k7BQ8p0X+PAGi/rsAnFA9RSBhvidELLzPwhS9EOC0PQBLz72Vboq9DrkUvV+r5T0zN2S8oMgJPpPxubyBZhu8f2uLPdD67Dzvhx074sOwO+kLRT65pNo8vksdPuQQgj1VaQs8d9UyvdkeyT2msIG9p0zPPB4e1L14pYa9VmpIvXeCjzxyHmy9SmjSPT0mTD1Q3OO9JdujvAMKLr3FEAu9JadDPOwQNbv1y6u8","7QytPUuP177ljs48olKGvLHuqz3dk4Q9jsuqPv/czj1CvsK8Un5VvBSHIr0VgHk9B1pAPfbXmz33Rzw8TekAPTUYqT3O7kK+AmV/uzScDL6Cqd48oNWqPFsKFj5mmoK8Lu14PSdkBL4KDNa4SNXBvWnjQ73wng29Pce+vVVuWL42nm+9foVgPZxsnj06MAA+eHM2veuKDz6hs0S+pv1Lu/Z9H7u8CFq+z+U+vPFwvD1F8w68Gzw6PgHyHr6vxF+73AjlPEfX8rulFzU+YlYGvmTl+7y0q9E8kTmEPNXXZ75Zcv688v3iPas0EL1hGBc8UoeCPfQjiryXerQ90HlpPPOt+rzhiOY9Lf+dvo3CAT7q8m+9gfL0Paby/T2Nx6q9jmx9PEQRlD2rP0g99FWPvRWcw7182Om815v7PDjwnjxcrHq737HxPeOlc7yrZZE+EOvdOyyhobxSsKe95l+cuzgPoL1d6Bs9vnNOvcanWj33YpE9KsOVPozVSL69+7q+QzKFPMNCNL0ksQE9G6QLOMuVRj1VBh498BhuvkEsE7zs4JS8jk8qP4g0Fj31D1i8wKYlPlUtEb6+KCK9FnRvvjtD1z3jkDS62TRlPIqQDr/2UqA9ezZmPBXU3TvWIZ+++lqZO33SPr5d++49rgM5Psodar2d4b2+kosSvrFb9zz5aOG9zcMFPuu3nj3PY+A8hyCbvCN7ubv2jx6+z/KUO6qAGb28Jva7rpfFPMlYPr1gnHy9yO6uvXK88TzURUi+XGHyvX7S273ksgO99N/mvShMNT2dOis8offbvHQVpbxj8We9P6Y9u1BOtrwtJSQ+Y7Y+Pv1zvbzOQDc9kvwNvXV/OLtrlgm8I9gzPR070DwBcAS9F6DEvHJv6Dx0vEq94xQDPdlC1b1+JRC7LEM2vUguvTzK48+9i7AevIqRoD3RQ849h+oqPXj1oj1lNie91hgrO7Jdhzsp0AO9xMPRvZLyYLwzE+s8eKx0vbQRCbzvgP47Q4XQPDYE/DweL+E8","AK+0u3ScK7xAp7i9OW8aPQ4x8j3ENro8w2UFPg7ipLt7up48GWqFPan6F776r8U9rzKpPIwSibt2ELI8uHQNPitWWr1t+is9Qb8jPkfyhbuq/G89GewPvj60Bb2hqoO9JiYrvrOB8Dy7bFg9Ks5Xvib3cL7fx3M8pjKEvn3dCr17p0a+kNmqPArkQ77Dxfe9RR/WPZIGBD0uada7xAoHvXB1vzxD7HU90/7yPTBoDL084dg9x/ShvLeMNzyMENc7sFqAPSoPLj0qRu49hyIwPdwLRjuOGhK+IxbsvW7sy7x84oc8ZgsrvhN/kb1QIy6+f4M0PTnl3DyEnxk+RPICvJcGBTzldhS90zJsvewx/r7tp4e+mxDZu6wn8z2Stae4F3ZpvVRz3Tps0iW+nMcPPXcR3j1IgBq9yQGUPaRkED5yHAq+nAaXvU+e6r3M4hU8qDAAv9TCkj03Zom9q7RBvmI6Oj5qUA8+nD4OvdWU3j1wi6o9wfaovVgXBL7vntW9uOKoPKAdHL2UNYu97Q80vGOuGz457409gXF2PSg5pzwyoI47pyZ9PDOai7y5tt89Koj8uhuqBL1Z76s9TQ5cvQBpmLvWCp+9mOSWPX33WT22OcU8IRgtPWzxYb6km+86DSkIvuphXD5+bsi9bqTsum8g+bscl7q7TEYyvXJLMD0imjy61JAUvYsJ3bwcOSq97kfLuwykir10mJQ9uCAJvj0zgD1YfhU9Db7kPNqqQTtULAc8XguVPY688bxi6gk9KJwEPXjyuT14+Lu8XBZoPZKQrD0kYdE8WWXqPMj4l7wtvH49gHMDPQ1ntTuz17K7aZbXuw1ncT2ccwu+bqzJvJJluD0Xac08pSevPCmgML2T6Ye947EIvb4yvrwioW491zCyPE6LxLxvmqu94IBGPQ3yk712+C698z9+va4uiTyvvIa8U7ZQPSiQPj2dy+e8lKyuPaUZ7z0uuaa8yAgHPMu+LjxYB+o9DG+QPDnJd70OOzG8aL7zPEN3sDuT4Ag9","jXSRu1elIr2qwgm8RhI+u1amlb5oLxM87KViPQdyP726iCc8dWycPX/amDsXLBu8gvZ3PPme9bokIUc81WWNu5IANLz5kns8Xrjuvk8BPTujU9C684fnvEQiursUYgk9UXBiuyLS5b5NOm68kU9ZvLE30ztNOb0+ntKdvn+6j73lNGs8U3y9umqhEzpW0NK6LkuSPAWdhDu2JAw8Lq/Uu6M4gzz2UQY8pxsGvBwMxrvqOGg8+umjvHqcfL3do+A7SdxCPO6fsjsD6YG8/Qz5Ok8zArx6Pgc9514Wu32U2bpAHo07/xbFvC9+pjwr/JG74G43PW+BJbzQkUy8/0AGvvnKqjzLpSE9Rc2gPZTJ/rycChI9bfAfO+JnNz1HRZi8J0yIu9+qHj3FVTk8ffWJPHazwz2Y25i+/3z7OcGQzzxD2jM98xc6vBQrrTz4s4a7MrFSvBQlSDyDqAQ+SbMpPMqVgjz/IVu8SMuaPME4IrywSqQ6hUBJPZJ9i73WZCI7Qx/XO/6vr7x1W009fKEkvZ24UDcwAiC+34w4PE0z0jyEctk9WF9BPeiZozzmhq+7wQPVPE3fw75jwyo8uXZXvke8dL26MsU8WmqCvlY4y7yxpRI9JXJ5vIXwBzzN9ZI+7OwwPO36Vz3QYMg6xzRKvQQ2/7xoWwS+CmoRuzcVxTtr14M8NwUOPW7W+zpixyM8lRe4PesEqD2+wt69ZO5GvaJfKL4HcBE9ZdqNvRn9l7wY1Kc9myO6vNDlgz32B9C9Y90+vGKMkbtOpGE9K8qGParaGz1lB7m9GHUBvdI4tDyJjPO8O/qIvTbBqDz/1S29aVHlvQ4Ovr3itJW9gwOlvdT38r1sg2G9rSvivT/tSLzQdOQ755FjvZ37kzt1rw+80FhxvI6v4Tr2by+9f73HvVaoTj0jWwQ9IWcKPT5VxzuYxw09GNfQveBPqz1aLFM97sAaPX8ne70vDF2+uzwDvbL1cz26PVc9r/EQPVBAMb4+16Y8FPW/vBacgD221Pk8","cD9UPFtPszxHGZI9TIfYvWA98buhN488UdM5ve6nGzyqKIk9ZZDQvMV/yj068g69R9RuOwxMvL7nnew81ICGPTAzdb1wzR694G6HPYV2h7wDBCy9T2IsvG36Kb0lPlI8GCr3vQjO7ruzguI7WCOhPcwFSr372S+8rV7SPXHVVzxgTPy68qW1PfyZkjxFLHM9beSPPBCqUL0pNoY8w7aUPLTtfz2bHwC+ND/NvXJSXr3j7wg9BMcYvmwfZb0nBf29bOD9u11nHj3uy0e+M7G2PJhn6Lx2xKG6lcstPTId7jwQbKe8BJbavFiE+Tvpihe+XbXRvLbKer2Yd3K9ctaVvV0+YrwiklW+qH2tOzaRArwTXKg9E6ifvJC7Q7uDKKW8RXEtPGgoczzFphm9eEjvvbOuLbwt6Rw8IYk0PHAp6zzjSta8pjxLPXCx5rkwJ568Cb5ZO7qglD3iSRe5t5GuPcm4czsJOAE9ISRbPGZHT7tmEZe7P/iGPcSMpr3Rzyw9lnLLO7rB4zxVSKY9Y0CMOwuHQL1/HcQ78q7Uu9aiKjzUVx66Rgj3vQTKMrxbR1S8TpmdPC5k9Lw+xKS9ebybuzK+GLyHVqO8MbFmu8Th1bx8W049mU0cvOCOu7yQ3Uo81y1cvAklnb7DxKa8iIq9vE3wgb5ZY1Q7rwkdvF58AD4KARQ9vlhXvFBvqz37AY09pc4BvYoZk7vzHKu8mji3PODaarzG3wY96DK0u2zmpzu5bzg9EWowPY72ajxWZGa9xKApuhuB0L3v7AM9Va6tPP7KO7xybMA7RpEgPdbLtLzglsy8fc0TvWJtgTyJ4YU9UWTCPP35Fj57Sea9ePmCPYWh1bxL5wq8bE7ovGtvCbyJ0q68CutFPbYjsbxZxoA8WbUePbk9jrz/7gG+oW4oPWANpjysqDw8aUV3PURi8zypwlQ8LBiGvE39VbzzjOE90BaTPQ5JPj2JMhc9NZ93vMyNxLsxo4k9GvxVPJ0XYT26FDQ9NJtOvBRVybs36VW9","6+21vNqNA71wGz+9+y16PI11Y7y3at89q9cvPGt6lTqnsb48R9asPClDNrzvC848xwvEPTyF7rw+Idk7uXgOvbAWLDyfas88esYOO+lfcz2PKBg7mZklvaJo9TsUyVk8CtR6vJoFfDr1wqA8yYzsvNw2bL3zwp+99gAIPPIg1bwVJRG8Il0OvdB+Nzxerjy8pGq5u+hqrjxrkMK8EHL7OryRgr0P8cK5BN7JvXioKzyhgw29Af2EvQjTQzzlEF09Av73vZNKhzxnJTe8NDCzvt36WTz/9MK6bFB5PFMhtL60prS8ofeLvU74SLzPmPW4XU6gPEEC1L72OaA8u35LOsqjc7onvhO9iYcqPEIBVb1oIWY9LaP1OruOCr42aeU7VRpWPVUK4r1hm6Y9WbYIPbS6EjzkCGK9JH8vvujWXrwwbBS+LFSvPH3t9LxNaBM80Vv2vM6atj0BJoE80378O5YlTzycFzs9CosTveks8bzazJi85M8AvatzVb5p2oa7y7ytvN7+IL0/fju+Lck6vUHBrL0PEo49fpCxu+9wFr4N84g8LWygvTDmv71Na8o8aJ0KPf0CFj6y3IA8przRPCpDNb2SndC8x1EavEODXT79gd69xDE0Pa82ML2wAdq81C1vvY8Ogr1yIUY9Q1yKPMXYnrtxjSg91jefPX37OD1YNkm9NbVDvNq82TvkvJ+9Pq2wveeTDr3Ka3M+yQlju2h/Ej2ZgVY9cyeNPH+0oTzrNAI8HUfdvMETOj3pEEa9G8nTPUV6KLzgjCO9c5KdvQOdfr1eZVu9urilO+DWJTxzbQ08JBwfvQOKHD0j+I291qmSvHfR9LyhE9S77QYqvW1Xab3OVxu7bu8sviJzLjsySzc+P8G0PG9pAr56OHm9wqimPfMphj0zoVi9NXWUPGZxGb3UedA9mAwzvFRIbjxw36+7OlKxO5yBhTyHHL49BJ+NvQw9Fj44+DC+xgwsPRUG+DyhfSW9rpvCPMah/b1O9KC+f0o1PWGqjr3bVu29","nBFFvagyQT397+E8EEkMvcgEpz0KOxk9Hx+DvaEH4TzXvrK9dfg9vdkm0z2Nmxm81VOhvDrka73ZQj2+7yE9vXJ1AzwGrxe+Jt5WPZbqmb3WsAy9NWSgPEvvAD0dnYy7BsqNOx9CBr4AKAa9JPMfvW29uz0qHe48IMcyvQMnHTxVa6S9sycKPaKZmL6IFg+9/wrfOm/Pmjw1ZSc7Mbt6PP1rHbsEm9k9PkgDPtkiVry/wbg8CmkXPlTchT1KM6O93hY1PEDNJD0MFpW9iUk0vtu9dD1Zcwm9Ae1bvRMyiD1qmEI9bq2RPY+kEz1vSEY9Zzdtvch8RD3z1Bw+C7V5vWjNgb25DIy9HqLNORgaKTrCiS8+wi2YPR2BZT0m10897BoxPdSRt72XAEa+f43GPVuMVz1DZN894HEtPQ2aSD0lVmC+AKhOPSqPLT5JKpc8mPJzvLXkm73SZhU8mq5DvcK0Rr2iuKO91gwTOzBMgb0Bivy9NrszPKiUnjyxcYK90J6GvfUqyzo87Du93/cjvqOQGb1iQc27Vr2RPVwyUb2uOpg9pxrtvThkWr634me9SwSVPT6urD1hKae9yQF5Og5x/Dyrdx6+GF/vvUSyHL5Bu4w8UeMRvQCAZTzqygy86QGKPZFm0b2P2W080gl9PDyHmb0+qJm89aUQPgcPYr1jmK6+1S8HPCEJHr4D/lm8Ooo+u4NctTsxEwq9miMMPYAwY7pQGrI8cmx9vFgwCTuQEig9zeNvvqWavDzHMQe9L1gaPFJm5Ttk6A69AyaRvTznkL7kybe7YUSkPSw6eztUPnU+vYk2vVrPCrtAF5S+txpnvZNHfbuYlIw80L4CvC0XrDuQ6Zo8f1YfPHuUnz3ZcBe+JbiqvvNJMjtNkBe+9Z9KPk33VLybB/C7jyF6vdK5fzyK4Eg9/tayvGeolL3lm7W8q5Mov8sCozyAZes7IjIaPItt+Tyy9cI8K6hXPSihgb5kvF050DhUPAa9mD1MtSc7xfeEu0wzMr4tswy8","wWoOPaMnujstDZK+UvgrPVARe7yIQWE8TBzbOdVAPry/Z1+897tHPXEQjTxo5TM8+xQ7PN4bPr55kXw9jDvUO67r2ryyWGQ9kOXROseu+L672u27kt8jO2H6hryqUuw5MxHyO9BMOryFcf87VY33vH8QgL2YBxy839ggPa5avDsWCfW7qWC0ugfwgb3CHZs8+cu5uvhndr7tBrS65dXUvWmYNr5uiJ+8iWOVu7mbnjsdDpE734RIvta28DvhOf68gDmXvY4N5rxCKl68nqnHukLkUTyianq85wJqvKEiWr2a0Nc8Ie5pvTmaHrxUlaq7G1sgPJxwbzyo6E6+bdVNPKUxFTy9LKW7TOpPvRL1kDsOcRK9pqgPvLRF0b5XLMA8D06ZPMQ8bjz5HuS8nSaZvOixybzMMTQ+F+TRvO+qcbwPeC28Q+x9vYHrNb2lifm+NgNeu1OqE7z6/Ha88eWUvD3pm7yGf+u7jcMWvArWoLwWjAY9CRNuvVXfOr6PRUI8lkgXO7WfNj076n6+PS04PdATIbx16wE9/noGvT0IHD29To08bF0TPeWh0r6yHv080L6mPFeKrL5+7iq8QYKqPO6n3L06haK8PDGDvYpcSr2M8g28gwtSuhzslDxEb0a+ZcZHPVHN2z1etSs7pmTwuxQihrxMPR29J0etvqZqJj02fVI98Clyve4uWT333Aa+g13HPNcAGDxBgKI9iKybPTDwdr1KKR88hSWiveWKBL0RddQ9vSw/PXtfrTsagny9BxZgu1AeDbxdYou9a3/2OjLzUj1MI9G9pSdtOxCJDD0xHn29fx1HvdYSmD0BW8w8Ckllu7YxYDydRHS80LQ9vcV0qr3PZcA9FLS5vQHwEz5Q+Y2++rn/vKHOEr4/tiC9xSV/vWv/c73cnss9ocrJPWGWkL1vKt49GrKIvYk93TzxpoG8rImdvUTczb0k5mM8pEslPO4f0jyUyya+4JY2vI8ss7zYhBQ9aMB9PUyanDylOGw9GhAUO8oAgjvca4s9","TzvOPAHW2bwvOVO90NiTvPkc5bvj2ME6jH+qPIguoryIZxe9LxY+u9O/6bvDC0G8SmizO5j9dDxx//s8GpJ3vOmwLzwoZtw7CMd/PMIf57tBLSi9UYpevOrGdTymHF28JrSbvd8dxbxzgJq7NCyMO0A7LDwhp+y7tj0/O6thjDz1iIi+lfphvwfXXTyNoPg8O1A1vl1+DT0d2J86yCWzPCG3bzyykTO8euYIPOseLLuEPI68Biv/uw7rabzmI4a8qBEzO86brjwCcLc7NFuJugF72L4rvhq9KSNhvQ2fW7ufm7y9YAk6PXLCHTtpQZY7nWcbOwJ0obuHSjC92nahvCiRHD1xDoK9ziazPaFKKb9J+dM9cYrIvCXFaj4bvgU9IbupvZk5DzyWXs+98gDYvUZib70Yz8292IAxvTdViD2WN709VnkUvS85kTxp1p09KVdCPVRVrL1E6sa62hB4vTfXDr06Pg48Pjj8PVRFiL3bzhI9SmsVPVW8vjwTn8680SS6PZ8x2zwDhbs9LoxpvKub/TzN/VE9rMi/vOjlhrwZAIy9TbaQPHXdczy+tIu988mJO2m8nr3SaTI9W5cJvVt27j3LbJO7M3WIvd/s/zyyh4m88b4cPZ76C75tZPQ9T2wKPQdPcj75/DM9INAbvPnoorommpi99oI+vUx2rL162LU7Md0MOVKq6Twn0cK7lp0FvAe12Lt5jqi9SU6PvH8XsLrU16e8kfMYuukFmjvzIg08gUUavrz+BL1y2es8NZMAvMGGyDwNcRU7m5iHvtEA3zw75ZA5b0Fhvr5SG7y/UTS91BApO2xOmzx3FAG9zY9fvjFfGrzrHIK9F3SqO23TKDw7uPk5kdt/vb0CaruWrVG95QpAvjZuHbp9d8E7BajFveiuJzx6UVU8FdR1vHqQkLxV3g68+nEaPFikVD0G0LO+9pkYvQTuAr8bNAO8sFY6O8TRrrwbKu68S0V/PVsRCjx0aoa8J0AyvXkK3rqQIbc8ua+YPKt7WL6hXRU8","9GKHvXubgL4bXx0+Z2pXv1uBNT4Pp7090KEoP3wsUj644489qHFzPe7ya73UOkw9NIUXPqvaG76xEF++sXN4PuWt6b3gnko+3BtJvl+SkL4pPxS+KXrLvU+mvT5nsVc9+3mHPgcMuz7FtT8+1AfePpnKpT1Bch4+G8YsvhZbxb2fmDu/eFKPPootkLzPdAS9gs6vPjoAHT1B5DM+67LrvmTXJL518we/fbNqvmBilj5kpY+9jxQ2PhK7BD4qJbU92+R1Pheq1b5gJBo+v2GevcwgbL5ahy8+qGWJuygxID5mQPi9RKOTPvzUiT6Rr5o+ESwAP3X2oDt2oI0+4MkSvn10tT1wjlK+g2lGvg20Hb42tro9bdQ9vrShej7N8U4+alNzvltXAT1rEbi+UI++PjGmLT56X1A+TPJkPUmAK73N58S9ATQ9viKoT77H4WE7YKhmvGzNAT6lxVs+CNIqvsboKz284hC+lMJTPhsIVzwq+Pe9pUnsu94NLL4tJvy9ch7cvi7giz7GGp4+7Ws4PlfO571Ftjc9QPaZvMfRaj6lATu+TFNSvgW+Ij6nvYc+0ZZSvIU+oT5xp8i9/7OLPduC1zx+6eA+x5/VvZP7aD4CcoY9obs/vV8uhr4JpFe/GjuaO9N24z0zA0S8lh93Pc/FSL7XuiC90DiavvUW9ju7Auy7EvXwvJm/WDv0ETW7xZcQvV9bR7/BFdu7TrQRPOeu4rrowHe6iou2u1gmMbz/sNA8Yuy7OgLWSjyav+e7T5IEO0jvCrselk69jKJ3OxOB8TrajCq8Z3lzuwj61LyEKeW7ZV6avGlftLpGKxm8ARtSPIy+g7zFvBc9kvh9uk5OoLz7vf274tjIPOkU+zuhvl08waoyu9fsITy2tz08pXLoO8buG7twGv66w043OQrR1byouvq6dUPOuQ7rR7veMIU7IC2jPDxSWbuJOt68ESLnupkAi7rIStq728nxux4gJby+CWy8+LrKO9mVFLpgQF2+nPySO+X0PjzmEUg8","yWRFvG/Xp72hqaW6JL2zOj3c6TooJUS6OTEzvMaeUzuQJmS8tpa3uxU4qztrX8E8P4NYvKzyE7yg9qg6p/ysvtvbPzwKOsW8dwoLvaPFGb8SJJa7XBREO4TJNjx1IeO7R7USPdoWaDz7gD25fo4nvMEsqjzSL6K7uowXuxtOmDubpt27DvN8uv1ms7uMF2++K6kEvMOhsjwGDAg8pHkjv6jjgDvJ4Q47L+qbu8N0Xr1+JdO7P/2YPPy6RLvkE2m71Dm+Ob0mmTyMWmi6ZdPUu71S7r7D/VA87tXMO/ZhaTws0dq7LRRVPNSJML6f5m67Una2OBGM5jpq40S5txYIPFSRCL4Rh3w8Cq6WvBlkN76gew+9AMETvqjyUD3l+5U63mXUu7OeUz1TdBK+cs5LPduYCD3Bni6+ws3CvPyPtz22bac7+RgPvoK8gbuOhBS+a/WIva7ZSb1xFb+9ByE3vUpfer07DbO9C02jPEy4DL0/Jwq+APBOPI1X4z0A3F87kz8GvXBSCbwIxSu+mTKQPcH5XryfboS9bHIlPHlgjD2fOPk9Lm+vPcRzFb7hvYY8KWkMPZkUFj3jsKO6eTT3vTHwkz1oZRS+GphAvrDfyb2kwAO98xKDvUIsqr3SQMk8Bj2IPb2JJD4//Ko94mODvu661L1b/lG9zwLTvQtbC762INq8OgyKPH4XkrzJwyw9VGg5u/ODnT1JJCi+DJ0OPSgwO73RXSK9duhAu7+NsLsQWP49ksQcPkw0xL3iz+g9UNR5PQspSDz9CL09f72lPdgx+jx8sQy8x+NkvQhJZ7wP6N+7D50mPRrZxLvcZ5y9/g1wPCmzlL1Ms4M9Ik/CO0DWDDyHTWs7N/wHPgAuD7wmpwa7tUm7PHa2Lb2M78u9vvkcPCairb0WM5o9qKQTvkw4i7yKpmm7RDRdO3Es1Txcsz88AJnnvXxWwbx+9To9TCeYuoSyED0p17e74gk6PcbCbrwI+S2+SRKwPc9BJ7ttTaE9PyOovMEggD0PmES8","L1+/u07EQb7ZB9a7Bomwux0b9Lu/xMy9MwXvu32ZyTw44cg8iD9SvaYOhjyg8pQ72r16PPzd5TuoK448gyIMvDQeu7tj6RO9830LvSa7kjsyb5e8EYInPSm2CruI2au8IDInuP5fprtFfSi6JYtJvHqYDry/49I7H1HPO7CYWbx5Bwa+mHMou832Dj2N9Uk8HrUiOu5h4Twq2728a2StO2z1HTxsbTi+wtUDvJ1Z2DvSioO/um9fPGjhpL3YH5u7zp1qO6k6Ubx6q9w5GB9cvC4BuzyvU+g7Qo0rOhOPrLwVo347R4cGvJ21wLszJ4o8d3+IPRMESDuul6g8/w3GvIkXUzwwKzS94w05Pdl0ojrXiVQ8BApzvJm9JT3xH8Y8MAzLvaocHb0P/ic790tsPfaxgb3+MLQ8BySnvGqJaj3gdCc9CG0EvoDc9LzDeNu8WLpGu8SSLzy5p0y9ZNIHvJc0g77NGeG9JbQ+vU+onrx/iVe9WXpvPhSWEz6IhPq9lP+XvMO3+7wCxLK9Oxe9PF6SyDyhaHe9CdUqu/uj27zFRaO8jNKKPbPEpD2iagW9oUARvkEXJj1DRn690enTvXV+ebx7h6C88iAYvZR8Dr5Kuii9Neg8PTzjMD3y5Lm8cPxhveE2A73DBDw9tfjXO1NwFj0JPfI6Gy4MPZfRBL723m49JIIkvSdAbDy0Dea8ommEvRU7QDwYfRU9fv5JPVbPfrzPgKo8SkMFPOnhIr2cVNU9HUlFPbsF3zx29s29jy0pvS4bdb3Dsi08gFsSvTYUET2cnJy77DVbO15oErz44Ic8YFMDvexKH7wY8uC87sVmvVbXQrwiOaw9l6EpvU6Wjryq1gS8ahK/O0RDkD1dqMK8ckfHOt98GLsF1Vw9zEaRPPct7L0TUa68huJKOkFZ3bxX4nS72nI7u74hcr1iwhs9hfbtPdRJj7yL0B6+WEiWvOIE1jw4HDs9y9IbvjgvOTu8aXG8S68avVwZPzyTYcM8pAbavFj9ojy8lDe8","BXwlPdBtc7rz05G8pUc7vMkcaj0lIA290weDPaKYGr0aQM29IWh3PCpvzD2x5S2+/clZPH01szzgdo48rP0ZvbVr0Ty/5qM9RsnQPR/KEj2VTD09PqswPsmPKr3Jgcc8r2fdu/+U5LzGcUE9SyWDOiZnPTyESIy7wsUdPXnmGzzKCR49Lt6WPQhYdT0pZwK9PrY7PV8riz1inXQ9DVcBvTfeeLtODYm9k2y8PHPlsT26WrC9nTcUO4LgTTvanIa7VOz0PHeY8T34s4W9ylBovQtIvT3/M9M8wO/8PdwRlbwyJIS9PgCHvRzWSz1O9/w8EmjuvAa3WDxQX3g97DU9PZuTArz8Z2m8xZQGPVSIxDsejiA6R6wYPtLzp72Dh6u8w9c8Pkw1KL1Zivk8FhLbvCkERL66CcM9XhrBPaKE77wMZmG7OhC2PSi9Pr0Jr/E9oisbveetnbm5GZ89uQgovEVumD3TLwG9hD20vc4DDL1VkVg99ckWuWcaMz7HIiA8VNWFPdhTjTz2tZ89I0VsvDUdM7w8bCm9Y4gHPWlvpb2m9Vi9+ZbjPSepQjwC1jQ+gKGgvZOCFz4yowM8NbekO/c5i73bkTO9tPRNPd26BT6iY7M9PoCiPMSeG70fBYa9FRATvWw4lr3GrGM9BC0+u2lkFD7oDrs9QbmDPczC+b2eamo98fusvrysozwW9ri8XnRvO6ZXLT6ed8w9rrWCPGvLgrzZwY27VtcMvBNHEr2Aa609BbPCPWM9pjxgyzY9siRGPNwgM73Eajg9DyXDPFktmr3Si4Q9hHHdvAsHwbwEqmE9eLHXPBApWj0iXhA+HikVPS2Xmbyr6km8gAItvNNNAz3mNCu9EwBjvaXs7D1P+868h6fOPB7OVbvBeoS80/EIPZqUHL9bUDQ8zUglPnGUlz0ksiI7/+GUvFTIubwlnnu9mEejvEEAE7znaka8UDs5vpoyrjxV4SA9qEcjvZKsk7ubuc286PtRPVbyKzvPWyk+tj4uPVvBOL2aAHm8","O30tOjtdQD3uA3+9owp2v0fUwj3HIIy9SoILvm+dwz3CulI9Io4mPBFMbr0B1yK+TXXKPQFwgDoPGQQ+YHmYvfKrcz4V0Bu+QGeNO3vwzr0p9He+IN+DO3K9+jyAOPK85WLaO2gdPjy6ZGA8qx3TvctMJL2GMwO+ePKevjuerb3b5JW6+WmGPOy8gb3Qots7DAKnPXdQpLy9Fg49xM+2vSxJaL1ySxU82gQXvSjXyz32JNY9z9d5PUSdGj0H8P29Y+pfPdKk1j0Ywo28cUKdPd/vNr3zESC+D3eWvhMOoDwcC6Y9E0gavLdck70M74y8Q9y0PQChEj1SmDy9gRJBPXS60zvpB8O9SUaevejtBT2CNhc9pqNtPb+BqTwqgI06oiH/PChdmruWCbS8VjMLPSA7O70QrQu+GxhBPR9kqbwxEQ495x1RvZJYKDyJD4a9j80UPS1U+TzlPqC8Q3uIvMG5tr2ong68O8wqPSqdGDtd3hG+wHuhPSabB75wQWY9VZ3YO+mGmr3dRCS9vi9lvZrzJz0rm+S9VFoavSCGnLz2B049u6VePNWqY72Q8/O6Ne6WvBSpLb6U9Qs9LtfLPO7u9bstdaa8OHulvtnXGr7DbI69th/huxiHlTyiUTu+wFUcO7APNj2lW7G94Nt0vY+m5TwA8y27SB9evioh7TxTy7A8uVV1veoMBTzR7Eo9Py75PJCH+jtGwQu+s1B6vT56wLyMb2m+JMYBPL9JFrpY0NU8VETwPElVdLw6IS++102tvABcsLzauzM9X5lyPcuF9DwOrwY8icmDO1deAj0BdFS7j+AgvRMhw71yzyw9NPmIu09v370Jq5e9nApOPUhVGT5P/xi9otyzubQ5H71NapW9ySRnPXox07tCUby8qQaCvE8zYr3ZgPA8tYNiOgDLRL3nO/o8hQ2RvEUtAb02q6g7Z707veUYS73zBDA95IIjveY+Zr19lkY9xELnPGoNrTwq/KO8JKFxPQzg0ry8C5S9hw7qvGjfJD0PYvG8","W5Q7uwAiTj1VWt241cO0vIZPZD3w44O98LQePpOdi7vTd1E9XStHPG8YRjzGXp28c4mRvcjjFT0YPoM9wWoAvXpYIz3Ytkm9JA4Fvn54Z7v9fVg8VBNIPYW2J72tkpi8FgIqPCY++bltt4A94LDPu1jyVL3wKVG9cQCLvQprCj3VTFS+M8KFPMRXlD20Pq488L4XuL26jj1myPI7ICWZO1e7ELzNSv68joSzuyzjlryiQDa8IvSxvYWgpLxE0GK8r+dJPPfbwzyPcTm9DI9RPStydb167AG8mPZWPBtCnjugKPS8jHwZvbxrlD2ON5M9YKfCPYssrDy5ogI93mToPBjXQjwC33c92aYZvj0VOb1TZyy7i+Q+vf+4lDy05C48BquxPPfqJDwTkBu98xfEu+MY9bxWgs68L3P7PHi4nTy2j407QqYevody07uP3SI9aiwePB3xVz0upsw9ADfLO3VJU7wNrxa89gl+vJ+o4bxerYQ8upkgvNJOOj27tO26xvNCu6h0Lj0VaZK9GbTeu0U6AT0siuE9GmjqvQnaL7zHQ2G+eWY9PmBBjDtmKrm6pHiePaWCqb5ojto7MT75vThzJr7rftm7yo2SPG/w7r5n6FY8QDAnOkeRH7wPXB2/b8Y+uhLxQbxDR/G7CamzvbTz0Tz8UoO9TcfuvDUdbLyEBqy9HXysu29HCb7mDK287u8oPVevx7tdSW0+mLYYvfaKmTxWn5882GYjPJ/C8z1vBd09Xb4qvRqSMD6niG67XBb0PbedRz2tXdy7LLBUPbltiLwqUzM9tim5O+AgmLxkeIA+eGT1vOIPM7xfiv+8294kPdElrT0SoX2+AnY8PYg3AbxC1F89L+6Cvap0BT0jjaY9WrcvvI9s1by/65M7eKS2vXSRtbz/24u+ZJauO/kdwbxvzck90ivCu29qib3bgWM+lnkVPbBn6T2+eZY86WdIPV0e37x+c4g9mvdzPCfzsb2Hq+u8JNvPO0A+lD1/SGy9PfmGvWwOZzy0uw69","3a0VvGk7sj388aQ9sZmAPMM1Kj2geRk99Ae+PtHQzrtxbso9SbmjvX6+kDw4R6k9ehA9vrmQPL2SeUG9FDqwO1MROz1CukA90kMQPXnsJbucZUC9Bp2iPEIm/TziM7G8oAj3Ouq7nz0NYM69EsKqvMSBNL4E9Bo+3V4IPcVFNj1lmfc8LEpqPclot72MMPq9DJocvhbSdTph5es8As+avCjnsjwptjW9bNqxvXHTrD0/js28aFkePZ2ziD1mBMC9+r9RvQfw2by+MN49YincPR+qtD0v4tu9FdirPYaCLj1QVz09PufPvVhY8j0VjPA89G6qvGhIBT11onk+UUKhvd0HyrxtKbW6GrJ9vXTgobyeVlG7XRzpu2s5MrzbKNY7sBicvRs7kL23EnG80rhdvFtjvbvjmKi8SKsUvW6+37yvYou9jMaKvN0gDz1l6wy/5TwVPUE4tjwk1K88DK9/u/4zDz7AAu08UKiEPD5YBD5g10I8dj7VuguKIz30Zck8zTNRu3017jz6NgU807DmvDw9Zb0sDI87iYQUPBYJiL2GkY28VOL4vDObmDsqCoq9j9k2PR5xDj2E2+C5of9OvR3kvDzK+M68B3nhvWamSzx+Ec2+q0BfPBz/Dzz3dBy8EWU2PbSoHDyUoP88EG1WvPVzq7wPXY27bMALO6pEQjuXYDW8X48EvJjFBb02SR09/l+uvW06uzuvz1G9Nz15PAW8lrzDi4O8OS+Vve6prD1Z6Kq9KFpCPRQGq73er9Y8/hM7vWPNqryR7EC8opSOPLlAnz1N1Lu6lt2tPC+3lr2yeI68noGQvVstYj1mUEg8KVuQvepbhjzdU2A8exuqPY2ghj3Gk4C8SuSUPqyuJj3lejk83iQAvXrctDyb2wI8c9LZu4PVOD3usc49yn1MPblD+DvQBts86cEFPFfDB72SSkU99LF7vb1ol71W9Pq9/L6Xvbvhiz12LJu8CwSFPbAcAD5OCE69lmIzPX6Ymz2t2+Q9XRmquuZNTT1QThY9","+Syou6HRALxwAEw8wC8iOlgID7040x88XNxTvDedWTxEtTC9MynbvEabyDzyfoS+MK/+O2+w5buSG8C5AQGhPCrHNz2Od8y8ZWaAPso+rDw6hZy+HinkvPxe3jxcham9JV8jv3RWOT3zshw8Cq9lvCOvHj1UwhW8EFV/vN1lYrxRv8W+2Zrju/A61bsm0aS8H96+PbXCgbxONz4815rgPHHxDLwScoO7lgdKuvYFNb0fepE83i8APArE77tOJBQ88Q3EO/MGMrslEh69aKvcuzBFQr4wE6y7AMaKv+P+D7rYzDo82qeQPG3fljwECdA7ndXyPLL8vjrgFL27N0MkvJRg8TyRAq88pq4jvXmjnz0YYMI8dHpKveR3I70PHYA9CognvgUDfjzBWiO9ChCoPMiaVT2bakE9JqHyvaYSz7yjdjW8pvG+vSIHarzucNi8eM3LvWFKnr3tEdO9LficvQobWj349mM8agJzPb12DD2s2H88UHMXPBSBTzw+O2E9vmkYPNCZ5Lyh/pE9R09SPJAhDT2Inok8zcTxvUkV6Ty01mw9PGUZPX8umb2Itxa98ISAPXPo/LwrWnU8ZiTRu0Tavbu4lbu9um8cvTzY+zwmApS8/KaGPU+pn73KaDG8QcJ1PVbcIL5VQbU9CuOpve9cur16mgi8hZd6vYHJG7zf4w89HYM9vkCdUD3X4R89rSnhvPS0aT7o1Gy9xq1ZPbcN1L4e+Oq98MWhvIRNor4/LOg9BirVOhleCL3uxIu++BCsvUuABL1bBm68BOO/PLknqD3G7xG+p8nBvEmx1b0PRoK978adPXF3ED3ZXm292S83u6fYlz0xHN0+QBYEvjiDH76lzCM9cnLhvSyk3j1T3Mi9lTYhvX9GkT3yUrS9ZrLPO5D2Iz2KBra7WjpLPPpXKL7FP349FqKGPOvsybvyDf29J+civY4Trjyf1u86v3eSPcVACb2iP5+8+swyui8G8L0iivu97DkVvXjB67076E4+D2Y0PYnN9j28uFU9","xi4Fvot8rzoK0Mi8LdYVPdjnXr19hDs8+038u0W/8rzQ0aS+2t5DPc+tk70+efi981YDPH629bzmKow8eqcnvmVgFr2XIjA9PTZPvesFzb0LeZ88+H0OPQDpBL7PTR46/EmNvmMaCz5nEAs9/GeQvSqtw70YtQA5tkd6vT27j7wYnYY+QU4IPfT+Nr3pCbU8F4fePBp/sL2WY4i9c4GcvNTCrTx5jV+9Kn5HvQEfsj3BYCA9O8hUPf7Jmr0FxHi9BDP3O72nWj0ipPY9FdV3Pfhri763uYC9sIYhPlW/S70P+Fi9XeCMvcFDF77z58c844IVvTIYjzxk24E9zfcQu/WGq70F88S+9z4zPBLc1j2QY+25zZCsPBplgr6A8la9niIJPq5h370RH4g9b4vcPfdPpT0RBRk9O/7DvoY9gL0SwPK+ED3KvYAHdLwk2eE5/MYhPP+3ojzBA8C8hScOPPZdmbyrkhA8Kpz2PAEBxLz5EZm97ldkPpY/ED9gFbC75winPEzVKr1PffE9RT9VPbPfur3DY/K8GPWbPRNMYz2/A/q6coivPizS1L084Gq9zKm1PVfitr0zV2O8ePTrO2BWGL59rf47X840PYAdWz5a5gG9tjJMvBkP3TxOFrW9ctE6u0ALAD0ZAw09CKrqPE2PiTxyqxY+17FyPV5rDr69o1Y9OI9TvbefsrwWdQm/gKGEvq4d1ryPnn67e6+ePaYBHz4XMJ29sJjJvshHiT69NIQ9eIvrPYktSj3NiwG+ehhEvnb4zz2kbke+q9eHvuvGP7z51lq9e7vJvYtQ3r0acUo7FUR6PkV4Vz3SCqy9NXQqPZUQhL2g0cq/L38zvm4jXr5Ikhk+lXsRPpb5z704qju+KIklvGu8cL18UnI7nMGgOxq5c745Nvk9FPt1PKPBmD32I60+rTUIvnyOkr3k1rS98q2Jvif0Zr0tojC+d9WHvoof+L3yLf6+ap0FPhNl9TrbTyY+7PjoPbrHSD4Q9Tq+wbHZuhRBLj7/Y449","qFJbvF47FL5PeD67b7pjuxUPFrydzjU+xPSGvPeC2rsVMoE8922zOX33k7vs7kA8EZIIPCJKZDzXWbI8Bw9UvK5ArbtslBk8UqYTvr+7YLxPsj26jUmGPAd/EDzkAg48vmq6OR5qVjyRs7I8qeUUu8Lcu7sAlyY8AzqzvB64a7z+d/6+vIudu//h8byPGNE8PeCBvPm3sDw4Wr082FGAvMX6QjzL/4a+MemdOXz9aLtSGPW+gBRhvEYPrbzta4m79pk6u03UEztshQy7JNwlvHviijwftwW8XBAvu+I/gTytI0g7wMW6vLQA3Tu2AsG8plvXvP55mTyAcb06r7KBvfNmv72h+d882O2HvQPuYb7ODYS6gg/HvEkfFD74z2e9TFe5PfI1K72KlxA+5HrRvW9b+r2zgA0+QsqTvQVEST1us7+96SaFPaxki7zorRM+Q/6RPe6lzzuNNpC+7/B6vOgmtb0WHJg9EYNUvJmIqL5bxiW8acZ3vQglBb0OoOC9p9uzvAfJqT2yo4M9yG39PLNXhbsZYeQ7gH1vPVRUOz6YerC9W3OIPehcKj1Zt8095XM9PXXg3L0an0u9dEnxvE3biT3OCYs+rSh5PbMM0rtIO3c9XFLcven7prsGpZK+HMqYvaF/Qb1yPLU8E/wBPT/v7z1svsI9P74vvjfLBD339BS+X35tPuGXsj3QRqi9ZP5AvM+C5TwYRDg9EghkPSLKvzyYFpi9CFslv+6+hz1IRSs9jqW8vD4PtD7QVD88iRoYPT2NG7wTSZW99WnrvKPGUTyptEC+w4nzPE93Mz0/O5S9drG6PBA3xbsZRqY9wcx2PXsyA76hDZu9ASpMvmIUzDzcZkc8s0xaPTedcz2fifA8GIvDPF5GAL5egY+9G263PXyG171XgIa9GZz+PKcBCDw1PRg93eCpvL+eGb2eJFE94qLNPbXTfLwRHHy9TSenPQdEDz5SppM97EhIPUxdfT0du9K+BgaZPS/5GbzlGy09k5PTvaeWMj2eEUS9","9WsevHJECbwk9BK8KNmVvq5jnTzadVA75WDBPOQgjrt9AUS4fqsfOxazYLwNtZo7PBJGu7uWBLtY2EW/F09SvCF8Vr+BDe47HD4uvNSpY7rcceI3bn5qu2g1aTresRs8sSUsO7pNJLylnZU7XSDivF23L7yayeg8VDjGvgchKDtn3fc65Q0yuyygez2MawQ8icaXO8SdyzkGG3A8PswMu4Bu3zswqgm8cysrv4t3Mzxeduk78FgXuiY45LvW/xg7+qa/Ow+cjLqcfIO884c7uzlmq7wPOuE7XbRovMe+NDxcwwy7BJw/OU8KkrlGMhG7EOGSO6XaejtBHAC7BKO5Of1tL70wy2i9M8GzPT2LCT2JXEs98j4WPjGVND4xHXK8CxGOPObnUTyyuaE8qCchvZ+rXL2TkHc9JBBRvUvmo701TLK9ldx1O8B5Hz1aFOM9jWpuPTNF8b2r9QM9QYiKvGLMo7xuyNA9LFcxPeIfzTydyCW9TC0cPktUrz6t9FW73j6APRBj6bwBDUC+ryhOvVuVmz2Fsxo9pd7LPVejrbyT7t68EoIGvn0lRDyRm+49Qv8DvaYTtzx68S89RqK7PDbEG714+R68FpefPemoAj5wHew8CGlwvYQGLD3iFEA9aqEXvcSQRr5WANK8HDEnvZR0tz4QU9m7cUyjvPXRg72tnVe9T8gFPSlnYLwdkXO/kgcgPSrvDD3RhKa8VOhQO08ERr8CUji99O2BPWCSXDwKPn+7mgrZPMOK0ryDhwa85oYJPVM7ez4x/7o8cs0CvXH0kz1BJYO7hFEcPNkaOLx33Eu9AO2MOffbSr2EwEE92zsfPXyB8jzYi4W+IadUvY9gH7651he9Aik8Pcp+yTv5966881atvFQMCTrfHb2+PCuRPCfOgz5Qa3M9wAorvWLOrz2HKX+9L3X7vJkAGr5YK6k8r1NYPEREw72CkEi/ejfmO3UQtLvqrx88oryEvQ+YorwuQAc8kZS6vAWemzw8luG/lgtlOxGoL73kZJM9","KNqAuskMCTuhcIM7LW25uo11UjvsJxs8rriguo45kDovC/k3gnWHPGSkAr1Cpha+uXuxvJeuGjsD6kK6BSODPLAzMD20tRo81SHKvKMOcDw4YuO8DQlrvcsHMj0/vsm90OBcvQquXrxZ2xM9PHcXPOfDtTxKApg6qh4nvOX2iL1v92A9uGCtPCb9iDxUVC88ZG4aPfFPqTzduyY8CizgvMcHQrz2hDY8zGIuPEU1Az0vyZi8TsRMPEcA9DtfaSa5xuAWvKgvgDzrghQ9/GYKvPt4Wz1fWe2+W+2+vVX7hzsHXRc7i5g7PkalX7q+3i06U+r8O2TMw7ud9D+8Blq4Ow+mqzy9mRe9Mb3kPLUg+LsWfAG+i+mvvOXNb77quRW4iSLdvFJx0jxLxxu8PRqxPY2tCDwcATI73LkOPYLcXTyYD5Y81YLtvFLLBb1J6By9u3uruwvcIr2HbBg8UtXau+b4cztIE6e7aG2cvbWd5ztwXxk9TOLhv1fZoz5+b6s9McFLu3YKbDsVuec7t5a9PIpQiL1LR6U8t/eZPNJN9zwvoWa8tk6SvCIHab2mGj67UTbTvOm9B7wv0vw8JCnRPBfunrtysH48vInuPFxKFj2QfGU8GyKLvMn9uztTMdC78PowO/LLzr7fQtC834cIvIv2Gr1vqAA8aFudO8HZNT1D9OQ8E3TvvJtszrxxuRG+hSuJvTBDjD3hsXE+8fPoPTWxYDzDlqa895AbPieH67zp9D0+l/jZvQCyKT4pE9u84h8PPYqfurxUG7c8lcsoPbMB873w7D08UpYPPeOSiD2e7dY9AcMkPhnsDb3uA/Q9HlsPPdAi/Dsl9Ae8jXErPQTzPz14DFg8s8HFPrTpoz2zGrs8G0Y3PWsljDyMZCc9GWRpvQ5vpD1gK7a8tyZJPjrkPTv9C609SbKbvCcVfz10J2279nWIvGY8tL3lsIa82JZLvV1phL1Gtgm90EEuvKHdh73XkcI9CldAPNLT6r3yZ4C9rmCsu8A94DxK6Y89","7M9ZvYBn9TzhQFo9mOC3PS7YGD1RpnS9zBmPvd+Lc7yamBm9bTOEvLWMnz0H0Dy9WDZlvc3VHrzVaA69xjn5PODSFDo+PLS749NovYXMdr0zLUS7qP1YPXOqnj09JF897QFkunLz6L1p5yW8OlIGvG+ds72BsQq97357PVzHRj1bg7U96h8LvZqXgT5fDqs963a8vJfHhz3m6A48YGG8vUl+CD0OJA+72lbNPXoNTL1J2fS9K6UDPPaF7TltslW8dNWPO2x8gr2HWCc9rYovvUyFPD3IUBC9l1I/vcHRDb2UThW9/qagvbseRT3Pi8s9EhRyPXfyZbq76Pm8KFszvAUR5T0CDqq9Lnm1vWVTHr7qVhY7xA9tvWvqqj7XhNo76aeou8PuHT0LJ4O+rJ+qPfXCVD5+O4U9sfNBPsuinr0HYUQ+P9rwvOj7qD1DCoK96QARvg/sqTzCueg91TfyPda8sj0tOQY+25VKvddsvr0rO2k+2oERvdAOjL5Tk9m+dpJ/vnp4wT14jXw+6iD6PS75c72+Yuo9Q7LTvJO9Xj565gy+ZHZJvS7e6T3uM3Q+XfZhPStPiz2i+iA9X67MvQ6jID1szKS9kbISvmycCT6qWoO9DSJLvgYe5L0ULXu9H/4FvuD9D745DiG+o6bLPSI5oz1r5428I+1GvRPlBD4UMKE8LykFPgTIeb1692E9vF0FvFNsBD09DBa9TpZkvWJYsL2SVxK9xSNQvaw8Sz0VH9w9/q0+vbmTh7wVCms9kEowvdYuDr0Mtao9bFbROjp03Ll8Xco8S3G5vdcgmz1L4Fw8m/kdvGniaTw4nTI9r+MDPHV7uLyABdM8XijOPRvBRjw73je8oVAvPJxW1b0Ug0M9oZ8OPtvh/r2T73M8HNfyvbm1Wz5Kr209gR8XvY+Vm71lc66+u+7RPeO7XTxrCya9p2wqPAIsyD0cGR89fKMEvQ3Xjj2BOVY9T02ZPXXwzTtzpV6+851BPAlfmT105Kc9THCkvZB84j2yPqs8","0+oMvwe6CrzuM348fSWtvIJs1DwjWlm70TymvK/CLTyNhAQ8Ry4dvAP69rlVvk09M/vAvMAwE77b7Rg9dznMu0eIr7zh5T88iPnIvGknYjzgXmU+0EsYvLIkLT1+5JS8kNxpvVtsJDzlzba8kMuGPepaMzyq3XM8gz5GPI8N3DpO9bM6kxERvHtZoL0BG2k8XM1lvhtRlL1JwKk7Q+q1OWKTLz3btL+8SebnO8AftTxnx1+8HvyDOoDbkzwocKU8ATKjPBL5ebwOWaC8V8KRu0bNDrthuQ88aZCHvDQGU7xrAe2+IK74PJKKQDzkbG28+mXqOj1OBbwHnIW+9dK1vA4mrL1a+js8Y95EPkhxdr17Fl49hP00Pf+n1D1ZQJE7hZsDPrLtoryH7Wa9SS7qvWnBBL3P26g8NNsnvFjHMT4IbyY+eccAPaSS+b0ef6W8zMyJvQPVAL3eO1m9pSoDO6ExFzw3a0y9KTsGPShqabwGG9G8wQqMvVK5zbwCNyG9l44CPZdkLrxHVSO+BmRUvYwLkT0t7+W9UiSZPeUxND1EEBG91ULTPKXRUr4hylI7pTuRvEqLgb3xAEg9JOpnvNKGzzu75+E8FR5svcV7Dr57GDm9iTsRPm9HlDww5JQ8hs+9PJXyuj0BzV09KxinvAQhLr1brvK8B0LlPQnEnTwvjQS97JgPvF7pP71mjeS7LS0YPTLkwrwD6SA+SCkXvG3XQ73kLdu8+nVQvB3bPz0NqX889+ThPZdmKT1bUNM8PTTYu5y8rToP+Iu9wiybPYps8r0sFwK8RhnQve1EHj1UaGO+bXdwvWcQ9ryYed094YwJPdnI8Dx0aOg7uTatO1Jy3Dy1vC6+CRhjPWLW6j1TPpS9LESUPYdsqL1TvPk889XfvFgp4jrjFsG927rfPWzHI7y0cEc9kIBNvMqugzwR31g7r3gkPXS2+z19dw29L8m5PIZR5L3EyiW9QOsqPDOcSD2fInM9Hj44Pc2i77xYQO88CO8wvEqHGr6vL5Q9","B8obvXVvejzFjwk8zZ6/vf76D7/i2/y9fpoCvuD+eT001Q8+tNyvO1p1q75oLEs+p0+NPhEtMz7uiao+3+jJvVOMob7+Vpa+prCxvQjy97mHVxu+BjrBvTisMr6GzBu+f6wHPjCaqD6Cidq9S2tlvfq8dTzuAr29MOVPPZOgxL6XY1y+Q6YfPS8V+j76LVQ+NPz6PTOXNz05tq+94q3bO5kcJbxMiOm9cZeqPg7d1D0EhZy9WQuRvmAyPL5mu908TEAFvQf/qb1V2h69IDuwPF25ob7oXd09m5/XvhPzAD5Q1gK+jzg9vnPKw73ZEZY8s4GRvu17qDyv1eE8U52FPXcn87yRatS8i7fBO+Yngb5WrRA8FB0vvUnFub16I4I9d4iPvI8Pkj2foFk9H2YQPtcj/Lwagds7YmS3PUX1Nz3z/4s9VsWXvZjY671a4pm9zz0GPv4tND0t6d27PWrxPeDnUb5XHHQ94aKhPOdjJLzJBYO7nEpXvc092L5k+QQ9EYnWPH4kpTw9Dh89aKGsuwoRZr2m2Zw7JpwxvdmGB71Q5IU9asJfPTUtoL3czQI96MNNPTLPyr1HXYe9DA/FPNiyXD0rh0q+JfffvTyNVD1SyIw9qXuYvoBkCb8P3eG82Cs3PeOfRT5isBA+s1IOPcatZz10ZpY856YCvdRsaD1o0NU82g2MvLvw7zq2YE++3LkwPH20a7tE+Ku8YZ0FvHHisDtuH607Q0Y2vPxv27xDO2m9ijKrum9KVb/RHK+7AoxgvzG3tDvgrzm9GRM4u0EYWL0A4TY6mxSZuj/UkLusGK08VXVhPGhVEDzfgck8Phc0u7+Fezy+QuS+M+WBvLDxVryt4+Y4rNQlv9S+s7qoIfS90Qu6vAu4ujvfxNi7AFO1u6d8DLy2Vxm/k2OxPD95KLw5iaG732AJPCv0nbu+LpY7ozQDvPS8STxAj6a7My3hOe62mjx4rFE7nu1YuotZPr3FZUA9P+3DPFz6dLuZrka8hTLkOaSDR7xys0m8","nBpIPO6Tr7v5RVE9O4tPPEQw/ztxhHe9HIt3vlsVG72FhI47lUrrPOZ95Tw1+AM8ZfsSvqWwrzsZ7wU9xnoWPtCf8jyF/6u9xHqaPSz6tDx3U7A7NPXVO07i+T0RLEw948KSvWGvrD0GtVi9YEWKvcNOQz1T2xe96CKVPNXHobwEHq29GHnHvT99Fj5xmf08ayIHvgXr4T3BiJ49qEbfPEULC72F4Yy853NgPbuh87xlnh09Y4wpPRjgdb2ehUk9hvYGPHZWkrw/dz09xj6ePKgZjT1Befs8AKWfvdyi87wgGfG9aOF0venyFb38zOS9MrJ8vFe4FLsRFaM9c34bPp5lLztqoh08utupuyE8rTt1PA+8REUwvCkBKTsI9Jo7/S4AvAlNqrx+Soy7p9rjuxK5sznoNrO9j25svO6LDb5YC4S8XjLFvczMFD3czTi+bWLgvPkwrbs6b/292ZOku1HdOzlp0TE9kvexvM6esbzA6pe9pUWXO/CtBz1ON2G7TtaiPL+YBj7NTca7isfTPSZ9EL3XpQe9EFzpvDXXKb6queI8ofdJPIPVBzwI/Qu+4rWWPJAggT3LzxA9AzNiPaIKLD1aaxu+GhZWvWTd3zwAV5W+UaEyvUExGj0iu4s8RB0lvCmPVjxpqsC8UlaOPTN+0DzziI88vaLdPJBODbtWdNE7DxYlvmgKFL4z15g7oYjOvAWvCD5HbrQ+xLrDPCAfMD5H8Vg9iQDIO/DE3j28niW+HbfMvX4nt7sy9TA9lYVLPU8pjz6/VQc+5sUIPTT8aD2gjie9v6UlPEvyIb54u6e9YlgZvia/oL58gri+nuT8PbBKHD4/e608fBKOvu4FtLzfgq67lPBvvPjrlTwJUZK+XEuJvbSEjL30sIY9C58aPW95yz53VB483/w/u5MFlLoMWdK960PTvd/HNL6CxNY8af6LvSnuxr2pdV89N6B9Pm4OI72LlMY8pdeYvQuz/jz1ena+OQKQPYn9vD3k5EE+69cIPY2DtD3k/Gc8","XpQ/vE7DEz6mboo9KbHTvTURqL5HPD6+9lQVP/ZrAT4FgFo9pcxPvdoacr5h6Kk+++wwPsuc0L1LynA+bc2hPXd8lL5ajp++6uSivvdcFzxeTAM9sC6nPR+g5bz3HfG+KWIePnhMq7pxyiy9iutOPcg+rzy5pow8Zlt9v03akb6/odu8S+ejPvi8iT7ZjeE9OLtQPcvHVj73txG8ntbWPV5/Czwx3Xm+YSIVvXH/Vz4I/go9IdYqPobGaL4ZUEs+JsTMvc6C2z006JY8jCgIvFJzgr2ZvyQ9us82vknGTj2yZHo8EasyPFkEib2PKye959C8vohMVb0dlKw8feYCPqCwJ70iBwq9G4AZPb2SPjy8iJ090tFAvKMEFz7ktFU9H0KbPRzQLb2XEU6+2AwAvbJf3b27Ta49kYLNO6Os7bxAFRW+q1FIPVVlSr0513U7hOdUvYPYD77PlZ09cuTavP6hVzy+52U9F5FcOwC6qTpG/be94vjNPVQrA77yC9A8o6zFvDwAvjykD427j7lyPD4+i71twLu80J3tPBK4qb3r6Zs99KdQuyE9KL7/lj29XHucPbzQTz1+5LM9frBVvPNXBD2q7zO+f17yva6bXb39HSM85j83Oz/TFr1kes+8N/WjPYv/ob2/iOS5eoEjPQW6/LyNMw29fM9EvWYNibxkA3c8y3qPvYCcAb3hnyS9b8EoPQ1llDyikp66oFXQPEQxU7yag9s7b3ATu+JGDLycfYY87soOvc3c/7uAvHU82dghvATsqL1ljkS9qTfcvGsoCr00Wfi8yrZ+vOZe+btxbYU+hyUgvSzYkLxROPO8vQ2PPYosdL3IsYy8w5wmvLR+QryDwYO7sn9Eu8rCgr0zOIG9s3DPPGTKDz2qeHK6cHvCO2xiy70nUFM9hRxEOoNpKLxU1yq8PyQavgJ0+Ty2BSs9eZtlvBZD0zyniwK+szkEPAuebLw6/3e9tz1BPJ54OTuOE4E9xuI2vMptgb1c1Iy94hMUvODBsb3MXSk9","4qVsveJ+lrxJxNg8FJ6ZvyFHmr0pKQU8GYBaPmPfcbz9dXG9utsqvY3Wnb77YeY9RTH2PfO5Wz2F79k8Fe8BvV1+kj2a37u839Yxvmga/b0Jmme+Nyw4vY+0k73y0Qi+BJFUPT1aNb1GTlu9uhIFPYkPU71CDC49nOIgPK/2rL6yZAu+db5rvWrMs7vnbWY8JPV7vdbC5rxUq5w9ASFFPVBkIT1vGac73G4yPLy38Tw/xLI80QoCveragT1zWK69wBwPvHY37LzIS1u8wXYTvawKiD1uHo89A0DuvVqzP7zRY/C8Xnq2PnSJyj1scKM9/aoivYK1g7xWyOM8AjSVvHY77zxWTqU9Om4NPWB8p7zFJOS8srQiPINdgj1QcAE96Z1vvdgcoztoP548ePpaPbJfq7tQlDS90BtGu/tLYzxDi8s91Bg1PPvMIz4j0Ca7g6IBPhffh71DyLg8vQLXO9pI+T2JUze9v+1zO5ZcVb2iiMU9bvgHPaiWlrx8WzO90CSZvyOBozxoYBA+PXQCvaCECj1RCJQ9HpdCPOc+0rwlyQC8pegRPYlE2ru+WYQ9HWPfvPbYKL12RGg9mCfOPGAq4jxVK589387cvOGZDj09HVi97fAhPmlA67y9yGK8WL4KvXXr/D2NzBQ7jfSEPd2Bv73k0dE60HoAPW4khDv3hDC804+Pup4qgjxinS88OT6Nvr9oGbxQaR69f5kzOzNp9LzvjwK+lkZ4u7Lem7xeB4a6QPYbPGV1TTvCaIe8DilTvS2m2zp+X8w9imcdPKByBzuNmA6/oQh0vFqLEb/FWpC8EPeWvW02072iwFQ5peizu5XLE7waoxy/AXy5vv5A6LyAHfG7/5JbvArrTTw0zFo6sPVaPMs3E70NxAc8dUFdO+gKdDx5N+a8HnIhu2SC4LxnyWu8rV6TvJ6K1DsSP+68wjn7OrXfkjzJqJ+7KQmXOrQ+G7wJvXY65Z1EOSrpArxSKwc/zAehvB3PyrzPEGK+SLZhPIKSoTzbOi+9","+GDGu7iWkr2sppI80wDrPLFfET4twg69r8EXvn7itrx6G1E9sZOSPA8KQDxRP3a7agHWPKFsE715DbW8UOoPvdmqMD2vB949UNwuPWmakr2/LcC8NwNmvLeb2jx2W4K8xsACvmvOgLxvTEI9rSCTPHY/Ob1dA2g+1P5PvtPXiz2QT/y8qgBePW8vkbyyuKi8iPK0Pdqc7L2ziLy9XLrMO4leRDx1xia9vdICu5/ETL3/14q9Hmk6PJj1H73svhy9biPGvF3kt7hSBGO7e5T/vSv2hz26E607gRlqPStoLD0+G2c8oV9zvd+XdbxOX9U8w0ASvhSskr0BdJC8mksNveT3QDyF0B69gzdzPVNsSj1M58M8tSk+PRiJqb2EVVS9vAOdvff/rz0LivY9/MT4vKNEZb0ttIU8/C5gPDMO+by7p4Y6R+aIvROR0z085Am9NuxsvVBUrDviYjG9kYOKvONN173mITW+GX83PaC73bwR+5M82rdKvQ9vbL36JYk9gb5ZPoD/rr0Vv826a1COvftgZ77+Jb29+vbYPCMBFL3g4V294F4Avc/ZiTs+dda9Ecy9vdxuGr3sFoi9tI55OheIyLzm3bC7hqAAu/E08Lv/SA4+MOxZvsiKKz2O6xM9lhAmvbjA673fgOq93V3OO+jydDtGQLc7jgxdvdabP70Adp09MjAtvB3Akb1zpFG9M8CevfqBljyNi8S9hxzKvS/4RL3QmhK8jI9sPeKJ8Dzab/a8wYw5PcDR6jxKjwa++jaHPA6WJj1X9gS9nF6pvVq5Ob0kYh+8KhkqPLcSbry6yYs9Nehxvdwi4bvvnCm+FkSgPCqi6rxPOUI8NzurvNDpOT0uqoK9BAmiPbO22LxGcEe+rHCevXsaYTycuv+8JVI8veLDCDy2zf28b4Njvkr53b23wkY9M4FXvRKR/7zdizg8eKEDvt03Aj2NoZw78WMIvJ+edT2x4Is9E+cqPd6qt70gfL+8Heg0vZ74Ir2iGY+9ZNplu2vLl7074Ek9","M2iJPeIlF757ZnC9TVaKvTYsfzyU4Xu8R6pOPR+8/zwCBs87DxwBvCa3rr27Dvo7EwhUN1sl+jzZCom8iEKePZjbljtofBe+f95cPGhRmr1rO/O9bpxEPRheFry2Jek9boskPdrXF72rbki7Gh6WvUnFM70Yi8A8FRDgPCYKJ72DyeO9zIdwPZzBLDyqZBO7fkjVuw5yMLxlHpE8xElXPaQXG717wna8/SPzPf2aeD1I1D0+g3vFvSiJi73zjA293w/OPNU6AD5l9Q+8XFeTPdG9tLxrqZm7d0vMvYcmX712em498S5yvdBvJrzBDRY8N/7XO+bYFj1rRZo9gWtwPfINcr1got49zB4/PayG2b56vSO/Qf6AvWwsqT6JmyA+xcB2PfFByz12AIa+CaR2Pip7JT6Oork9034cPrjK5j1g7Na9IYogvtUtFr3b1nc9pOR9vr/P3b0FoSC+O+6Xvi/8QT5qM78+mB5gPTO5E766CYK86xd7vjT8cr/8qrC+iMXcvmCdrD5B0QM/YhZTPr82kD5i4dw9oa6svPWEp7z4QZs8qemBvjWOD73ghrI+PhSaPsKldz4ZHIC+3+vSPeo2D750W2K9xTKvPRnvKr34+Yo71fOAPvnsrr7oO8U9Zg75vJCR3z1Evag8OCmBPa6XUL/QJ/29+ALjPXxFuj1UuaG5MNpiPCzkEr7afZk8lcg0vBW9iTu6p568kKHHu/WyXLxJnXS6xUIMO/JBpru5UbK796TpPH0cp7w9Fqw8xJm+PH+CdLx8Q9O5mxnYvt9gVjvm2R+7PrqcvmD75TtPFha8OftpO/eWNLxB4mO9BoOnvmKKJ7zKpzA8BDwLvIXk4Tr0KfM8C2kvPaP2ybzZyec40NqHvljfgzv/7Pk7T47NvjwnKzz0eRI995kTPPOxoLyEjyG+qA2Nu3cxPb1LORe/a6YTu1Mn2b4mXhO7nI8wO8kwRbwe2ro7QC2CvSS8STyyOBg9h++bumcIj70N3mu60iEOvtu1Fr41X6A7","sjlvu0U8kj2BRUU9HdhavBOYGb9KXes6UWMJv8BtSTx+r029+BwovRb4zjvtM3O9NuzfvY3Otrz9cDS9sPQmvBsfGj0fN1E7k2lCPv/UI7z+sMu8qiPEvZM+/TvHJUu+F3vjvcgP5r2Ue2S+u4DnPMgxuT3AwhG+Xu4aPuKFab/RKFa+EBZAvLJ4uzw1Mv48I1qjvZijGz3qLZ29pthVPNjQLrw3fRe9EMuVPXkBHTx3jDa9+2Y8PWNKD73C5EA85nfgupcxDL12qjO8kuqMvBeFrTzw0V+9ma49Oi4yCry8Kxo9T4jzvGtoUr1lFrC8jX6Bv6e+iTvrbre8M+mWPLFu4LwNl4y+LXAiuxEhjz2jhl88MsJePXCQJr2cRjW8K+SnvRwGpTzCvVS0L73UPD5DCrvEmXi9XNbuuhSpBr31zai9ScEGv03RAbv9tyM8wT0AvQH2xDsg5wU9SZUNu7J6ML2UnMo7smrZPI3LTL1ORju9q7XqvVJZN76H9QO+IpxxPEQXD7xxMAI5QusEvNXqK7z1a5U7af7RvVLZ37sWMPe8Z8WIPizMwby4lY+7qJHEvKphOD6BH66+N1OavOWRRb6qgVk82ArEvNZqjr50z4c9thlSvD13krxK/T++iLAEvQ2x0z05ey67mjGYvfg2QD29CNe90HeVPV1x8Ts2gGg9JKKQPMcGW72zBPU7ezjJO7wier22Way90DkEPfNE3zwHnx090viHPeqPWz0J2YS+lBfHvGdnxrxLlDi7JYDjvDXeFb6o1FC8XFpGPW8/azu6NKk8sLkJviFhfT3gr6a9xM2iPCYeDTwsTBm+HFMBPQ0fhTzOQYm9y+LSPISDKT3NdQe7Wu5GPlgeiL0sarQ7sK8Avs2JJT3E9h4+LL2cu136RjyB8cE94QYUPlICrTu/SWI9X40ZveM987zxKcw8ZoEVPf4qMrwhBcA9S50UPvvA471GIuY8Nff7vf0EBb7gsYC+NnoBvbjyn72IgRU9MGbKO16oNbuKI708","IKmrPCn/Cb5KW+k8g+wYPBnj/jxKPQc953dePgMKo7y59ow9ogXPui9kgjwHBZM82fj5PQVBQbz5o3s9roHuvGqMl70bOnC7qn5PvsgbwL33Zt08hvY4PavCLj3hbti8J3ABunX9Cjw4igy8Ajs/PSwQ8zylc+o9BTUoPiL5Vz3nbvs9u6+CPWVhgj6rU4g9eS6tumIL8r2yRWS9W5yKvRqk2Lykbcs812RnvkspKT1S3Dm+6nCTvUSI5zyLD7k8IHi/O4Fodz3ftsk4p6tyvevR8bxBSjc+CNxpvBb7EbpsJgA9J6HJvO6mQD3/Vis9lR9JPXw37zuMe7g9aHx8vNFURbwQcPk8wkEfO1MRZ74OUzk9mP2nu5Rgmr3w1SI9WMMvPADpKr2/RrW9AH4oPR3Vab2qDXW9jUdOPYd0DztFQSq97uixvtqThD3trJ481FLxvO17ALv50GQ9OFc1PJ7F+T37lXU9JD8IvZkb4D26IFW9Fx3BPHwcdz0uESa7qeuIPFVTtbxVvoi9YZBgPfidpzxg2Ae9vbW6vAVmsrzmg1G9vTo6vrn7YL7rqog8HoHtvTvUCr4gMKu8isO0vee2qryT69K5d8tmPLdltr0/XKC8HBsUvKQNkb1jmgW+J6VYvEApELwEsMw71ut3vt/mN72vSBi9+RjePDIeFb2TTTC8GCziPSlObTt7UFM9MzYlvaFb+bzAvqO8wujUOfZESL284I67uI4kvQjPcT2jx5q66DqePEYXNz209cu9yFDVPe5F0jyIy0Y9jpijPaNr1r3RJKu8mFxOvXDESby2mFC9nLg+PARXlD1Dqzy9T8hOPf6tk70EzL68cMK3PHPtu715vMa96ie1vkUDvbskbiq9MtX+PRB8Ijw3/Os8Z7mdPInjAjxU8NU8H09vvEGGvj3fsw29q+rgO20Sjzw0Dhq9iTs7vJ97zzuM3+y9neKguynnKT396j29aNhBPCX6Gb0joEY+fFBjPCgdOLtQ3jw9Vf8HvYo6arvzLag8","gg1Xuwqrkr1MMvs8kgGdPFNrgz38OAy9XIhlPifWV7170cs7H9WpPebO/D319b89DkEmvXuhfrzn7Rm+bGTdvfKmUjy5wt28Bp5YvTzlJD5v1Ni7+0hSPaHOAL0p9Ik90D6XOz+7273qvBo9mW2dPOZYobsyUBS9eruqvRqDnT2Ik129s+eCvHRSPT4BK8S9cnTBuuwPBz3oYxI9KqXfPeyHcr1n1SA9LYVdPfVJ47wPIZc7tyr3PcyuRDxdTgy8Ut3CPHPq7Lyei+Y8FxzDvAtiqzy6jAg5EdoyvUOjLD6M+8M76U4YvCamoT36exu9d7mIPaZufj0tlbc8EzGBvfV4xrybCa+99ppCPEgh4byg1He8KvwJPKPFGb14XlI7w/9HvI/OvTwBpdg87I6CvdGc9zwHqG28ewg+vOyuK77xL789zC1dO0Bg+zxPXda9dCAHvKMYXjx1qoC8gyINPBIRgzwXdCI9nG7au4Vw270Hp0i955aSO7159jwSiVu5UYtaPTwQ5DuaJw+9Zf4gvoo+iTxljhE9irf+PAmBNj3Isco8uKFyvW6GBTsySdq9IZ0mvdPCxbyIJ+y6fJlku8lKBr33rSK9sYc/PYfxdLyA1kK+yhORPEWGJLxi+c27idEKO35jkj1KSZ2+BszrPJqF5DwMgb682fSyvRg8Yrw3leC90VbdPWbf6b2Ez+C+Y3cPPb9CoDwlJTA+ISFLPqCoDj56MWO8YjLXPYMsGzxD1bw+/ksAPxSgUr5tM5E+B7wZvBcd6r22mTi9kuidvgyDhz5HNiK8bNXHPuBNC76oUiQ+dmskPfvgsj3DAbO9aLqSvsxviz1yLfY7vK0KvlTVlj4mMtc+iZ6lPv6EKD6Oexm+QArjPYrQhb1BicM+WnQsvoxtmb7qFRo+jqc3PikPF75om5M9t0qsveAR0T19YoY9GJT8PngJpb19VCg+7vS1vvrggz0n7AG7qXyOvh/wnz10Doy9p6TMuiCcCL74UNs9rkWtvZHrAb6CsnI9","lYfaPLwyQ7ywf0c7+Te3vGJrWzw6c128zwjyvuvydzutgVK7JJ9OOuLy+brKfZm6uEOvO0YpTDxWgDu+cchovCPeVr6XUpQ8fI20Oc0l0bz9l5y8wrt0u8SahbwORwS7NIrhuvwuPbzs40Q7W6rgO6db3jst3cS8AHTIvWMxKDonYTy7qvFZuyY4db9hwMe70YrFPOSdUT6REws89Eocu7mO4DtIS4E7RuQqvRHYvztiaXI9b3o3vdGQEDu8Ufw8CSPbvKnPE7yKjKw8nzf7u6jt1bt2GBc8PJGYPJWi1r0uCYS7M4AcvTp2Yzw2VDc9d5ErPMxNxzz9/bm9wj0sPEHmaDwS7wE8NnklvGAbCD6YyJq7h2wsPfOMLTtVGzi9M+cdPRj0N721zIg8KiwVPuP9CDzQzye+K1UzvIW4rj2T+6u9ynAxPUETnj353QU+wYk+PNFGh7rTYdy7Cs9BPMy/lDw2Ay69lcmhPfinLT4a8489BkQnPEG6GD4L8wy9h6CmPaMSCL3OqIM+cjiXPTuelT09X4691K+lvaATNj16iKa90+Zzu/onbzzvEde8OwycPZxv173QloQ8I2OsPO9IqryNvRA8UQSbvOZKAL6klXC8DnWOO6M/zDxqJgC+nKvuO7uuGr7FDN48ijjkPGBFtDxZRIe9KNkOvn7fkDwz2Js8JzkKviVBkz1DHUi/Ow+dPcEtkD7WsIQ+oMaoPYgaUz3r1/g8anUmvr8ELDz1iPi7h+5DPDoHgT0K10g9C6+XPeHoP756aJU9RN7WO8N8S70SCoq9h0tnvZaR0jzaEkc+fXy1PSMnhjyfIyw+3g1YPOzKBL1q9zu8cpJoPe6IML0tskA+eAhbvmD6tD2KfW09vpUfPsyxPr5y3Iq99YUAPdQsB70oJLg9+b/RO7unJD6msUm+A4HNPfxaNL2JEa893JeSvCVdML4QxcO9zkCLvckybz2rK4K8Y/vVPX8/iDxDk+S90A3APajxH75I11a6L4Dhvfntoz0TLy6+","AWi0PKM22z1CX7O8kz3KPf0GBL1XkUk9wVT4vWOEPTvV+TQ9o6kzvdzwnD23Rng9mJkGPjBimb03Ymo+R/wwPTVpIr7sW1w9JxGQvYPHo7wfFZs8W6vQvKDk77zjShA7J6XNO8Ev2z3bN8O7XXUgPkaVvj35wiC9LtntPlEOlzxbH5I8byCGPLwlGT76l7o95tIvvmBXmLz7QSa9v4SdvcplGj5YEDS9auoevn8y5b0HroU9NPeLPtjWbj1R7828FHOTvbD8fr3+nfQ8oo9RPObUcb0K8Ac+lA5wPO69Dz7hrUY936eJvkecrDxrPsC9WvJWPTP8NL0N0l48EOJiPJdsSrxhv4A8OMoYvSs8Er2LAuu9sPTQOqGkWLujem+851MJvjZ9I73mux68Up3xvrq317zqCh47UYuwur6LHjwGijS7UgVgvCDWZb5oFSo9GBvrvPHlM701zFE76P47PWLTtr4fnp67ynUjvbsd9bwv2Ik7LK8SOy3LgL7zKqi+IQ9mvhitGjwUdym9gYZEPICWKj3NSZw87/7lvCiUCjwdJMG7Jx07u4ZOWr29blW8rEySPL5vA7wThgq8p2MlPDLsHj1AwgA6nxmaPBus+Txv55695uJnvhHIob2j1dw7G3r9PF96g767I7u8r+sMvGQk1z6B69Q6tccyOy+ywDwUcWE9QnygvBvz6T0XlAu+diaOPYIE9rvQci09m0+ZPe6wlr2f8N67absQverSYrwjHBQ+8ZEWPVRtJz1TlmM9ZpFqvRugjb6hJoW8yjI5uxZ0OD0tNOm8BrrAPXgemjzg3Nm83k8SvRCfIj0jCAY9miUAPRL4p72JMGC9QGKDvWt0B70mSMm6AjPPPR4+mbwnX1k8vR45vdYMPz2dux69nn0EvR6OOz7nbko9LcqZPaSz5D0XSES9aCwgPfXXc7u37+Q8GHEmvQL/nb7HVxu+sy2WPPHnXzz5mby9on+9vcWVJrwQSBk9xOFFvcPEPb7wyFS97cj1PDBu7DwECpY9","y3Muvn7+nz4zDn89cq4pPY37l74lxbY9px/svZokpz1zQ8w+3sxmviRDCrzXy/M72ACfvBVp2rpZoX+9NfcivuwHST6Rwig+vQ2lPrIN4z0JL0m+nqGJvfSIOTwo3GS+/Tb1PQoioj3EhqE9YKj3PH8mPDyKHwQ+yc7BPl+Cjr4uIBC/04mDPizjKj4xcY0+KvGPPrfKgbzrLeW9G/oHvpp3wj04miA+KRBcPd3pVj5k0tg8qFqdvXhS1z1Ko7m9awkAvcwPUz08tRO+xyjxvXatg77fCHA+eRfcPR7nEztgMa49izjTvtk5Ij23dpE96G0Dv061V72XfzS9gHSvvQD2ir1HPxU9e7xtPdxVyjyi+ba8rZxiu3FyrT1Hkb49fWvCPbMULL3yCQQ93EOFPRtSgT1EYo09iG1ZO82VDD7JEbK9zklPPfAyij7MwXO6omh9vD3MQr3MCNo7A60kvb/uK75bYZW85MVaPTsps7xuTCu8/k+wPbeGHr3v7vy86pv3vPbIJbwXJ0e7r/Ksvfj0hLsVwz29CIqnOSNVjL08JmE7YdKmO+WZEb6D7bS8yW+LvXv/Ervc48c8wimLvfw9Uz2og0a+b2RBvTEDgryMjtq9mRXAPYSPMb1L9k29kz9rPRWZjL2hgqC8/qmpvSwOzjwWRrK8omeNPG0pmr3e/RS+EV38vUVgnr0hs0E+NnmWPHpEEr512Qi+RpqpPeciWb3rkr490PgRvYwOo73KSj2+vRuRPVUUQrza4ds90afevQU6mroAX4A+3m51Pkhgkj4qCqi97AAVvSr+nb2PgNi9jw8ePiWchL4JIyO+LuQAPZIKdT2N7KA97q3lvm6c5D2l4OM8k51LPhcQMj4zYmg8gxu7vYr+Jz06S4C+MQj7vLgX2D5jsfQ9Dw5jPA8ePT4I6ie+pzhgvrN7Eb4Iapc9mlzqvPTvIjwvSfO8T4FXPjdQ3737XlQ9SV+wvtIQKr6z3Ri+d9h4PjjEhj5y95i+VolaPe6jGr7oZ9O9","3vWvvOpydj04MG69deEHvXbhO7s0tTA8PNFwPb8QVLwplMw82UijugzthDuYbys9vYBDPbkkprwiTmm6fyPMOwYtvDzrnlK66zSBPoWYQ72FBTW8z4AZvFYj4jyJcr+7G34nOUisxDz9m/46I2uevTYai7wHtFY8JWfMPIH0Fj1a9w3AvgxdPCEssrwpix28v4ssPdb5HbqGaIw7cZ96vT1lIj1e2Ik91ZVSPRVlgjuqv4E9PnStPJA2pTsgC2Q8e1EavPJ0tr1RnW89yYWEPZy76b0JUyw9k7OMu55DRzwGAS094MbyO84LQTtb7cI8zxBRu2nTmzxiFSU+PfF4uqH8b74nHDs+/aJ0vtqx4z7QWYk9QyVYPTfkLD4ANCC+uZI5vzV2uz3+hAw+oOtCvWAyGb7W0iu9vXEAvvp+bz7X6y2+JgtjPn7WJT6SQNU+92vjvDIYQL6qf6i9maY3PKaRP75uAKs9vGqvviX3kr2gdKU9ANlyPvGhV7//wQS/4n7SvuKJJr5l6zY8nRsTPmPhzr0Xg1S9ThKavh4jw76owKo9hi1FP0objz11CI+90/qvPsJV+77gJea9QAgyv28qmT5skow9Zc4Mvug7O78HrfY97VDyPHSOMz7DnfS+KbH+PJBo5b00LBc+XT5xPkkwfb8BTnS+4luYvq41JT6pt4Y8lsatvBUqXT2fLmi8RFPrOnUOG7sCyaM9KTGTPOM5hbw4nRy8YQpqPRHRf7zU6aG7awwKOwG1CT36H6M7HhOyPLJWCTvvTKq8dBcVvCA/b70V1JS8k9lkvEud3jsUl5e+hU+ovDsSm7v7cTc9YqjIu9GAqzteHYk6QEzgPH//HL3MmC2+Pw+zumjhsbz18mw9fKW7PSv8GL3F6E09Jk8FPMUPGbwQiRU8NKkKvVAe2Lu58948ol6FO/ro/DuWie85sq4PPdMvZbhlf1C6lrxSvzlPJD7FAFW+HapnO3P0rr2UnA69tgnfPO2TkTzl4Gc73R6Bu7qNAL2N6dS7","cUuIPCjOD7z0R509gglDPC2iRD1adO68NYoSPU5PKj0Kxqk7dFCCO3LhxrwI0a+8bfYavcKxCTugVJ097GT/vEPTEr34iAC8DBHdO0UNij0WL888nIH+vMYy9DyBz247PO8uvYsoI70WFpC84O3gPdCXl7y64sc8ibnMPFXjDz1bK0O8om1jvG5v1jy20qA8oLcpPFt0lzyooMQ4WX4KPcMeCD1O/YK87hN2PZSmzDrOhJQ8I282PRpjjD3HVi29it+PPdtalr3zb7W9szXvvVvAtbx7zSK8iJUrvUbSkzw5Ibw96UMEPQCsNz2OCoi9PzPGO5R+arw26fu8UyfdvOwZnj3GU0i+7mOEPEJ4rr0ykcO97RZru2gG5D0WieE8tGtpvWtoAT6Ydza8DWR6PR6lcDs4jS49VZGgPaI7gDxerw88Q/ZXPlP1xbyi77c9td7iPGSQpT248Ac+eVNRPLVPED7+vUI+4UiVPX8RlT3WeQI+8ACdPpW0hT013/m8Q10OPSxxuz1R22c8Q0U8vZqgcD5X7v094XkcPncTNbwB1fS7pObVvhVEH76bNAG85HrqvfuMYj16FKS9sMxdPheyaT5ldxe9nowjPhV4F73RqmE9hgIEPoW/yD1IPrE8lVMUvlz1tT7PeZa8/O+YOz8Uiz30LNE9qUJoPYU8/rwuobS5UnXOvJ8doLz5yaS8HgG6PUCHzL7K62k8rU0iPBdh6jz5cDS9SVz+vOd/jD0Kyco70fH9OSkuyLvf3TQ8ME/QvESsVT1wRUe9EituvXgZCjsmS8I8QFK8PC2nBr6GksG9FzJuvhbmiTwkpmK8kvg0vUlv3j2deay9KOJcPAjDiD2Ige88/KJ3vbhC2TtT2r08D+aovBNFELxafBC9WWvPvPphl7wO/Ta9arneO9MZlbuc+xU7aAftvPiFtbxVOOC83YmBvEsZ5LxPu5y9Bg/VO928CjyJvbc7jepgPE/DsjxArQo+OH2/un6fe7zThhW+NkSxu87AgjwGHV69","+E8nvU4smb7lz069aOS5vn1BcL19FQu+nAHNvdfEmD3tvVA8vgItvasOE7648uq9Lsg7PROliLyYLEu8R1Y8PdHuqr1k4529FBw0vnu2gj2tjz08m/POvHk+lD0HJM69ny8bPXZ2DL2GiFQ9iZ8UPfVYCL6YwqI92cU/Pnk7472Hu6Y98HqKvK5tnbxejoI9k5MnvQYY+rxbfSU9VaoRPYePkz0Kohm+qqiBvCh3LD1/pCQ+VvfKvSXpGb0eiEK9InbIu7pyNz6Na4C9umMivgL1xDvU6U0+4Uz/vMBs8D0Oc5G82afiPQVmtb2ZLUC9PhBtvfDKr72nGDg9BsgxvXoWAr6ceH+9x7uFvS11mT17rmq+le0KvQwwOL8S06i70Z/GubPamD1jhxO9D6MIvZaRfL2RWVI7cu6ZPv1mD73FJnW+nMCTvDoaHD3vR9k9GMnWPAmZODw61CI9/3WCPUBIYj2qrnU8WZ/7PMzUvT2x5oW9E8MiwFn0uT5pRoM9/x3ePCN2rL30gLc9FYHrPYQ3Zr3XwcI97YUYPZ0WDDtCqci8qkNXvkZ6mT2GwVU7bAXBu4xueb1xzCu8ltVevAghSz0SxiI9T1MSvdm2Dj0AqnC9tV9MvBi9vzu5Xok8QflcvJAz6b2ab1g8VW4GPchPdj3gy3Y7DMztvLwvxjuG/Ja8FZhVPLY2Aj1dorC99/JHvVRU3bwQqXk8b8wWPR3W0b1nAC+9ttIkPPN6Ejzn3yK9n69JPYkM0j130LO9bVK/PJukgz2CWi8+TvFxPHTaET5cZIk89rxnPKpulLwFYEg9uLiXPOQrt7yrhZm9d7qDvEsdjr08I0w+DUZVvQqiUTx5dKi9WPqGvd0FLrznOZu8en0KvfuUID2RKoq9rKDoufOyCjyCxxs4JfTKPbOyFz67JXu9sjSePLGnfrmTFBC9uhymvo7BwTxBVuU7lkECPNV/7L2jPYs9MHbNvaQQHz2nSbi9C9HOvCQ7Zjy5hKe9o3QnvJKNQju3P4q8","bDV/vCghUb5iqaw8HmjbvRY95r0GZyw9BdkQvs7OjbzQosY9I4SFPZtEO72ay5M9pqkqPRYsCj3pvoE9olQvvXEnFD6h9wg+bue3vTpmdzyihgY9/7LVvSZQ/70iWgq+mN8dPXj+ZTzQOC6983QcPMbJw703uI48FdO+vtbalbxI90U+t3m/vJInCj0t5zk9ANEVPCO1xbw1XLW8a9QxPGE5Mj1TT8A93RTbvYAjvr1QMBG+wpglPTFMJD0GNXC86FEKPayP0z3W0Rc9HWBUPU1rBL5Tqxw+JX0cPSQ8Mr1/7p88xwiBvty5Ib02RjM7KwjQPODDv7wgYhQ8o21pO86im7uAKq481efLuzkpoL9kbGq9nzNYO4RlCDv4/gQ7DTs7PMSWOr1HAUW/j8zMPPM+Sr347zM8TDsVvr/t9jp0d9C9GZGBu1Btujz9Nu66q1dCPBK/sb3hwyU7acLnvSJpBrt2GTg5lByEu7cmbDqHQ0w7WU2pO2zvnL5YvkW9f02NOyFnQrtQSN47j/O8O9xQizv7uVK6CCWFOwOKQrtyb5g7EfctPCWbpzwcpHK86iibu+06lrmkrM871cunO+dbAzu82RA7PoyqOmbDQDv17YW7JLlFu6d/jju5e187k+DKugsrezxpPym6DxdTvKPTNb/Enbg7sLvjO9cm5LpfcTU8j/ugvPss0DsJOew6kSp8PcQB3DymBM88zSeTPMvi/LsF4rO8+1s7PVKi1LyoczU8jCbzPX+WUzwiYmA8D6CGvTJvLb6yE449iTIGPNDgv7wmaLq6iLvLvZWxhjyG7zY87X0WvNw7mLv/D0q7htXxPbIOvj23L8g9oJPfO0mcF7uJBua8qbj5PFaLbT1+qyU8bmk3vmHEGruLRMM8BK6gPfLe3bzxffc9JKgXPJ70iLqey6M81W8MPcpds73mX9a9DXmpPECVmjz6Toq9q1iGPbOpeDwJQNY8umAQveTpBDxho/K9BH6svTLb4L3O2EI95GYBPbsubr1IiXA8","/O+cvcqXKjybA7A9jM0ovGVZ0TtDzlG9ddGpvYn7kjq/bzs8SsLAvKIhUbwEpV47fLivPfzWAb1BpzK8hXNUPRA8ubyDb5u8Es8nPIHfsL286/I8vfudu5yBCr4SJMW8D/VivGwHRzzYZea6EY0NPN5gAD6NS4g8aHgZvOiJC72i2Jq7doF4O/aUkzxDJU+9LYfRvKmC/T3MQNa8iiUFPaig473swwU9AQbzO1oN0b4j1Qm4rOvOvTDOTbyQ/5U9DqXJvUI2Br2j0Gs971BpvHRb8DzKcm08wbFBPO3T7T0mxYy8UpapvHYlOb1up+887Hy0OtM+IrwwJu49hrjcPOHvQ70J6Ii+QDmFPJaNVj3oXIS9UdRgPL1KTj0Zr6o900MePEIaST2quQG9kEn9vAnBJr1sTBa8agv8PKTuZr1jSCO9b/FlvfLXzT3imww9BDOhPBJxpL0m0xs8I+++PK84WruTgHI9ld+AO93apLyhnFY6UyCavX2RDj7nWp+9myDTvL7tCjx21/o7cwmvvINaiDxHnZo981OEvZBKxTxcpYI9v44JvvAHzrzxgBC9qRIRPfKxwTwT0r+7LcHkO86BjD3FG02+09BiPLmQZL2K58m9usttvbl35b0Ba+i96h6YPJFm6r0ycwy9QAoevsEQwbxp8Fq914cJvgQ8mrxmmi89iRJevXWFhj13n7a8sBmdPXI21rwBVz4+PT86vAznyrz1Aow8vUvoPUr90roPST+8w1mlvCl6Kz4sUb89GyRPPZ6SYD2QhcA9qJ9VvdZtrTytFCA+Vkebu1Wi7LzqqCg8wolIPThumD3sL5491K5+PfOBvj3V6C8+7p8hPWf6L73jiUM8T+2SPuiI+bylk509sewWuv2Yg70Wdfe80LXMPAdvSj0OzhM+4XcFvUMRNb6unxk8SiiGPQhNsLxL+Zo7cXtEvCYrRz3iDL89o5JYvLnbDTwp3eo8fTUEvcQW3bz/8TA+uSmDPfBdjT22FQy9W8C7vDcd0Dx7Vwg9","Iou+Pb8OiD2mQuO8sncyPl5Xlj3VJZk8zCKkPYVrcDzwqCK8AOJWvXFQFj3C0Ti+mC8dPdxX0LspFzK7I80pPitmAj70SWu9u+M1PXpDpz3WVX6+ra2+PLBJhrw84W89E1JuPDdODD12WI28hzyIPEdD47wQ3zG8P7WYPYmKjjzNjBU96ydjO1Ubib1Rq6k7cAILPhicJz0M/wk9AmcNvTogZD3yxio9UHBwu/cpYb755XY7AOBhPfJpmD0WOgU7Mb8NvfOr4rus35E9dZM5vRTN9zx3/Jy9OV5YvSEfZT3gjRi9Q7pQPOEsiT35HdW9W0JFvEUI6bsngve6SQeXvUyRlT25A9W9yGEpvIILlzxBn4o7bpQVPgsFlb0wgnO9tSeBvNCfoLuFoGy9mEMiPZvl9L37Hbu8z+rxOx/oXr0TpuM9A0uCvZ+A0L2xZ1c9nfdBPReYtb2KNQO9rHKpPRIc/LvRNuu9P1RFvCAMj7wtAF+9vCMIPHcnnLymNII9Cgxkve4nib1ohzG+NUr4vR1Y+LzwTzw8Z8cHvmNVFL1ULBY8gZFMvDYblDyyF+68ssJuPfp+vD2DwLo71l24PVts6zx/Ey68Lo5Ku8VONz1qBky870dOPfQfRL6iimi8NXJ+OnEfLD2wTeM88eTSvTCVwTwKxLy8miTaO3xtEb2HXJY9e+SWvfAdgTun2w4+PgNOvPCj2Lyv+xi94ieSvUl06zxeIeo7FSYevaePRj323TO9OCebvU5VejxorgC9G3Q2vX9bLT0hWi09c9Dmvf7PDj2xchY8TqikvF2NKT3LstK7DdYrvfQbsjx7V1880tsiOoj8mzzfYRI9JGkoOmOBtbhdckQ9zmc3Pcfy+7yMsvS8SybsvJkNeLvZwZi+oshjvl884zstsOM8wUKkvLPKyjwS2Hg++XNzPVJWPb2aNUI7E8RvvlciqTyavZ28Zx2RPTDjCzxsspI9HUqEvcxHGb5iB4u8GLCaOzcDWz7/DxO9UVZ8PQS5JbvheCw9","uDdoPAYOlrrFQK08/uuUvYg7iDw3vUG+eZAgvV+Twb1m69M9uAy8OljcZD1nn6A9Nfasvd+5FL0QU9Y9hC/lPXONHb7TGAQ+Y0bDvPPINj02IQE9SBejvYb1Tzw0h469MbVEvelNPzwY57G9eubsPEPogD3yF0u8DNeyPsuqS7xp6HS9JzEwPfrnvD3vHjs819FDvOCFTbzv+k09Xn0Yv210qr3AqQs+1GufPQ2KkjwsUNO984sUvsnGwDx96De8FzujPRiBm7vf/Bc9c1NAvhOOkD3wwWw9T5t3PcdUtr7ANLm9EQVNPal2kjwASGE+88mTvbp2Cz5on3O9j9B2vfcsOzu8TjK713JyOUX45LqVtKa+o7NAOgemAz7/kcC81YF8PLqxg70lIle8rsujvY92pTwxjG06resyu3RsejvnwXA9vD9qPRzsGL4FYcW8LQ+IucU5wbzzfR08z4SevF9rnD0t0HG9K6fRvC+SEzyCpNY7zxK3Phj3yb03xk2+s/qkvOkcYzvFWy68RnYmvIFmZzyEWYg8BvuHO8BkNbwRj/m7Pwizvcfm87zdtl48SJaCPdAWX7xMMQW+vMt8vPismLtJ2zk7uBeRvBo7LDyvVuS7YO02ut2BxLvKfVS8tX5gO/tYUTv6JYA92OmQut5NM752BZk8TIgtvL+Pjr5rDsi8twogPTRskr2k+jY8OlqoPH/vo7yc+fy8z78bvf31S71kbfm8ABuCPL7NvjywB5E9wLVHvUt2pDz0MYu8Vj+nPEEQoL0W0SK8nTjBvNZgFz2Xv9O8/XuIvC7oMr3ctWm8yw0qPdcpTL2YBJA9YR8SPdqhCjzKjVS+Z/YJPLdRobzca9C7tKbMPUSDfb3A5Lu9QJ5VvaIbFr2bVrO8NqUNvYs3Fz0jeR08JeVVPe8jzb3EnBG8Oy8rvOzKG71K38M91eV0vXE8GL1XBaE9VnmEPSnlnzsr9uG8GEhquz+pkTtj9As+MCYWO8AbgD3LUBK9Z3K2O7j3BD3eVQ28","3gU2vRLUgLyomB89GR0NvXtMxDwpsEu9sYIuPmzWcj0S7Ec7+MqWvcfECb4HnYE9HZ7BvHqovD24hTo9N8yDPcYamz1ywne9l4clvC6DpLyXiq26XnY6PN00jL10Cu+9dnu/vYDCLjyVMAO8NSJ2PcR+TLuGVl+7xBkUvfnRXbxUUTK9OJ6bPTMEgT3iSu092OWUPS1MUb0VjYK9I4egPWI8kz0p+rg9nV/TO5Dmer1dmOm8JRiLvVLbeLx2H4+97L7zO/SLAr6BGEi9G1wCPSARcz1t1P08ATEjvnQnp711rSU+SJgFvRb82Dx0IVQ9ff0QuwE2Q73L4Jc7cPy1PFKKhDwuE4M9JCWXPEcvaj3E2zS9voGgvAWaXT2uK/E6xp1XvSX/uL3qXe87PXRRPr1LAr1tDdm7ThrfPClzZj1tiC4+NDuUvNMqBT28Q8y7G1QLPfCTDr0miya8nar9vfFFqryYgAs9vh+jPaFdOb0PnZE9BhnkPWHfyL3z1Fk9k6fIPU1KVj3jpZm9eTEivago2T3xZDa7kLmTvbJhS7zQJYw9IKNjPM18Sr62Ejs96iC9PRglKz2yfY05iIGrvDTYHT2ERRw9xpeTvDuCJrvBuom9VBcZvaSqqL3JqwY8Gn3lPF0ua76ZBIY9xTVzvftxXD1yNBW8JhvYu76Cy73ZKJa9nIgbPPgGMz5RpnS9ZobNvNg1or0VVJI9okV9PUkY3D3EwZU92C2xPdd9b70+XZq9ijoNvExVHD47TOU8gAcLPMPJcDySe409eIoJvl+n2DybPTm9lcMrvILNw71Wmfm9KG0PvRNcyrx8rS48szBAvjZQ3zj7biO+b968PaRD6ru4ACg+qjeCvN9E8zzXRgs7dexlvk0tCL569DA7S9DVvHIJFT7xh1a+wAb1Peywhr2nSq88NVqLPSPV5L3CwVI9f6QqvnELVb70eRo+CdwovvA99rwuMEq+/Lf5vL9Ggj0QLMq9LKzLPVWszjznYAC91Q95PFAotT3GWTC9","m8MkPYoIIr61xiM9kzqcPH83rrwT5U29wj5GvpupPr0gRoY8ugzxvE06GD6VRGk9b9HlvZ0HUr1XJli84orCvV5G6z3oPAI7ZcNjPmXzaD1/tiE9bsVcPChZZL229V+9l5KCPPPDeT3Yxy+82g0CPbSDsjzWRSu9j35ZvYuM6r1XIQ2+guQxvXqbNj726mS9lCq0vaelFbz3YTK99p0UPTPcoT0Dheu7um+EPTd83bxSO2s8T0piPr3DJDsZ+HA8feKhPT2mFL3TvX29ASlCvohiRTyOBoU827kAvh0dZz1GtwE9JKYUPgc3tDwDZRs8Gs6zvUPXiL0TlRG+x6uwPM4IEj3BIYc9WPXSPWCYkT2ojhM9bRUzvZGhDj10rJI86DqaPcGcfzywufw9QYU4vSBq/b1KyyI+ASCLvq4g9TwRneE94mv/PEw5uL23FjU9UBpdvfCnoLygj5a78aZSvWHccD4VC7w9LAPxPTU6V7w05+a9LRbkPMw3wTzchxE+0oXivXhpuruSmck8EI5wOi355Dvjle29U0eBvWcl0D3yAou9phs1PaoxZz2eqbO966cBvnswoD6wxYE9XJ6MPDQKwzymX7K9S0WSPAclrD0UJ4e+89q5vAUM0jzBZoc82WRwu9m53DyfiAE9HgHhvC0N3T2n2Sa9ASwEvU0y4j2b+PE8d670PD9yaTqNpdW8xdTBO5enCDxlg3E9ZV92PW1ghjwHcMk6rETuPJtxzTwSzB+97f9gPIegPbvK/l28BCz5PIvv9r03qiM8sEijPd4EDT25oL07P6OdvZuRn7zNDSW9JkmvPCVTfzxj/5E98hUGPrK0AT0gTCw7JmwIvVS2FD0z9pw9fqu6u8lGIj5VZ2E9EF6/vRDe4rwurg88GghdPTU6C76IJ6Q9kIRDPWGgmL0xTzC+QLu/vGHik71pMVO9fMbau7LkZ72+1MS7VvsCPoOIwbunyCo7ZREUPapOhDsNhSE+I9MrPdQbgL2GtWK9XSqZvX40gb2Lkxe9","vpyHvAFpvL1U8UE8f7MMOyW3Ob0JIvq7NArKPcjltT2ZF9u9BuyKPWfrXz0D+gk++4zwPYkv5r1z+Pg9J84AvrZ2dL4ubCK90fCZPMEX9Dt8UtE90HjAvBiFqzzc2iO+nr3jPaLcbr20T728h7uWPZQ2Sj39JWC9FeWCvMDWhDublhc7MtaJPQDwqD1uvQA9jC1evSdh/Lzt9/69+0UpvdJe0D0YGxW+19K6vECrEb74Xpy7Ui+vvQ9gAr6YgBe8gZkFPVl6xz1g5du9fWPyvZfqrTyCM3891QAJvhykYL1NEoU9+cftPTW+UTwWScW9s7uIvYCMETsIloe8wuWxvWmUjLuhJ7O8Iwuvu9RFrL20KxK9zGatvPlEi7z7bqM8Nh4mPBx4jDv8g9y88e1BvXbmaj04nE29BMQhvYdGgjrn+++9t0Uzvchttr0WJ+S8chqgvVrYuLsSis48nA95PYVRDj5njYC7DTunPKoCdzw8Wxe9AXzJu4zGHrzgjdO7bzOsvKPMKD1ARMY95oS/PSyk0jweNHK9NvZTvXSLI7wjZUi8Lkb3O2UvL71eSqA93Z1CvICqrTxGO7O9A/IVvRuCZbx8hr48xs7Fu6fnj7zRcwW9dWeZPZX70b098py9JahDvf+Nq7uc/Li8loEwPVTUmbk1k0O8aNkyvSJx0TwWR1e9iaACO+zogj3aj7g8eP/pvOacKT22ny+9Ka3NPIYxMTxChog9Bnu3vVU6lrxFkKs9ujjLvP00Kj4uAwq9yWMQPmPYIL1FMIQ8jOd/PTBT4zsxuRk7K/KJPNtiKzvb2xo9oj/EvHnaS722myC9n+gUvZtwHL7qOOk9NHsuPTjmSz01WOg8STUvvQ9rpz0fB6C8GXJSt79lk73o2H48YjOOPJKlv73QjqK+anntPUKfXz3O8uW9Pk2ePHyIk75jxh08UnQXPYrcmr0BL9i9K04DviYHkT1+D5k95yp0PXm2jzwAfLK89hASOnerCz4DZLQ9l3VWvmAXJD6EJLS9","c72KvGhwBb8DQW4+h5gZv7QtUT7enYs99d2QPtOMjj78Vik+tbq7PYJsEr60vVW9//eYPSeb/T3SNRs+2VJ3Pt11OD41Myc/t/8sPrTGnr6WeSU9u1khvdXAyz6LtyW9rck/Povjfj4tkCM+UeifOXhDIj3/jAw/OFUVv2IQjL0A7QI9F8i7PgZLIb4RSES+1uTIPm28lD6DL7M+SLB/voR8Ir5ZPSC/8G29vbMcHT7oquG8xc4Lvm/BoT6WsuE+nYW+PUqNGb4cGso+3uFDPb0ZYb41t2E+QyeqPh/mgD7HRN29+BghP8JSQT42SGQ+UmCCPqYAL72z99w+cngsvm2wJj2hhWm77B/lvZ8dJ76xn4K8tiCgOkSXED2J6JS6EcV/uhDdFj1FCLI843ZqPCv/+ryOVsC7kx+pvCE4a72TxXw8owgEPQLv+zzftf+9bpwvu+Xenju6O4e+Wz+jPGxVVT33+iu8C1ILvB0hbztD4jy+v4iNuhgs1Lzjb4M7jIesvBh53rxbxrm975JWPMsgj7zOQ6y9yY0IPHoTGL5lFA++wYPBOQA0n71dvRc9Dl6HPYkNQb3AOxM6ahCDvmCsYb2oTie9S2pfvdVfWby6Dqm8SDsYvL1LuDx+3mM93bMaPYfCkj3/dE48rAHrOy/HaDzNLfC85jzGvQfUTzxd4ye9XmMSPYXBYjwnu0y+m2zeu/Mz5zsc69O9Ew0HPSC0G72DE9S9OLq8vY6lrL3AVYq9G0qvvbtBoztylPA8tFldPULjHD3RSVW8ty3PvfoAAT0urYC8exKtPVgDIb0rKDA8mnpFvs7Hgby0+SI+euWOPKKoD76UHg2+RJunuGCx970lUPM8Gjp9PbrPwryYLSo+X0tCPXAmv731paa9iUX8O9fGxj0G5Y490X/BvfBRSb0IP5I9ewYkvXv92b2QTNs8sURrvQ0wk76QIS0946CBvR121z3nWpA9kozBuxVnRz0FOYU9wGgRPU007TrNKgm9/f/fvJmfM70xZic8","E6movcgw5jyaFZ69MFFIPvTpoD23A788LyrbvmaHxTyNUKy9JSiFveaCSz2Fc3e+PhSUvWlRaT1/p5I8mtGPPNU+2L2lPg09hEwVvX5ZCj2F0qu7WE7LvVPccL3WATs9jtwLvQpFQz0O8sO9AUaIvGYbbT1+mpa9xdEAvuGmmjxEsik8I447Pd3TCT40Hd49Su8kvlcMgb01sww9bExfulxkEz2vBp08eUXtPcAaLT3mrJM8M2Hyve0czr0ZNUK8oXmNPDvzyrxtSLE7WR8cvWcirj1UvJy++fvpPLhYHr3iqgi9dGg6v8gn7DzI32A9QYqevT/9jjth3zm9o6+nPKDTXLzH4H89XP0qvdzmjLvXIG+70EA4vp4Jaz2f4Em8x14oPf1aOb32WmI80Vw+PDZ//Tq7NiI6lbC+PG44UTyeOaM8jWm9vIXiDb7rHyk9YdHRu6hEC72nMoa61W0fPC6H6Tt1Ua88bAm+Oz4n7TuWKwC9q40XPMcPlzzGYyQ9SF2Yvp4OjDweduk8HdYLva/8JD0gAbY7F2crPaZOPryI/lC8+95hvhGtMz2YKp+8xA8gv3FxhTz3uB08ST8bu85U0rslNg27a2AVu5APrrzw9+g7D5STPH32Cb2Nru87+msGvd+E1DwhMaQ7Px2nPEOkGzzUuNg5ybgWvCSgf7yK9AY9ulCzO5Zpw735tY+7VHNWOyCTc7vHJGg9d7YQuyogNbvcW489eiqyukc/OLtLPsE6hMe2vnJpB7sqcRE855zCvAgFlTuNuas8Z9CSvs8/jDxxoCQ7Cz5wvm/HHruMUqS8j53Wu/WA0DzQJFo7nf2GO3HhVjzfdZM9K3m8OU0sxbvhO8g8VacaPTQYhLwHTic9WEzDvgDBCDzHbMe+YdGBvg3O/ru9XBG9/2UPvbqJv7zw6ce+Quc9vJ8kAL4f0SK+1PY5vpAwwr5bIC28NN6NO+D/qToxWQw8fpWTPmm9aT2c+Kq80j+2OFB9kDweCNc8Bj2OvnAmm74aNMU8","W0M5PTwAUr5yB8u7Z8pXuveTI7wFMxY7G8NnvH1LCzwcmuS8oacsvAeN4bruqCa8qPHJPJ6fVjxinqs7ZIXSvVG/tLwSWSU8SlmKu7PZFb6pVYs8stwzuvrQhjy5ZYW7BkbxvPQ96TxE1JE6H56uO3SKnzxHxW07rZ+du3yUAjx3+dC7Cji+O5qwzjoq5ca+XiY6vY1Pizx27ZW8rVGivjHbbjt5Fxu+Wu1gPL0Tl7wuL1c8dpA9O1iY3TyvuF28IwjavTcIcj0gKWY8AoZKO757hT3/O5O8mO6rvLoInLwStJq7A3MoOtkPBL+WIPW6bip8uy15Vjt4vB49BAiJPFIW+rrei4U+MqMrPNUmVLmI/Q8+WLQdvZfveT7CdMy+zeO2usmOEbxxnDi77fp0PU9XDDydMYI6wI/sPGLCSztrnES5wMUnvRh+Lz2X6Yg8a6ppulbWBD10zEo7gR6zO/r/Dz1WZqi+syWlPViLibxrL5M8BPPCPXzPIT5s0yQ+crDYu3a4nTt1tws9M1IhvIh9+rucfKi88gldPcbB7rsRBOI7B8+pvj5dlzxQECM6rYFgPLWUGLoq/P88OOiJPEhvrTuB5e67FUs1vFyFjDwUaS28AQAuPQ7zLLtoums6bc4Bu2dQmT25+dI8+i1FPDA+FD6oey291Un+O9ioZb669Tg9/N+yuJT/cT0u/os9qRQhvHtSzr1FaVC9rQQMPBAaUr4BD8e8WAS3vWdIML6th7c8FpHQO5iZF72771e9Ang5vhkpaTwfMAU+eySFPTAJwz1RRIW9ItO5u6Twvb13UJm8yeUvvn5EvD3M19I81RebvY0kOr1oDbO7p5CnveGR8T32x+S8vph/vbpYaD36CNC7QIQyvcoAabtaZsq92KOSOIY9Wj2sIkG9+fIFPdlSRDwywQa9XsU+PqZ46b3lkxk+0XGBvQa6kDtyn688BC86vgScyL1SR1y9fDaAva+qTT3lbgK+he09PuAHpL0/eAo9CRedvAhoh73oOe87","YzOLvUXIlL1CLxs9z6xhvM88Oj31rCo9MMy9PUdLCz0Rkx89qU/EvU8RGL1xiio9JDKDvXhrEj4fVcK82HkgPi+dl72m+zG9bi/APYg7ZT4BCrQ8yg6YPBLw5Tz+ltW74GV7vrZpfr1xrAo+8XbIvTHBTr63n689JaiWPNvLi70DJeO9Wv3fPcx1+zwZLou97nS/PeUHvju1ERw+ndkJvXjhSj3EwWQ8m61tveKTgL2zjdC93DkkPTL4hzwL5Mq9/3svPJcBDDxzpRS+QWqfvbPiRT3WyrM8mcnnPA4IEDwgGBW84GFkvXXp/T0gdZa80zEQvX7KCD0NOkq9eDKQvfGl/z2QKTq+xhISvduOnr1b4ck8bt0TO97xYb2ls48+GoPouyP9TL7DYVE9cMvEvWfxDD9jzo0+c+JWvpl6BD45iak+qiVOPtVIsj2Hli+/SRHTvfPvGz3ePMU9BQTgvVkrwz2k7lm+tEX7PMaJR77liTK+davtvfcyIr1x+tK9OYoBv7jp6T5AusA+q1ASPsskLj0r6Iq96GnavRqOuT1QlES+P8S5vdxF3j1keoo+MDotu2H1E77dHzM+51OuPYT7Cb4we2c+o9xLvqsYsj3H+Ly++vKGPeC8H71eD9u8OugePGzELb5XPhO+b752vZUXvz3ujqc9fX91PYzQn72lWOi+RATrPG+WbTyDv/c6rmlkPBGRy7t9r969oZ8yPdpbbjyME8i7WDWzOxQ937ygAPY8KyQJvvyuRjt9yso7STiTPAf/nrxEIHe8hICVPpxJwzwJsX878qiAvqRPCD37XK29BmQsPVkqdTwxd3y7as3Dve+gYz0Qhgq9zj9ivD/dtrgGKu+8LWKxPW80BD3lIyC+EABdvvEcNbshoCg9sHuXPhGEZDzlwhC9NXWBvCYFmTvGjsC+JH2Puuxugj1Rqaa6OUyHvoEcQD2PjIS89F2su5nlC70pUee7bBRkPMHTYD2uMJi9D8sBvX4MRz2YQ9G8jlQ5PD53tz0B8aK8","B5g4PAtIBj4uwoG8R0i9uqsVKb1R5fA8cf/ivA7JRT2PEL09s+givJi0p73gYBU9I9X9PIytDT3/H2k+vu5DPFgFaL2c6t094kyOPuw/Zj1E0Km7hyznunWt+zyoA9e8n7MmutBG/L1lYwY9gNYGPrf4ET3BFY08h8PYvFhAGr422YO8JHqcPQX9ij2M3TQ8uy/cvII3Cr7SF8u9id4nvXNZGD2qHwc9JHT9vSv9/Lxa+7K9Wag2uz/brz1wMpw7j6yPO6TERjy3h2W9hLI5PVor170laio+C1wevaMB5bvUbeS7HU3jvALiej3CuKc9/9BfvIIXCr1antg86z46va6w1TxxTRi+Q7osPMV7zj1BK8M9aFesux6GLr/4GqC9KQn7PACwhDritwK9kyIQPl1RMT2gQ1O9BnkHvon9A768Ea29QJGAvWJb177R6cu8UQIAvZ70nb0pES68DUsmvNFR0r6Tq2u5kIoJPbH9qb3mR5C8UboLvgO/UT5E1BA6fSLuvRNxN72E5867O2MLvY5QJ76kjzy9U7OYvSi+U73YaxI8e1aKOmp8rT1y8jw+syBaPVKsqz3wEHc7VRClPejrnr3SHSg+y07CPDQ80j4CN3y9ymaFvOoZf71SN0Y8aF/cva/vn76UVEM9kQQYvsLsHj5KzV49QWunPTI+fjyfwJ+7spyHvWyxKT3hA0K9LOv8PGNnAD6IVg8+jUUiPYwh+jvIWP65lhEGPq8lZj1+7bk9a4MHPSfSzz0mTim9hoPYPbSFiz4C42w9K3Y+vqfKM73Otje9c3YtPlo5cr1GX8A9zsMWPcLzij3OTds950yHPJ6LI73Jau87hfaLPbGzaj28noK95Nouvjb4sj27K2Q+BaY/PrR4Ej7+7gW+mbBwvfvltb4L8gq+Ui6avYVfET2EWG68/vaRPWaJ4btYbZq7IL2cvPkCgT072QG+02QJvkOU2j2LAs88LVaKPRK5izwRzUI+qDwMvEeGzj0E1uk6pXqRvOYpFb6qYdC9","0B9ku9u83jyo9aW8iTViPG1Nr771plS58qEKPExDvTtCbao8jI2FvvFjZ7w4aDG8grMRvVFrXbt7d7m8EI2bO6LTNL27M+I7DGDQvs+mMDrB0iC8O4v6vXsfEzzhrfm+PK/BvVoEJr0bZNG+W2GVvOzDszp1AXE8SYUkvpChoT3B9GQ9UdjPu5CXGr35lfK7ims5PSxPIT0qlUa7JcSTPA3GGbr/9wq8WzxQu99FEDt68HI6I7FPO0WA/bt9l5E8gJWLO14QtTys7xM9zQTkvJjZvTsOuAu/ZDTEOrfGWLr8GwO8hTIyPuNKwTvERWW6rQZmPiFXwzq6Y0Q8kb1GuxxzarxjQNU9TFOYPHxZHjzUX/+8Ep2JvC67tT3D6vy8PF7XO98ChL26qpw8MGdmPEPwYjxOswm8mjeAOzTDJry8U1Q8JH38vqV6kbu9U4s7ygwbPLlrmry7CbQ5qmtZvPc7Dz3QmXq8SHsKvM8o2T1ddSk9omjGPCIszDx3oc26e/2ku250XzynIYw9J6GMOefmvDznOky8rsfNvZ9RY7xJmlE8AOxJv+QrRzz4BSq9uTcFuwVqLj3Pkzi+jAiqvG/0xDyMmWw7QGrePMvRd729Nls9zqSpOnNNAby0k1M9WyUxPPOOgT06UfY8YE0pPPlxbryMrBG/WFNrvRDErL0e5Q68STmmvu5UhLwZdNc6SIIsvMgEcDzBjg++e10PvRtAkjxpy+i8bK9qO8EFoj0dEn86JnaZu2ArZjvkqWo7cd/Qu4YOYL2t91W8VnPNPDUc5jt4Uk+8EbSVPNV4zbtTLaO9IaQ9vlhpnr1tNuW8ZOKNvLu+Iz1v7549ILaMPG9furzZUc07LCi3PAXIObx59a88s3O4vHBNPTzzeRi8G9QBOvrL37yogZo8CGmYvJ6dEz3cXlY8RHmPvgHfDLxGSbK8Jkszuwxl3jzb4zk9S+J6vBB3A7s1/8O7u96uO+2srbxUrSW9G9HmvSorh7w8sXi8NqyOO+qtiTzt4NO+","ZgWSvdgf+DybJmw9J4OkvdfMVr2dfN87wJ7mvWXBTj1SNL88YMtMvcMBXb2LLTi9yRVLPaPl073IvqA8xvVJPcI5EL6Nwr+9TnVePT9+Jr2mfrA6LWBqu3+qvj24dG28Fc6zvRSRKj1RaqK8kWxdPrYprry6Ko49U6PbvdexEDxRt3k9vvSNPGUiSz3G4y26oYs4Pc1UtLwPEj28N3+lvQ+P6L265WO9cKBrvYne0jziXSe9ppuUvb1ebzz4SWK9yBYaPZVkvTwrrWy+HN6YvNZTG71sxZK8XoMOPVHPzrspeiE98RQsvn/Hdz3zmgo9vr53vIRuCLtSH289Rhfkva6OnT3nwl29Q/CsPQ+3M77xqXi+PPeVvYvTl72c9OK7a6O+PTRAFT5Hyti6LIFgvvhqID1G2n88nW3bvdAwCD7049m+62AePpGJ1zwpYOk6igEjPS1DnL3SiKU9iAC1vZETKD42vQo/Oc3GPdPLmz2c7hU+dGiSPWSnJL8tCgo+RojIvv39/z2uvAQ+0S+3PVMKeD4rjVg+uLYaPi8h+L003C29jHbCPYKJqD7vRJk+AAK1PbAnDD4zoQG+BW+mva1HEL4MUp69pU9fPnVNXj7k+Ba934dsvr3eAT0pknc9FaEOvXdqhb27mli+HS+tPWmPB76zRN+7lz72PbRjpD5jqpE9BEcyPWJo3LzfzO8+VJWUO4ULBD0QOJo+fF6iPPbLUj3HAMW8wZLZPRC0aL2pgOG9ZbVduziCf74qdQO8yRluPb6plryttDY+A21QvUZPwbzr+6a7P2y7vem4m7u5HSi9/uKlPRdqaz2tcjg9TRNKvaMSQzuo9IU+mF3APDcYlj3S9LM9DolYPjJ007yaHye8ToK5vJI6Cr0jBsM8FlSdPCKUkD1yMKc+HU8EPiWOcL4PmvQ976SLvJqULT3vNzW8HwOKvWE9LT00PZO8w6iVvdiD7r1t7Sy8yVi+PZT8nb26/T694MFBve6oxj27BzM+2iBrPTnq+LxwjEI+","WyUYvb9YMDutrt096HpSvbfjL71+kT69JqAJPVRhYj19KQy8YXubPFmHMzu12xU9ZLs6vYO4qjwRKLi6BLnVveTUmTzuyTO9TmOHPfHM+L1lgSU9UtcdvW9r3L2RRHo8YCmevZLOdb2mox+9o43fvdQPgD1s5lg8Q55LveIVp7zQK368gLuSvs6ZsD22Vhe9qGwAvbW1q70PB/Q6koHmPu3Ieb27//A8yYnlPIPrYL2nmzK+tzFmPMOjsjs3qeo8Qq6GvE7Ri71TzjW+fIfnPMPA5r5CD1y99OFcvS3hMj0tS5c9J3BIPaInwr3ymW69cyVGvO+7+brytYM7/TS4uxHkuLx7dd68CzRVO98WfLwou0c8ClT6PSySCL2qovY7c4d7vy7YkbuAYy68I6slPUmfYDxIuhO9W7GEvMrxpj2Szya84hEPPc2gjjrINJ+8zF/SPegIgr5DHi86R3Neux6Q9ryUGRK8fVaju613jbwN8U88FwHzu9eIsL8ORBw8nK0ZPHWWDLwmOVy8iLgWuhzh9jqCaE28FR2bvCXDgb+yueU8SVgoPufx3rwAR0K8xdwCPdCKh76/No++9r8AvhODAD2/+F09qikZPaQvk78ABDO8JPi1O1KMpzzMcfW+x/beOnb/X7zvCug6y64lPXntdb9sFP2+NCW1vHxCpTxombW8fX0KvuR3Yj1cj0k9ep09v7Ttkz38FxO+yoC+PPiEsjsdf6O+HUpDvhhPsDq5nhM9to/Du15Rx7wM8G6755ZFvC7rqTsmICA9ZHQdvd6jwru0bYe+PADBO9w+/b15G829KM8dPWBaTb2XbD68kN/4POCpkb4nXlo/0nDhvWbIY7tzWbA8+hU/Pf97Ury+5Ca+i3TGPGmp3LzzHtc93KSUPOnLMLrJyU69kRmgPENkBb1vIb+8ucaoOhmmCj1jgpm8ePNqPVEfIj0qHHI9A90zPc0c9rzdkc28zT6TPPLE3btGZhS/tr3LPOtn4jxRl9M+d3Z5POXEDDtdJ069","jcskvIzBs7xQdhm8TLEHPXx6BrysKva65qYevuH9crxjqo+96c3lvcp2RL5ifom+4Yu4vu1hgLyYWMa9enS6vOPHI7x1gjc8MndIvmY3/LxJkRs9SwDrvFsfELu+B9k8Mol3vXw6BL18Dp48UD3Cu63CHb26t2a8TfCkvQ+rXb3utNQ8FbPguvLNQT1Q2Pm6TFIRvqPjLL130YC9hfuCvMmiY7yh1v28LhsivpovHLzgxYQ8RKUPO7aJ/rs4HYe8CwoEPKLWpLxOQqO9PQxFu1LxoDw/FP2+XS44vZ/sKrxDo9k8V6TBPkm8GzwlG4C7XYgTvZHnb7zGJqc81058PZqQwbxJ8Mc9YDaaPGtmir35yms8LsuzvE/j0T2MBd88eXB7vRDSZb0tbEG83zU4PWDA1rzaqVk7L24Svp+yJjs5pz298U0iPIxhsb3Fa9U9ejlFvOrEc70Lfso8gOZsvBH93bzAVtg7qSisvTp/Az1AFx681gpuPXvMNr6hWp+84cR/PIaVqr2NG2C51aDYvI25Bz3QgQy8WTCmPVBqUT1O3+u6VEiMvPFYa7772gS9IlZLPRoOBT6nQ3481HbJPeN9YD2aVLy9ysB2vTF8JT7crCy91JvZvA+igb1pCs08XXyOvD//7z17MjY8DrA8vSw92zowkPc7vSiLPVElvjyAQ/86x2hDPGkvFb2HrW29+bGCvd3sezupAXe6y5EDvMq2Jr+4ywa9kwm3O5NZir5L5K274lBeuzJlNjuDREA7Wm27vGKBPzywtGg9iCJ0O1vMCD2kswO9lOptPCopOD0daK2+vmVAvdj+KL2be2c8IL8qPMjL+rzZ+Q++m9HEPCnR/b15RRg8YbeAPESolzsxfZg9RIkCPZyz4DzcmLO7JuAeu800+rvOeak8FN8lPDFctjzz7rs86XdSPKiEAT1Ywx09BnNfu+/2BztuWsu8VcAmvSW8Ob0oIk++WsaxOgjbJb4Ax40+Raabu+l29LupE/y+9oOeO+m3FLzy7RI9","KTdxu5gQEb2bEAE94oxAu7vCg77ZktK7d5EsvZrEKLxszCE9Wwl4vq82EL3F8mS+NgZ9PRDkz7o/sGY7FlqSPN2iQz2HIEa7HupUvZBZOj3ZOhQ9i/KtvunkSTyHA469cdKmPbnqgzxBHDG/UmezO2xngLwyGro9dNg3voqgNr+Glii9zW++PL/E27z+SEM9SOmgPXW4ijwflFq8xQiivNDWGbztDEU86pZBPPf4sDuKeOg9kCU6vE+uXb0sPPU7InujPCf8ULyAb206B/GLve1BGz0Hqbw9C+SyPJW5OrzgTsC8kUYrvyvirDxQ0ii7/6qRvoHY0rxOH0s8eRAavhgkMrqo6FA8hAKzOjxlIjr4Q/G8fIhUvwlq17tsazU8MQE5u7iVCbvii4c6Tfg/OqWb/rsXVd05BQuLPAb4GDtyF6s7GWoXO5eqr73pz3k77c42O+84ojsRxCi73BGTO863CbwPWAu7ZHckPF826LyDrBU8ENZGvJ39kr2i9CS6mKyJPDBho7uLqAs8wgyKO9IcjjqFe/67jR+LO4O6ujldls07iJT+u9dBrzyjd8A7Z0iPvEuICzz9Occ8t8fhu575yrslS+G71B5LvAvVgrywqoa7whjwPIyGGLnndV47Q3/suyYdDLyKWRk8SvBIuoFTt77V8T+6AXwtvHjOLjxXk/m7f3iGPbL9Nj3C5Su/qY29vehaLj6dy6g+DIIMPm/qwD2jm9o8CzUuPqJnEz1cmFM+Moe2PXm2DT3LXSq7NdTyPappBj+trmG9+VvJvljOU76BQKu8bucAPy4C4r294Qw/i1coPu+HFj4fVOU9W4trPoKcUj75s/i91U0JvoC5LT1KW14+OIeoPV2xVrtvYbQ+BzTCPYluyj5Lviq+3GyavmCKNb5DKqO+1Ga5PfYfCj7MU8e+sTEYPBekID7DfYo92b8UvQQNoD4zLJy8ZH7NPce5nD709uU+zsQCvqaIIb7nBLg+qxQwPq8aGz6ZCxq8bwh4vReXsj6IIEy+","NBj5vCGgIj3YMbA9voWYPd3RRzyE44O7ZpSFu6j1KrzncXA9ljxovbJEDT7DBjY99sFOvSP3iDxk36Q84TpTPS+7xT2t6Bu9OYQpPdd+z7zW71E7Rex6vWTz4Tx/WNG9iqj8vM1kwzyQnBO9jMx4vA9BWT2EhP+8oC8wPUybpb1Y4UW+y+ISu/shGr7dmya93uUbPWxsOr3emgg+XrJyvV53hbwZcWu9dvQGvvO48DqbdHM9clA/Pa8iX72i77a9CDLAu2GW0r1HG4e9kBspviOGJb0wCiq9TcuzPbGjez2bOa29wRskvafHgD0u9dG6JQdMPYvXOD2Dh/U9i8YlvYx5CD65mae9QOT2vbFOj70+wS8+jh6tu3v63zzbHyu8h2uovS3wJ773mp+9fOfEvdX4/r1nSVw9sP5YvnVarD19tvY9B3Z0PYlizj6QAwS92ICovN9jyj3hnDm+2HqNvT6uKLvNBhW+7yEbPWnwSr1IfTm8aBoOPV6BBz8DB1K97mRGPYgTWr1uNlm9Gv1zvlI7cD4gMLq9XX4yPfhoizxN6kY+i4jsPMNjdjyVvHc81ymAva1Zh72kmIY+xXYKvjFxIj7XXNC9aNwjO1Pmab6Qe1M92zdAPgSvdj3es4S99hMHPt5bSb1uydO83ByIPa5ud7z9gIu9Vn2jPXy/ir1p1ms9OO3Hvc4/OL0TDVi+7ZWdt93v0byzmcY9/sYcPZv9/7wbHvU7W8gVvY38ajy9qJw8rwnHvQQCTT2Cj2k9t9BuPZSTCzwfgdw9uFyDPK0bITyznIC9doktPWaBPj33HLW8ZHqYPBJZWjvvV+w6a90/vHU5djy8nsw7Ru5rvRTHIL3U2Za9G7DNvZrpLb4CtMa8hQGovZ42C7136ZM6lx94PcJBjLvQ4VQ9iUYLPNvG4L0au6A9wdmvvDrOGb3rNgW8HVqQPOHCA76stK68YEFfPO1jErzpxCi8wMzJPcQKDb23IrI9eqtSvbY4AL7qFYI9/WAePLBKUL6XNKG9","xoZiPEfTZL0C3hM+Nvk4vlHZqj2WSbe9O9D0PPbSaz2dAOg871zdPCaNt70y2Cw9GecgPneWjb0R9Tk9UJO9vRZcMb760aO9qo0Yvrw62L2Jl1E8ByfRuhNo3rwpgoa9WX4KPZAEGb2N/mQ9BwOtPmgCDr4YVyM+NaoMPaS6M7wgZzy99zbCvEt5Qrzkl8u86keRO8TWiD0laIm8DHTFO5t2LD7m9y49kcSCukiE8Twucxk+q6BiPGEk3byRKnW9HtBNPKYRBD6eYoy9f8NQPXiKgryIuog73SEIvi5SRj074/S9Q/M9PaFp8z1z4uG9MT+CPesFr7uH/vy9I1Iuvfx4e70k7IA+k2W2vRpvDT57nIi+w+9lPAmqnT1DXJA9qlw8vqECeDyNQcg9TpsqPiRukrr+c8C8Bo3oPeXQtb5UomU+HiCIPeuE3j6w1pc9LucqPfwTlj15eQk9AGIVvoENg73D+x69ANLxPIpeSb6DcvM94+WLvpXlpb5xPDK+p2uJvgsdC70fgAu9w3JTPK7LB7ylvQC9hlL4PE3hOT7h0uw8JJIbPkvkGz5x5VM9toNIPp+O0b2JVHE8TrBTPYmXHr20JpQ9iaHpPAg6Fr5DNNQ955lMPlAmdr0Surq7B+9XPjHlSbw2Ms09JBJdvTptJ750bFI8M5psPbJ2Dz03GVK9wQVQPC5QIb4ioFS/RpeIvcWh3DsROxU9464PvZw+S70xVMw89T3Cu6O1XD2npoo84GKkPWudHD5ugX89fkhCvY2Pg7uCnr+8LCrmPWs1/bvjD+a7y8FfvUbGjbyXtaO9Cni1vGU+Vryiw188W67ZvT1ilD0Bwrc9BEieuwyg/jyij0m9cKOZPYo2KTwBXny9ut2QPBBEujy/UPa8RNz+PcykmLvJ4Bc9sFvAvJ61mjyjAOS9yGxsvXof274jGTc9kYo6PV1zg76Fd5M8k726OLJZO7y/5Da9Tlyovd9htDxhc0Q7PGitvAT64ru/3bK7C/inve8Kyb5Fope8","OFEaPepVvb02hQy9L7+iPOuShL2m0LI7J8lzPticmD0Oxm89eQNUvS/UL71V8UW9aaOqOx8Bgrsziv29OZljvFhcQL6sIB++FexLvY0RYz2+SWE9D6TdPbUXor39tEq879m7vREFtDzr21a8RaFEvYfvpb3BLpQ7wk65vVo9Fr3uJ288l9QhPVLqx73IzEU8DscPvQLDWz0yMCY9aiQCPoWhID3wcAg9UOSaPOcf57vs7Ec+sLkzPfvVTb2IEti8/WiGvdsu/z2hfTe9wAPFO9uDbbxcOg6+nwihveaMKj1Zl/U9BB/APEs8gr1NYZm99lE0vZOHcr3wLG47HqlGPctEKT2UH7+8GWUZPW1cDjqUZGW91QpcvSAjAL4TZlY+5VoXvgcVP70pkCi9qveKvH0hJT07JC+8dWrPO8s/HL4DEx685jPhvOg6s749gy696mpBPrBK4rypCMI9hFPmPeE1dj0ObSi+HMbFPGZ83LxtJ/i8tuQGvb34KrpRBYS8fuFnvkkLirxymy49uOfnPb1PUr4eYQY+h5+1vZUyjj25it29e3yGvlHT3j0w2kk+xiI7uQOsR71226g9Zn8UvHKlqDxdtRk+Gwu8vHjoBD30q6u9LQOQPLIk5r1W9gw7jUCXOWj1Ab03Ue08S4PvvamgeDz0aT69QgPNvOuLA73sya07dRAdvJ4piL7wrq69GkrKO0/Urbz1eye9pD0lPAitHjvm2CA9xiiIO1NqxDu1s0K8UkrZvhNK3TdUALg7OlKXuxjcMzodcG06mjmZvp6YXLtOEkG7BQBdvrkrPzqE33g8n95dO8I75LssnC68AwnvvvAxprwEQEW8EPHBOpFMvDlOlc67/6lBvO5vgLq0z2s7/xOZvj8K17xyA6k6lAzXvucyFT3Tzkw6ebrSOthd1zw7o7e+k7bqOJjTw75ln6y9agLMO4e6Cjxz+9g7PLPKvC8MlTsGPme8f+oTvn5Fgjktv7I847LZOlfYO77O/Fg7r0g6vn6Ug77k5Ki6","/HUBPOEQrr0ijhI8yZNnPuysdD2oiGS8YXl4vsA9C70Z76w79GYWPQWMpz2gG668hiYRu8L0cz4Ruv299f1yPFxIVTvc34e+DG4+vcS3ib3d4CE8bK4EvfGgprx2nnS9TXZhPK4xhT3CCUS8T2WrvffgK77JOIM8MPKCvaAsOz38f5i8/uHUPUghSj2SlX09ugBIO0tuT7172G685tLzvHUpJ71Hzva+g9pSPgTGDb0goY29VpEgvtcFp73ZQ6U9f8CYvXjehDxLpw296pd9PoLmoL1hQ/i7yxmkvClpOr5j6cu8Tm+dvKuAe7zqRzK+Of7wvLN+B7sVKAo8efDlPcbyFz7T/BO+jzOQvQQZortqIEo+lbEJPd6ELb57buA8jFTbvcopCDzA96M9u0EZvo5I071sXru9dJKlPaM7ETyjsq0+07Q9vcdxqT0MOIE81bURvjU8gDuIEgm+XBA7PXKiaL4WwK08pipDPtnN2r1S96M8axEwPgoFvrzIfSQ8kNNZPvMKg73Ur/29ekGUO3fidzwpyvw9Fs2TPUT+67z5gRA8GNE/PYc4dD7JgWU8/2JNPbhdDb3l6UI9VmQMvU2IYrzHNlW8I4BEPEfzr71vXQ08aIh2Pb9XJ76oDOG8nrf2vExB8zy83/47ySjxvT2hn72Xr5U8FYamPeNyAj6l2tm8eugbvhVU/Tv85ce7GCLQu9w5JbznRHW95lHNO386i7xxOpG8TQHDuxhWJLuuDE69i88SPfI6z7w29Qu9N9STvHIV2L64/CQ9Jx2xvRajHj1zwys71YMhPb9kLTya8x+9cHClPK8MZzz6VTU9/5oxPCJK0ry7YQa8K3CXPAPSsLzSBBW8eWCZvTSRqjzQ+Qw8QI72use7ez0VUwK9c6dTPRvlBr+Kusq8loZnvJPrDzyoqdG9CF6DvE5JhTwI6TY8L9YQPUiEV7pG1r08IOUDvBd91jdnALk8MDGIvUX5TzzpxmA86lvTvV00nD1akTy7IkdnvMcJ+ryJE6w8","yokkvHR6Lb3EXYu8MqvDPLMjwbxMh3C9rt5RPTJkTzyZFK27CdOBPPY+8LzosAc9oBaCPTiWKz3ECFI9iBtEvYZDQb1R5zc88FRmuxXekDzib8O88oFZPc5ANj3pT3c9jJhRu7td7D3hZeY94TvaPT2JMT1odoW8SZ/LvCZmIbvdmBe9ekjVPIZjCj7QjBK9cp+9PR5IFT1mvzq9Gx2uvG40Bjx1AvO9u7S8u4XfQz1V9nW8ZboIvZLBQTr33Ji8IsDuvO/5AbzWYII7uphzPdqjmLzbqE29SmqFvBAOcryWILi8ZM8TPo/s3jwrzaw8pk+XPHm8Vjw+GEY9HthdvHmqIz0VGpi9uXOyvfNO8jsYWUs9YuTIPJ5hgj5rGbW8DI1kPRWQljySOhc97ZzKPObU0zyKjw494GPXu7f0A73sshu9yVmyPTE4Rr0dkoe9EwmyO/gUYz1xq+g6Ii+DvMgqrT397yg9ZgY8PaVRKbvqm0G97G2ZPHGKWr3FSME9zIRlvG34Bb1qXqY9+F8SvBWZQD3r49S9e4qUPdrLi7zoiAs9D4jpPPXpJrztSiK9FOPauzGkCTwSXa49mrMnO7RakT3q5ku9fPEcPCN+Hzz+4Yu9Oa+IPd0G3rt2Fho9M1RbvNMc8T25Qc87PBlaPXu6s72ITiY9otKaOtpfnDuoeAK9rQ7OvXfQh71b6006VCUlvfpXw7wI5x2+zZCHu4RdfzuMBPm8ykbYO9VnlT2vPZq9W1HhPU7T17tEnUa+br9pPX1XhT3GIqS7b1OxPIO+xjzFu6i8xdZJPSTJL71Iglu9Co9LPc8hTL26vpC9F5WFPEwskL0la/g9XVXdu9sv4LyXZxm9nJKAPXS2tzxUcos8iqv1vJ6AQb3tOMS9W0FbPeydcD1npqW9OS4xvTUkVj2S2m+9trGKvCCUE7t0bhA9AExEPSItC700gia+TJgnPfk8rbyDUWu9dlEpvTHa/rvOPj89OIrFumcbnr1THwU9gduMvKCQDj3F8ti8","RYvyO5/JSb8enGc5FEKcOzRI5rocTR+5ZB6DO/UWGbwU5Tm69dZJPJs/v7vxe+g694D9OkZZFbzspeO7Ip8Pv/BqwTtlsim8F8lkPIS2Q73lnSw6uLzUu+PCCTvyG487Qhw0vHvsULwpTgm3ydjLO/lrlDsbGIc7uZ6hvCfqeLput5W7uTHAu8Id9zt3kAO/ggdcO3v1EjrA7YS7dMgCvxeuRrrntGe9O3oXuwQArbzQ5Kk7VF9Qu/dAkDoUs4y7s5fpO1dwoztbFp673NnnO19RMzuzww68tca5OnS3rroVwAk8mc2qO6pR0r4RG927KrYevIznrrsgOTG8H1IzvAHlML4LsQS9UDq7vabSgjyAaQO8F4VFvZaVwb0whyG8kTPOuh4xHD1Rkdi8ReNjPZJtsTz7nBc+EekVPCa0Xb3L5rq8rjB9PO8R1rxIEb89AKCxPZvsPjw/r5M8t8FFO/TxTr3nURs9XgIvvGSsqztXsBS9RsnPOkxxYzsfsya7vg2rvLuzN73I5ze9ioZRvWXJu72dVZi9lfd2PaPgRr7cIdC92q47PZxcyryZq2S+PHqVu6UcG76dECW8Gt87PWnk57wrVUa+WAmfvjtUtLvO3bI8vpxqvffNfryXvXy+w7QVPTBWgzzP7vw8ylSYPe1lPbwwuK085XsCvqaqujxVK+M7EqOwvad5DL0E3YE8po0Zvu4D17zgZjk+hzYBPaSZ/z2rLU+917K/vC8lMj6Yzmq9jMAQPZ+Azz1WCgs9qwPvPdoNwT3A0g2/rCwRvrV+/bx/XHG9hFs4vfpTWb7k9nY7YnesPlurxDxkVoS8G1mCvQ6mlT010SK+bkmNvc1m3L3pFBK98CrBvdZdIT3fEOC9zqJ7vQ64HD6VWAM9P8zOvDlgGL0Yjow9ZdSwPG1wCT4FN2C91tFsvhkkJT28mgE8ZcubvXu/Zj3YB3c+54LAvXRM573CICk9LGxLPRo1BD03968+MeSJPPz1gD3Egak9D/GePOkV+D16P2s9","lehnvbAWJ75yTgi9wj6rvkdvZ76QkVG+TSNcPO7qUj7dSYw+sllkPjF/tj1tdmG9yYQYPsMYKL5F2Ii9go7MvFFalL1ECIu8JHEOvt7Hxb53YjM+x6HUvdlraT4y/CC+CmN5PnsEqjz5aFA8b2ROPsRqBT4j+Dw+eGfJvRyZND6j/kC+DfxIPoPPGD52cFI+VFmWvcqndD7u5gQ+Y7OHO1Dt5b2TMII9budCPpAdWz6ob6u8sxibveh+KL6+tEg+dIH5PcYkKj4HeLC9k++hPpJMyr5nFRC+HF4Zvf4FWLosN9E97Od7PmW1Ez41jj4+ucz7vRZalL2wGqo92gAHPmaTmjzclPQ7i48sPQnFgj14aZ68B2hWPsB5N767s668NTJDvU1JYL2ySbE9qj/pPaXpWr1F5WI9ILaNPBN3sr19FMo78Ji1PcgOoL2S7fW9vM/MvHBtm72XuIi8ok6JvQhkkbwn6RC+i4ZDPQI3xz0E2yG8j41MvGbuLT5SzLU9ROzcvYpAn71vBB6+4FkdPeYA2Dyjxve8bxIkvasnrDyj8IA9XkMTPV0nAD5nC8M8bdWrvBcHNz7daHo7CouKPIz0iL0AYEg+ysCoPTDfHDxCEdO9bELxPTWoIj0eR6U9ab89vahakb2CPrA8MzWWPcy9Gz61oZQ8MZXpPMl5+L3piyc79jFkOjOkDLyiC6O7Xj5MvBwJXL+Fs028noh+vB9Jg7tQbTI84fgePMIVWzvoKI+8bC4YPMpWATxnFJQ6A1m5PGvSirvZ+Wa9qn0DvNTV3DwJLxI8lRVnu4aQRroJ9NC7ZOmpu2brkTxLA8c5k+zyu+xogjvYDoO9PdF4vBLghruYbWS7v2JGPLLSubvEcIq7OVV4PFB3fLrkFyG8kqQrPJwdxju3Kps8WL87PMypJbwRZ787BtOEu4ZaVbunCTA8QCzVu/vB7zvY09O7iOxDvEp0+rn+/RY7zetavKfpOTxM5YY8jcePO+W217viU12/pUK5uxf9Ebw5+iM9","yr+NvqytlLwB7T27wGkyOwtADL2kDoc7TO4QPdaEkDvmUdQ7Ue8LPOcN2Tz86X27l6SUPBV2JD2Zs3K9cfI9PJzZlrsiGoM8FNxZPELasbmY4eW+Sn/Ku5Pdib5HWS48KWuWvMlm/7ybpJ+8EOdTvQB70j0nXaG8PP0fPQBDobyEqzG8r5sVPZSYm7xnRmc9xi/GPSIIQbyxSOE7lXRovTvV270TPYG9tGXSvCVmhr33Uzw9J029Pf+Kpruwt+G8Iq2MvBAJm76xP4a+Sp0ivGvznDzBvCg91GsYPYhxxT1tBru8b2yRPV54sTyk2i28x1e1PIGvj7yC75O+cqMIvIVgkzyXb7+9uH30PHa647vyEOm+HjdlPMV21b0oNXI85NKJPcJN8jzF/oO8L7jgu4gt6LvaHfw8qPX/Pe+jer0Sxxy8kiG4vC10az3EqRw9zXYvPI/tN710aN88OtnOPSpSoz0R5YQ9yCLRvV9RlTznG9c7bIx5vFWeOTyp3K2/k6+cO8V/Dr175nk+aUvFPJGr1j36/LI8A2v2PUvhSrxZrLi7etl5vRAIm7zkqx87sqY1vaUDFD34PFA9rx5XO11bOj15sRS8blGlPThZZL2OUDG9gzE5Pa9ToTxeIgC9lKUHvKZxI7643IQ8xtYSPUgHBDvr7hO8fdHxvPqMhT2hCJa8agSCvvkWBr0TyWS8z/TNPDM3ojsrbKE70DsTOFKVn7ztEuc8m/jevXb8UTwoJcG84b4HvHWQar1RROa6e5jEPVjiYr26Wd89+Qwzvi6bYbyn9DI7sUgNPcu/Zz379NG7pq5/vRwXV71kqb69LnvHvB+yCr4Ot6A9BLkWvMMtAr7VMO276rIRvn+W5L0sOX49qYfNuvofCT59j5q9PgSAvQih7r3RNdw9o0vdPJO10T33dWU94WSjvfImpb1PnMO8JNgOvXPO+TvjRgw+Yo/LPQm4Lr20eXc9kT6zvX+sGz2M0oO9igtOvWk5zj1bSxS8xFimPfKwKT3lCrQ7","7QqnO93GlLwF2zK7DkP/uyMXnLu8+ps6X2scPVPI17t2hQK86FPEupbhuzuktkO7qYhRO7Jz4zyUUlw7yLodvH/gADwDEvm7MAu6Oxt6ATveBvW6nv0FvLo4BDoLgRe8ZoUQvQ4gwjwC2go79qS2PMZjSjwGEES8lKzAO8X0njkw3bO9ojiIv+YaeDxbfIQ78yybvXjFRjxbymC7ILdzO5xaZTkX94w79DyEu83Pmbr/pDa8mKiEuz2Ao7vpXBi7q8nROlnGijxiVGo8P5+nO2uJGb/iNa27QLCgOjRaRbujbDa9xWfYPN11KjwxZbe7BcF2O4f+ZToO9h095p0ivA=="],"bias":["V1v7Pu5hID86VSI+AfdBP9/Vtj43rUA/L3ZTPrZ+Bz3Sh4o+yuDpPkQuvz7IltQ+KtarPegbpT4K++4+78bEPuDUQT+Q154+yrCDPlBLHD8K2cc+Z4QTP4U9vj4xyRc/w1J6PnNekj6tO5w+J2OBPn4s5j4zK70+oFgCP2cbDj+7uiw/DBSHPlj5lD4xUZM+GnZxPsgVhT5wRFM+NAsFP2kavD6I3eo+gUY0P9pEMT6lYps+MfiRPtdDiz4Sxbo+vhajPoWN7T4h2h0/g8S5PgZJGj/4Z3g+/P0LP1XXgz6s7rE+gOaDPrMZAj5+zYM+dIUeP2rjzT7lyJg+DSuwPg=="]},"dense_5":{"weights":["WYYlvlt25DvQiiQ9FV0WPFTPzj1Py7S+X6jpPdCK+zsWKQS9CKnSPMjbDr7B+gG9jJWpPaaHOz3UfRG9RTEovmNpjb2S2WU+6L90PVIB9b0Jekw9wYOQvhB0nT1PmSW8rLkJPrI09jzm4WE8JzGpPeLctrzZd04+gwlwPaIJDb16iy2+2/ulPXkDVb2G19C63J0UPu+MsTy4Q5C9u1hkvhTyxTxMp0a9NjkSvtO+Eb5Un5a90OTgPZwB3TxPHJw9hTraPc/XTL2OWDi86p8MvpwoZj2q8t89TR/mvUgH8L1VeRq+vycBPYMEjTyWpKQ9ws6mvkyFB71DaYa9wkCsPeFXCT50U089N11HvcKc7z3egns9C5AsvlHfiD7JXoY86aNCPFp5A7zAdUg+j01mOdt68T2TFpS8SS9uPp//tT3IZps+i9S6vehQJL3rx7E8Y/QdPfdYvz3zFby9Q5cGvbNCy7qMs5a8Db74PByxwT3MuSA8nfk6PYpOij37PMY9/4EOvzWYPb4D51Q9cLVuvaWn+z3MgQy9EkfBvRnDsD0JZS08nMD8PLh/Dz6b+EY921kSvoWu0TzjTqK8qlbyPIc8vjzlDAw9awbAPYvpw7w8Vr48x/GUPVEGtLzJTfM7x3m1vSAmAL2GVWY94YS1uv1d473Qa0K9S7qavbZEib28Kcc8gJEIvm0Hzz18gX69gUgWPfbWhL5HIhs+qxSBPRwrN7xsG/89qSdjPI17LD51s7Y8EqGwvIkCUz3ctw2+cgtQPfthszwRwPe82oqMvJ1kGz5G9uO9NmL/PYUFw73fuyg+DWBMPnXrsT1LjFU81bfTPcxHpz57BB2+cahZPR6i6r1PMjI+1toKPszw8bosW0k+MLO7PbKLuj2T0BG+BM5+vcNWa77T8A+9xmTFvLHBFD0sonm9Hl1yPe8+QjszZLs93ysTvvVTg703/n2+SH0GPkD40T2V1rY9GjOAvjPUf7xspy8+VCO3vVF4zD325em9E+N7vYdJUL7omVk9","/cdDvZkWBb44mOM8+zotO8X2Bz28aza8tzqRPVsK2z2+LZe8MqgHvVv3AD5hDRm8wjGDva+P3z225bY7bC0OvLyGrz0P4JG9VeBnvfVFs7vMfRw+wMCbu3GqYD5KKNO8XXzZvSV/Oz7U0Zs9GbWOPWtwn7qNWY08S7bfuxlh9z0TNDy+jMVOvbd8kz3yRPW9b5/9u2yAwjzQz2E+IHXDvfW5qD0OwQG9jfQfPipcx70wfs689Y5UPGy4C7tq6cY82k0FPQwf9rwiRYA9s8wYPk+TMD1viV69I32nPcqvyLxQkDq59dDRO6bw+70dSM68Dg1RPXo32T1btkM8kNEAvvJKij0o0Sa9xzI6vmsk9b2SFzC9sSfQvQhrBj6Xxzs9SU0dPqJWbLuy7xw+ZhXbPbGk1T3XxYG+Ydg1Pguneb37VOo9Vu8bPXACh7xEJRi+Y87RvYBRtjoXdNC83be3vH2s5D0hCNI9Rgs9vWWSYj34DIM8n/ZIPL3tDjqhFRy9LqCoOmh7kjxzNTw+BdsbPuPygr1DXZe946UQPkq/Cb7wW209IOEKPcLq9D0UcnA9IJYpPQpoE75lnIM8fO40vYrcUb07xly+B58cvbx1qbwT9A080XCTvHMttz3ceFO8ctavPKphzbz9Tqu96qdDvfvy1DzwzUs97ZNfOVZCvzzKHik+orRzvd4RST4h+HM94SJevfFGR7xc1Q09a40svgQhkj1LuIu9flc0PuDBAb4iE488YLKMvQDUWj7qeso9VC0fPhNeFT53PLm90vZtPJua6j0cf7W8q3kKPgXEBz2v5yI9zVIUvj/AJz1uVCU+Hm6RPSUtxr2vKVq9EBCaPTqkF7766ZE7HFi3PfCMMj3HdE0+nQh9PfKVpz0jIxg+8gBJvPAVRb7qVrM9SwE5PvFxvb3lSOA6q9sBvGj5Dr20Aqg9RrY0vEsNrrynUl+9zzjxPc1c2LyXsrQ8HWSNPYjCSz08gau9GJsRvaiMBz6nG+S9SasMvqRdCz6Kxc+8","PJuePGB76L2QnG48mr8HPLVX7Ly+Pbe9aSeGOwmtoDzsO4u8BAupOWKqW71T9Y49FHo8PSqxlD0OTMg9IGZTvS8WpbylN2u9+JBmvl6EvD1mw2E+mbnJvc9NRrx0Jp28hJU9PJHIIjylOGM9LVszvfRBxT3Znfm8l4AOvrD5h74dYmC99je6PZnqCT26kmg8FknMPb2C0jza2Q09q+I0vT//2zzJDT6+9BQTPuUD8Ly76UQ91E+1veSBiL4FTEi9TMqpPSLw3D27aYu9Ip/7vNnjiz3lHwU9P2IqPS1/Qjxp3QS+ER9lPb6bujyrw9E9vZyhPeyREL4VtKM7GdNJvYlDKr5+1Jo9hKSrPbdyPr0vKtQ9pRwmO4+c/jzEP36901nsvZOQNrxRC/g8i2csvRm1Jjw9UY890jpEPXdiLr1waxA+Cf1HPU/wPLzNwQW8OOjKvS3SBL3nIb89ymGKPQQWpb2D2Xk6UzH8vJf9mD3wQBE9maNwvW1olryHFO89RQtTvVKXtr2RRnQ9YGo6PUyVyr06Vas97qSePZ1jdb2s6AE+2uylPXWbMTzL78O9UqbQPQKqHj0iUho+nY7HPcUHYj22RmS97H1fPf2LNz6Jp+08Io04vi6apr2xu5Q9ziwqO0S1NTx0k6Q9iqkwvXc0fj3HkAk+Yu0GPVXqO70EsXC7+qgZPQ4vyb0UDPo8gmYyPsI1Uz2jH/69wVkcPR/chz1ERHg+GQqPPYl/hj2vlak93g1Evqgifr1Zv+O7SEdNvjzVw73Cn4U+NAeBvXpXJj4lE8I9drIPPV4XcT38XPU8cNJkPucFIz2ukRW9eI1yvku/Bz7hXqq9ZRgUPrBOGz1Uajs94mqtu2zdAz6YYZm9VPdDvjEeYj7rEdY9oFAGvjDdZD1Imim+Bp8cPU0Ep7u+bgo937+ePZbwYb0sTgS+nwsHvtIJp72Df3k9nt60PNd0Qj3ulQg9hbgkvnfJDj5+lfc9Nac0vfv3Pr0eVmI9EKf6PWjUZr37n8E9","1gtCPXx7sTzWuIo9GMPYvUNRvbwEesy9/LI1vuy5nD2M8DQ+p48vPkSbZruYERg+UUY+PaKrHb5iU029AhNvvaK9QbwiiSU+nGGMPScEg7yi6rY8DyCxPVHigL6CjgM+B+k1vejCtb1UHlq77gD9vUyFBb0AkVC90ZLZvVc6wb3L2qw7nbG+vdq+L77IRge9yHkDPXO8KL4+RCo9IoDVvXzafL4sAjy8UNsJvr/2lLxpfOI9q/KCvaAmlTtbEQI6De9fvvq3MT00zxu+O9QQvrjaST2Y/409SU6gPVkAKL7oLxG90cGQPWPCUzxseD2+9CFuO1IiXr5QTym+MB6bPS20Dj1iP5c9FM0WPi98TL2PAGu950EkvTZHKr3XWia+F1cYvXYkEL20cjO9FNPjvUj6UTwmJmM95Nq1PfEh+Dy/lf+9fXXMPMgFU7tLxTu9scxbvSr80rwNhsi7giwSPBBIWLzP9rS9XBZ8PB7ppjzNksm9qBRTvSRjDD6BJa09kOm4vUCN1738GZC9sGsbvTCDqr0WDrQ9jMCbvcovgz0+DzK9vII3PcUrd73leBW9vrr8vdJfuD1APTW9QsuHPJRJqb3Gm8W9etPHOiFZQL2Cuva8OVg3vY8jh7309QA8VR64vSxdELxbIYA9Yw9JvJ4KID3Hj7A97o9kPW1bwDvhRa08Mg6oPOtGRT3sUBs9gJCLvYfDKL1Px7g9euEgPhd88T3aE7i7XtznvVb8OTxsWsS62Lh8PU8pwD22RZe8SB0svb7Xzj3xZPK8HJedvYtLB75EWm29aEZGPTZmj721lZk9m2XtPRaRIjwtebQ9aUyIvRPIuT3kJqG9AHMZvc84p733wuE89TabvK2pKD3JT1+67ttBvSAk/z0ePuQ9llY8PenT2T0ANtI8uexavKIMub186a68lLAKPufavT2WziI86k/6vAaF4z031E89OjwOvpsdFD0Ppam9MsD6PO5Gpz3Rc1M999TxPYGsiTsLUAu9LZS8PEWtlL2DbG89","rG6eu3goIDxOJBU9++cJvv6E272kZ6c8lc6+Oq/tUD1Qqxi9g6Z4vCTm5b04tgC9vh00PfssVD2WJu+9/SIHPfpp570R0Z48pShwvvJQ3j06kpI6wxiEvQtbyj29XSq8A6BBuxPoEz3QQDs9mPgYPsvF+T2rReO7v9txvkvHJ74ukDw8eu8/PepqUDv+/Bg9u5lMvfyc/T1nYuA9ac9dvSXOST5ULvO7H3Y0vixdWj1Z/BM+r1aCvXTZfb3KV9O835UBPnYpAz68Fs490VPjPBkrtj0CRiG+IUGsvPBUGrwEb6U9hEhTvvrd/Ltfo4g9umBzPA49Fr3aL6E9k42YvTWdAL6k0wg+YGBLvMU5vD158bo9tM8WvR/KAj4oN1U+zZs4vkN41Tw4GO68Czclvj5BJbyAYCS9IB9HvAiSDb7bzXu9E2YWPjcHSj2tckq+aB7gPKuOBD23eVo9b/6APSiwOD3IIyU+mN1qvb44+D2gPeM9kMcSvTo+Nr1NN9a8l9qVPJxIlz22IrO9sGPyukZ1ib5Wsqu83AFEPVY6br0wyH88DvwZPgpyir0qvrw89n19PUnEhzxgiYU+u5Q6PqVjwjzoBEK7lQCzPOYvYD6StRW+6hwPvcnuZz3e0wy+0/o8vTpq+T2AJ+G9CNtiPYOM1Dwe14A96Nq8vZaAmT0Rm7Y91toZvvwAcL2RcSu+IQorvh5Ehjz9A8e9DswuProu2T0XdxE+SIJMO1/O+j0POL89Wt3RPVJIvTxLplY96tmjOg0FcjuxUDc8YnIavVdYAzx9mt09qdymvKMXxjzvfe09PyVMPcLMpjwvRoE9IIM7Pff5FL6QXzW+VWbevY70Y7zOSH09Fe0DPWDG6Drfsak95GTkPYWREj7Bojc7pQckvrI2ibvp2aS9i7+6PLnqQz4Od4y9XnaIvUYhr72bk0q9WFDGPSjv3DyFSqg7OZf5vIZbmz1g0uG8i+kNvuIKibxtbg490E5+vdZCkj3zjUw9KpfFPSaYd73yd7E8","cDyJvcv/lb14LcW8DIwjvrNGEb4V6Uq8DhV6Pv1u0z4Ttik+4C1XvD1Lrb7n1gy92ab+vBhBy7sE2NG8o07TPnEDBj3Ct3u9C+egPcz4zrwXMKO8LdtHPfMdw73N0oy+idyHvjrKtT3fpgE+lMozPZoIQr7Yro08migLvjd4kb4sCjg7W6KqPkYt9zzfoDW8eI3ePRKiCL4t7lI+cysLvX1cLT6iUqQ9C1sMvu3JlT1E6S4+wyG6vTUTB73jaRu+F1wCPj64H77Qu0S9QbefvQQMG73LHRi+d1kqvkgyGzx8N78+YrcxvW+aZz4txQa9777svNja472/VQs+FE8Dvrwe6bz/bV89V0RMvUuVPr2ZNK699T24vXKdIb100X09/bjwvEiXW71bWii+fNYSvcoo/r061Hg93aVevkGWJzyjQxW+CGuBPUDGS77IBSY+bTULPaSWQztUUMA85sJHve9RMj0ArvQ9NLaYvCs6frzYVeg9WxBGPNr8Or48sSS+FZzRvTSHND35Ika+G8W5PZItA74Pcxs+y5+UPYKPGD7Twg0+rxUUPUJ8KL7mJRs9+f3xPWSh2b0GuUO9uaqnPAWcTD2oZCU+VqtbPhy6arxhrWo9pcfdvOrthT08xn+9iT2UvdFIL76XNvO8EwSlPd3Z6b183BI+l532O2YGrz1JS0G+56GcPDfarL0xK7k9UndKPl6Hhj3FNom9DYP3Pc5MOL6BDgg+VVjhPZgP2b2Za3I7iNJ8OzgB0b22IpW+p7rZPE1koT2fWOK8oeo1vmXmND7x0Jo9V3epPTaquT2gx6g9nUrYPU0FD71pQlw+1NJvPL81V7uhwp0843icPYOlPj0ghwC+LrkFvVvH770235q+Gv0dvmmBgTxM61S+bCQXPVd/uTyIX7y8KppxvkXY2b1om6I9gBsePk4vAD5J0xi+yO1BPILXO74ipus978+fvjH3DL2UaCI7LOiNPejbKz7SUe47Jv4GvvrHJ722UrM8mCkNPuSfrL25Qts9","WlWovR6PnbyiflS9tG8rPfbQfj2PzrM9CM0nvRR4AD1FUck7nDaoO8cbsbx4xtA9dJoLPoOQhr3/OdK5xW4YPoZnJ72DPLm9WGoYPoPHfz2f30E+kJG9PZA8krtn+oS8AT0qvfLkmj0yAfG8chwQPmugVj2/HyM+7K3uvZBEQzzzonc9aO4fPBo7rb38B+g9BpgWPbLdhL0L+rc8pf9uPEgEJD7EzDG+g7Ymvrjqpz0twnc9QSYrvnZw871YMOY8LdFAPYiqLj3HC5O9SJMCPLWBqj0Le5k9AkWyPe5qXb0+6A8+ZD1JvNn1nj3yiyW8Vl1fvft15ryc6Ka9hYEKvTnJET7eFLu8/VJYvQn8Bz4ErMi9SY1fvQU/Kj3Zg1m9iEaqPFtEBL0PV0Q+I/EfPAtPDj3GzX49qjjdPTa5ID1c63Y+uciuvUeXODxk37K9XfpgvMZvxDxexwY8zoOkPdxZjz32uUE91/3mvGQkFT4zQAg9s4tYvfqKjT3IKSw+ErNHvvxV0r3neBE+sNqTvbSverqPr7M91uc3vAiWRT2QbPO872quvT9XzD14KC+8mOiJvp6Sij20ape94miSvCwksr1UDo2966BHPttDtT0asII951x4PEsTgj3KF1U9bxpVPQ+4tT2+86080fG5Pa4dmb1Y3Xg9P2iOu8KAX72y+xU+C2jKvfpSFj3R6dK8fHd4vWQXob2+ZAY+Wl7pPYfDPD7is1S9Jm/Iu99UWrt+cGQ8Ih8fvY5Fiz28LvY8aFKqPGavl72/K4w9RqbzvTN/Pb7YfYs8/c0VPuLupDsqHFs+mV0uPvcz8T0v01k+9kRHPQP8SD61i0c9c6UwPtNouL2jzpY9vylGvSYT270I5C4+MzJIvSaCYj7IXus85MuMvanskr221JW9LQO7PXL7Tr4TJgU+rAmNPRnvBT6uj6G+X95fvtlCBz7DKY88uL5nPDXUXbqAGv89BBkQPdmxjD7Vx00+OMrhPeHrXzuLaAc+Y150Plfrbr2AMKa9","+1hjvSX3qrxsx7I9Scd9vaVvAj2Di4i+We4cPgE7Uj3tk5W9QwELPfC0Gb4LTCg+iGmjPYuQG73riZI9UK4wvVMpOT36yRO9XTwNPeqHgz1sb+89D3oLvi+o8z20A5K8tAJdPWVpHD4dqjS9w3zSPBj10j2jNnQ9wlX2PPL0sL0ZOEq9r5gsPgMKpj0PK2S900XuPMqZFT2e+6K8MbYHuTMSLTtODWO9ZJehPf3zxT1rcMu8EMrCvN8eF74EbOy759dyPfYwpL05xiE9aR2mvjPHMT2tr9g9Kk0MPTQKC74T5bq8V8rdPMyqGj3jIFQ9gRaFvjjwED39ZAO+ICJWPZNbJj3jmYg9zZpIPjPtlr1W/iw8AVCJvS4qbD2OH9q9/0vzPYPPIT2UbbY7tXs2vu+uBr5ZCv08EwquPN3U/LyOrzQ9Dbh5PoEgHr67Kae8uFvePWYfkL5bWR4921nnvfsdmzqQVY29mymYvU1wmL3zpbc9Z+TgO+nyD70IzRW9UdalPW+gMz0BVs87CIC0vY+svj3iVRY+iuyxPPPnSz0GKKQ93mzMPG9fpT2p2He9310rvdpopDwbHgQ9MJ+iPKyYnj3xcUO9+3GMPOky2Tx+NpU9b9v4vSyfAb7wcsM96L3WPFw28L0B/ZY79v/fvRx8Ib74c989XhLwPUaogTw36GA9dwWivsDblT1PcSG9Nv1fveDerzxP3YY9Ri0PPWt7fj01kRe9HAv2PLw9dT5btPM9xwUvPc3fHL6jMVc9RqfnvTR5Ir42w9I6+UXpPf8Quz13QK69iv8wPnHJ3jxjVPs9Rna7Pcdmajz/6Iy8yEaAPaKaXT1O/Js93xgHPaBTtz2Oqb89QYNhPSnF3T3W1H880ZaoPZK/4D0GM6u9WW0WPm1xqr5Y2hG+KKWCvShaNTsSO6Q9O14Avad1tjxMV7w9D4JaPewLCD1RsbA96eEwvODAiTzk+aY9JNcIPeU8BTznIAK98XMKPcZpDD7CZB28G0prPXGS8rzUnyS+","Lzw4PcCwozz3FJa9Rb3UvPNC9T1s9ZU9qiPmvbafzjwpxyq9HP8iPruchzu/iYA9KNU4PnbOBD1aJd09RpgLvkIGyj31AHA8df8BPjhP9b2kSdQ9SmaAPex/9r139hc9aA1GPW20VTzBwQQ+TrjqvdsahL1HB4k9hJSDPckHMD3mc+U8jmKWPB2RgjtsWwc+MWCLPUgxmz3OUEg977kWPpIiGL4+QGG9C/4fvg5P+70q3s88eWLTPFSWO7yPH7u9M6X9vFne672shoK9f2ArvsQWKT5y4fM8oLPfvIgMZr0N5YU9ouswPqCUnb1MhRu+dpw4Pu3qI74axNE84lR6PVOP9zy9Mfq94wX6vS3HRL1cxXO9RUCUvIT46LxfbVw9t9AaPhi8Qr3MgFq7CDDyuyWKUb0Kghq+Q5B3u/xFZD14PKq9adUdvnbh6D2h6iW+WXXpvVj/ST6lce+9ScZMPd1GCj52EvM9ceEuvWTM6jwa82Q8qLYkPbc3VDyf9jg+elkmPragJ7zGHCO9B4rwvTBi0D1Ld6K9/lU4PpyO4bw7qSq+81uGPT59S72mdxQ+UFpoPazFXr227mY9Afy9vZ5c970LFIu9jfZzvBa5h730iJi9gePZvcQaOD1NoSa9boLaPSqdzD2aZ7y9eXfqvRPmizynmtc9/SMyvZlQpz2rr/c8GR40vL69GD0UEK09Xpq0vM9cM72n4mi996CHPTilp7tuldq9M2GWO24XJb6qT6G9TqEpPb3FX74HOuA8Rp9zviCaa7x9O+O92Zz6vZROBj2plY49oKYOPcdJET7DHq29cNG7PRLApb3fUVI+hTg9PYoM9ryvHvy6G6mCPYXmCL7v8iO+A+H2vXdeF75TFQC+anpPvUtSYzxz/he+NLjzvPDJQT2Q7Qi96IXmOzK3eb5tnwI+0u80PamwYD0QHaq98/gVPVDf6TzUSpE9OtcuvtBG6zxix3c9WLcSPlffZT3mhzK9tuNivaeSxD23M9q8bH8VPuvvzT0CQ7g9","10ISPsT90D0Blg09QU1jPWasQr7u+cM98h/jPCRePb3nVCy9NLJJvYFPgj2sxbO9FrQPPf546zz2fLo9MooTPZwEOD5NOmI93Ab2vaVALbyy5JC9TgnXvOOKHD3EJ508ZsyVu6mpKr31RK28+xW2PfaUqz2GZ9G9MCyGvYULe72pHsE83bW5vJXgvLzMNFo9uIKZPYyJCD7JuOW8QissPtbq/D1fBso9o4g3PgzGOD6mPiu8cQTXPUdD57zbujo92CIHPrOBEj0etrM9X9qcPVdfXr0nOWO+BDKqvS3lWj5xiFk9a8KZvdSXlj0acws9MIGkvcjNDD7uN9I9z79IPMVnUr0NnRg9zi2HPSANs70+ruO9sDaavUv1w725QXS8Mhdlvg/g8jifs9Y846sjviKPBD2rYXO8h2cGvRg0SD7/2Hk9J7KlPfzAh72IMSa+Ld9ivonmML24+0W9z/10POKYdT2uJ/K9ax25vDufBb1JyKQ9Fa2MvTeXtruEH768rnHDvc6QXrx7Amg9NhiLPSCBk70PHd49xsmOulaEpzzNQlM9j+C/PcksnT1UGi0+RycJPTh4f71Kbdq7/xUgPtrIPD2A6+i9lqp9PT9YI73FdOW9ximjvrZOdr1FDlQ9Pkp3vdpPTTxhmac99MwgPVW7izw6m/09uJMYPkexcj3jYGE90RLAPVVyBj6DBiO+209tvlbMGT0bDiC8yCYlPYsuNT7kI1i9+fzLvXd3yL3Zvhk9MJElPirv7Lw1Lpw9Q1zYvUSXEz00oE2+xG+UPVxpMr4m+mU7oOH/PHbLmL0NzRO+bZVAvhv5K71qRt68ZdvQPX3KDb0+Jsw988QEvqHJrj0WxS691yNdvW3SOjwBnWY+wGjSPRHYbj2J/iU+niXJPeC+aT3X3bO97LOyPcix771nVw0+gt2gPGXQ/zz/MyY+jaSUPTDWVj7WF0S+z1ggPS2daj1tqgS+o73KPYpRsz0ERee817vyPMunpbzs1YO+ytMQvZUhTT6LoPS9","4u9+uxZLnjysAi29soarvSjY4j0Vcp09++jePf6Uwzri1+G84WogPmBm6ryqElI9uYgqPC2XnL36t9q8q52zvV72ir0SUtQ6398APrwtkjz/G6W7qvDGvbiScTy3xUC9gn/svBrx+z2t7ZA+v+etPPiUxr3LbVU+XUkSvUfPGT4GDvA7Q1G2PRmQK726iEE9FeNEPAazKb7sW588KmlxPG/Kob2bqTA9QedFvZjjyL235tc98HJFvt92xT1Zy1y+5dmzPWSHIr4GgQO+i+EQPJNMdT0oxQA+97WmvWhFz75+4Z89ZdW+PQHKUj1Q7gy9tSW7vTfecb33WQ2/cA+6PWfhR733g6I9W/b3PS498butvYS95ertPM6Cpz1+r7k9NsOKvWN1uL2yzdM9eVTXvRFEoD1AJO67AVAzPnyLjr2VhRg+bssGPlL+Vb6vUie+ShPwvefIu7y7o+s90w5UPQXUBr5eyxq9Di2xu/r7Fj40Pss9VqO0vAwAMT0l/be9dBg8vpO41b1Zjrw9II8gPDBEIr6v4nI9BKXzPVJOczzhg4Y90OWxPT9enz3Kcis9/yLtvZd5+z3va5Q9GyQDPqbbUDwwWuQ8H53oPQY2BD6uMoC+aEWMvdYsmjuTpRI+RxS9vPIP9Dzd6ie9ZKpNPbVkdzxaxwA+wElPPvaatD3GULK8XgKrPRPRkD2jHIq902kVvas5VL0SH1I8A2i6PZWzOj6nHuK86T4yvc0/oj2/lq29XuRTvqEuhzujiIS6BC6KvY6QMr2zwqK8qSZevt9NN7649588qs+Fva0KNT43lFw9pNgYPhOOGT4WLea9YHy8PR1xtr1D2EQ9tW/AvfHM973ROmM+rSmTPS3CjT1Q39G99rgIPcFYmz0hHwa9FDwJPoQFmj2aKCW+dHTIPPa1Dj7Hu7+9zKqiO3sx5jtXhoe9TYj9vSOKurwcMua9IgVqvQvml72l2EG9XfnbPQXTbr0Gp2g9aMKgvNYOYL103Gg+dtBWvajfpj3//689","GgX2vWHk1D39OAq9deGhvGIjCT7ByfE9HfzgvZQ1LD7dpGg9CKBrPf67Jz4qFXU9flXZvZfN1r2gevK9c3q4u2HWZzzaIaY80EgZPtg5hT27bRm7HtTEPQtiIL6LdgE9MteAPGCdtz32ZiY+DpNOvrlvgj3TjLC7dqIOvpwdHjvOORA7eiOSvQk0rby5IPI9ht4Hvua/kb5MvZa8mW+zu9mYlr0r9gY+6CoYPT3dI76f/6Q9ZlkNvjcYKD4cB5G9K+envdwYhD3/2yq9CHoUvMp7kT2/81490ReCu3GRl71n+L+9Wcu1PTyy0z0LbQa9CrHRPJQtvrzPsXC+XfMvPkIaQr5s8uE9izS8vDsH3LwuiRc98iWlPPvUqz2AP/M8JT/nvXB8Wr1yI0k9Ify9vPnahz2+jQO78EpEvQ+KxjwDklq9q0y6PfHoVTxcem09koYBPSlBI73+4R49g7AdvX6WqLvFze68qSsqPWzUgz2Ov0y8ocSXO8g+mbvgVCU97dsOvWUWfL2aQam9+dw6PZqv1r3BSjO+RK1iPcpblj0pnO28/hMwPpoAHT1J5/G9ruycPYrd+ryfiw0+O70BvQUxtLxY/DW9Dz49vqvJgr0LZ5y9d5GGPVJt7rwpk2q9DXVEvKya1zwLKkc9wL6DPZuJPLwAXKm7LnztvdleKT0Lfpg99J+tvQLnMbwximo6Z1hFvGoKgT3SGN2873A8PJRrCDuDrJS9/JAZvoE+LL7wdje8pakGPqTIMr4ObBI+q10Qvi0npj0kfZS+S+8sPA5Cqzz07ho9HZkzPhelvzxGAk+9xRybvXT0BLyl4cq7wvuGu3Ucq7x70MW8+I2APMg2mD3EiPU930K+PBuazDyNVHY8Gy5nvaaVdj1rOYw9vmOSPR/FxLxtuk6+WBBDve/u6rz8pBA9FYwUvRotCD2mNgA+jHh+vctWWT6IRCI+uN5mPAkytj2EvF09he0LPkMtcD4LgY29ZtobPlp0qrwLYhG9kqT9O+KbkT2OzuG9","vS2ZPaGeQb7qH1U90wKrPUDiND7kAVO9X8PZvBR/0j30oHc96z5WPWg8Fr12IyK9ObU/Pfklhj2eya687HWHPr4tALy29DO9QaYkPnxQ6r1GnuA8DL+tvU4ecL0AVTY9Cy9XviXnBb7ehgk+wJKsPTfjN71k9Z6677kaOT/cob2pZHY9bweGPWw+/rxELxC99sWIPfIXFb7ygqI9iZcRvugtHTxA4Gs9jkiVvSJG9r2Rwg08adr2vDmNWL3uX9W9/KGrPRbTZ76bfIU8G4G6vdqM/71ZJZo9SjvavNNzvbwRMFM+G3hmOleoKz51Pb+8pZb7vQX4KL1XdQW8bC0evqxDLz5fKdM9gdHAvSr72z1fwys+ihFmvFmcjD0xsCE+tXFHPl8hDT6W3VQ+ubswPYGSVD0H0PS88pdCPuTaIj2jgno+B/GXvctHiT1Kygi901lKPRBsQD1LTUe+YT6GPD3iHL1YTWM8IToMPjqs9j0giUK+CrIaPisbGz4uAls+pY45vgmdBj3gjL49z9KDvYN/Zr0vu929tAERPYUWTT2NrbS8otAKPhj8sLtQ3sK9FmjZvfYZ2bygeto97iravfsFvL2ESUW9UP0sPdX+Gj5mnw2+szCfPSfeEL3BT1U8sqKCPQIFWT1EVjQ9HAyHvOZNsTtqOam95HMxvvefJTw7P/o8iPimPfLNTr3cJvg62eI/OIijKj5yDM48qFAPPu7l8zyd9qA9i0lXPdmFnDzJPj68MQ7JvV14DD5zBiE63jS4PfFi2T3R8H897yv8PHMrCr5P7Jk8KSvku6f0WD0WAOy7MzuavIqOsT0RVws9TzplPTPt6ry9xZg9rzdXvSegyzw4Chm87+BIPWp87T2CD1S9d9rDvW50nz2MkY49rG+LvQAP/D2M9Fs82uFYPTatwT0x51O9vnc3PibEET19GFs9xuBivUL6BTxWeR09IyU7vZCyC70EiwM+oq2OvDNj1DwjDEU9yvQYPjEIGD269+s9Z5Y1PVF62r0s17w9","ZcANPutArr0G+t68B9IJPpU0xjtRHrm8VEaKvdSczT3LHS8+X8uPPJZE3zw2g5Y85WQbPUd0mz3rF7e9+xnUPapoMDxPewK+Jf3NPYk9p76luAS+qQwcPqrKOT3vkkw81mh5vdofET3bU+49L7rBPV9MD73g1CK96RASPh3dmz1VmKY9BHM3vfF0DT0A7ZK+taJ5Pb1YkzzJMuI8l8iPvTH6XzsUwTm+kFnBvSkGs72dB46+b7gfPmgvkj2ryyc+SFydvhv6gr1VsKY9UzzuPcLHIL5tvRw9vhsdPqrixj2VISI+Y/8NvaWoWL56QjM8Pgg3Pr7NPj7buiM9DZaJu3n0sT01aYG9YyuYvsWJuL5LZxo8ThMXviV9yLzchno+7BpTPumADb2nm5c9sU2UPWznCT50D4I+LlkoPXLBFD0Cjvc8wwnTu+t8hT3rm4W+Vf+GvcqAA7wj5kY+ZaI7PYzxfz57vKo9V+zbvav5XD2nwmu+rQxAPXn2f7zCkAm95EEjvnEPaTscOte82XbiPcw257z+vJ+9tLVEPoDxvj36fw2+K41cPVhq1byMvjk+0XJkPtr0AL7qHrO9mKd0vkr5Sb7T+kA+hgchvrene71CR249JdY0PX40Jb5IFNi+7vI7vnoAlj0wjQ29hZb/veKRID3M0tM9X1UZvnsuOD6twAe+KlsUPffTuLzgQHi92nyUvH7yAL3yyw+++mg/Ps+jfb5uFbC88cWUPBDQkb43lUc9ex6zvQ86qDy5Gd+7dlazPVs6hT3Y1cy9obhovqo1zb1XIee7bV6nPcNvUj12ZxI+aHXnPUFtiT3qO7U9cd4EvJTGOr0DX5a9DeEgPuBDlr3PyRc9c9CaPaWoijyylJO+6COuPcHpFT73plQ6ofpBPQodND2y7hu9DXexPVtEGLxkpkk+kW7YPXKKhD7Gvfs9P3N4PY4iLbyiXyU+VWyuvmiyVT1fBwa7kwYiPoROTL7nLtu84pmAPdIsBLurenw8wSQdPk7SfD6HhO89","jgepPdEJIT1e+EY+b7EsPXWJUL5NYaW8oojRvfaNi72ceQk9mJB1vlWkuLuPPyC+RTjgPCY7QD0fN209ef1EvcNAVz26wSU9ilegvorr2z3pUdY7LtirveYDLz5XeEg8hBfUvT+8Ob2T6EK+gUOSPcx6oz2tKwK+cFR0vZbMGL4Jc3283z2PPAQ7iLz4nnM9hqppvcmzlT36wQI8vjztPMJDTD1RZTW9aH4GPsEaCD57Kyu99r+wPIU03zskrsK8FCgRPjHG1z3d2xc9hx0sPbwlJj5sGU++DUZnPMb71z2BcUI8El74vekJfL2ZDqM9skm7vaHNiT13Fus8n6GrvWjxWjxlTnU8nlL4uzqFTj0c8Rk+oqlovTm4Ib7q64u9234NvaOq+z3IPi086NbQvKU3+j3OBy+9CPN4vdZnoD22TTW9WxXuPCirMD5hvBo+3StuPP4wFT1L1m29/vAlPKNM3LxiE/67HkT+PRGFuTztfJW9fH6DvTZHwTtGoHA9cdCqPa54zL3v4oE7K8TgvMH2lL2nHmy9XsJVPjUeRr3PeQe9P42HPNf367xpFiM9aTZoPTmQSD2sFyE+1PIWPSSoLbkfctA9kkYhvZgsxb3abay75b6yPeZhFT4QEem9l+Dbu/fn1j06NYw9elJtvTCl3Tws8xk9J63qvFGvs73Wff49gbI8vV5JSz5CV9U6TDIFvrZKZL3yF3a9r18PvhnmRz542AS+MQMcPHNKODwNbFo8CLERPgwxFb1r4Rq+g1WcvE9u0zypTpS9SFq2PBYbJLzdJQ+95qGRvFt2Bb6Uhti84iVNvTRzy715cmi7XyA8vNDYDr5rdCi+P6ZgO55nFb0jJpe844cDPpiEZr7Gcp49ZvqXPU0exz3ggSo8RvcMPb43WL5/ITG9e7KdPRtvKL7tYmY+jpyBvkpAFT7NZdE9c3OKPQXVELuHMtQ9JkikPIK0h71iPKY9buEmPlmMCj4uSTm+mqRFvktqPz6gRho9GCwTvZcEYT6vw429","zeoSPODShT10tm29w6gNvjFYiD39fLi91NJSPVY1zz2XOpA9G8aIPnQ4xz3sWu496ELjPbNxgb1c5TQ9fxsFvprn4r1u9tI9ipN1PpA2Jb2FXbg9eAbIvXZUHjwx9xI9v7JBPg07HT1Z8DY97hwAvCLYZL161CE+TZvBvCkKCz0D2Gc9Oy0qPj/VDL6+eQA+J9PrPYr3jr0boSc+pg1uvtUG+b3Wh6Q92UxZvm+i4jy0zWA92++EvjnVlj0SfUW+LGBvPQCTpL6xxFe+MOFXvuLfxj2sG+s8FqFjvLfNx75Roc+9TA4HPqDcBz5UhqE9YDq+vBh8fr6jNZq+QFYbPnbbJzxAdWw9C6MNvgiwJ7wb5LQ9UioaveXsgL265d88jj5mPRzvOD6urM68fotMPlwhiT1fR7i9aHIAvG2KlrxOeZC9O2LLPaoHnz5+Ndu9PEz0PAlZUD4ZeRa+mgwaPiGF/zxL3BK+K/RRPUiEJb4HPMC9CG6UvHqXjb08y7Q98rx6PVr7Bb0AJwu+Ev4jPg+3uT35zfW9bWzhvGCOi73T7bG98QvSvdCvKr5/NCa7A2qMPfQlh71akZC9FuNJvtHOob0komq8I9EPvZ0Ix73Rego+xHAgPoljgTwLWJa+B9eGPagypz3vjEe9PdNuvnmumz2AXq6+ByGAvb9SH726czO9YbMoPV5cKb4jQO88f/z/PXgaUD2djg+8+rpaPVQRqb3YaKQ9Z6sQvbrf/j16NsU9sBD4vXLtkL7DOsW7On1mvh2LAT5yzro9BLcTvNq6kj1nx1Q8aU3uvZ0hvz0qd066m7GCPUM1FT3NJai82lJ8vP1pqD1+T6u9eKIMPT5iAT782LO98ZLMvacqib1Gi9694QMTvnkNh7w4aTS+cm2TvZO1Qj31xMS8isLivcd5sj33Jxu8rqb6PbBM672eU9W8LDLYvYKj/r0KzgO9wn+mvHjo1j0gwtU7E70gvluT5DxqNp+6Tyl4PSfb8LxPNfk8CMBBPHX8xL1ujw0+","4FoIPcVqn7yYB4m9ZbyVPAg1hD3UlS69ZJ5kPt0g/Tslngy+X8ggvl01VT2mOOo8YvmfPH30Vz2789k924cHPRV1hz65VG+9s1zSvLtWi7xmIbw9gwLMvPeV5jwNBzs8dlfrPbc4Fj59zwM9mGc3vfvBGb0Gux8+rmbMPUvPID3s+sG+OTmGPUe6pz3CeIc9K7FDPc46SD1Y4Lk9Wy6gPWm4jz35NVO9OY46PhtUHL2bgi+9gyWkPaubDL6XRaC8P5WHPckFrbzNjfc9MYSMvbrwWT3d1ig9QaZ3PRK2IruqY2g80IARPQ1cZz0vnxS9JKCpvbYpQT2Ftxq9+CTePLiZGL4svl4++UH+va2Yibz6u/y8eiQNPeD9zrzzGdg9VImRvTmQQD0EcDM+WUXVvdcEA703AIS+V8/BPAO1NrzAVns9CEdqPUtlR72cSV08mbB3vokvRT2rs5W+GjEkPiAgBb6TtA2+PPuavW3sDb7+jWo+5QAkPX2XJb2vS1a9bB30PVRECr4nBqw8FKZEPrx1ob6QR1a+kKvrPOKnJz7+dEi95ymAPvNTZbvwT54974zePXDJWb2OnjA+xsAhPc9HCj4uvsM5dkDfve8UfLuNBjc8I0clvldorDv68si79DLYvs/NVL2VZeI9oax2PmjgoD2Bb7U9e+DcvRG1QT534Ke9NU98PSjWSz10W7y96CubPDk8Zb4UZ168XAWevXk+Gj1rEWS9BNirvTDpmj0sO2i9uFANvXNUOr5XGUc9RUUwvlHqGL3pLny9EIylPOpOeDzoY6y95XyYPWD9FL2jUoI9Q/2KvAj9nb1VdKG9nmaZPdQCTzy0a+C92q80PasnjLyXehs9qxmaPDzo8LxNoMC9S1jBPRrhzb0vXNQ9ITCLPdW5lruxxD2+sX1CveykjDy6DAQ+scaKvXpmhTzM8Ns944a7u51rXjw3ZxG9oHv7PBxYqT1lmF89i6kVOm/UYb0WnxG+OuOePW5lYz66hNK9F6XIvdZUETy91Ow7","j6hDvrh5J72L8VM+u85YvbimGjy916o9H0y8Pdkzgj60okW9v1fbPAd2zr4F7Ae+EoRQvciecDzPvjW8vxegPg8vD7wdHrQ9fXaYPA/vqz2dpo89/Xutvmie4rwVEHe+AyfJvhg2Cr5Dhw8+mZAIvVra2L0rHtE99KFBvpZoN74lm5O9rBMIPkq947s3QTO8RR5WPl9RkjzxWeW9oVkIPXoQ8z338Dy9bf4FvMAie70nPww9NPx4vSUbprwTl/S9qs2zPb5cKr3KEkK9Pfy1Pf1UFb0AmXY7wX8Hv4iuqr0xjn89kgKsvWTUiT7ta4W+tSEzvhpsnL4xExe+/v1wvTqnyz26pjs7eLOCPVJSjrxgEAq+KkUnvlHs4Dzszta9p2SzPVywdL5CQOQ9c/yvvMcuBzxGP4g9v60VPiTYXL2qGjw+P4zSPXO2Ur2L/Zo9VxIUPWWHBD7Uo1k93LVcvV9jQz0+cQ0+pYqfPeFdnrzfoWg9SHlzPb+8x7sm6bQ8asoYvOZBvD3tR4o9RsOwPfygWD6xU/o9inH/vbyAVD1FDeU9dVAIO0PcLj7gzNA8KA7VvNnjXz0wTfi8V5uBPRHQCD0fwbm804CMPMCAEr4p0jk+zvCHvbXwQr05ItE7e3UFvdjMrrz8UUU9DPL9u5JPTr2TBZi9QL0FumdpLL7VHKG7tEN1vcd3cb59/EI9eajWPdlDRT10wzU9pq3BPTENNT2/ZAg+hnOQPen4Ej63B7w9NBBAPcNXEz5PxCK9VO+hPgkgQb3UT1M+XyEmO5Pa4j2zuJw9+s9jvlPkSTwwn688PMiMPW7+CT5cu7s7CLmovf6Jgz3ac/I96uIQPuWHBzzvc329QTMJPmwlHTwKlQi9QqUSvt0AVrxqLM+8qMBkvcDtkzxXDGE9nKmgvS1QITugOBW8b/mBPPcHkb3/wv+9swIavp+GLTs3lb28p8DJvbtXzD28pE09sdxdvcWDjT0lBR0+XJiHvT7E+L2OvMw7VYoqvbo7672YfBY9","vP1vPSUSrD3YUSK9Ne3GPYcMlb3NtGy8MysRPgjFLT1tsUe+GgU/PaYr2D2t5b89twyYPZbBgjrc010+4N8NvZ3GmT4gQU+9y5MmPWwDR71Mt5m8Hhj1O6uukTsriOA8eSoJve4SfL11gmc7iUjHPYDPhzzOFxE9KacCPgBZlr3DmJ6+SNsnvKYKEz6H4pS5zAPyPewcy7xXZuq8WNWrvO6hiDyrrYw9q3pmPp/iVD0ofgS+2MwjPA3guz2Vy0i9MfDZPLS4M73fMRg+8TFquy6OhrtRGsm9nE8evfUz5zy7ZF49I3M+PUDkWzxw5LK8wxYrPchdYz2kyw+76Yf1PD/NlL340eq8ohdqPfVNUb2U8FS+opyavFb6+r3ca5u9mYlwPJd2B7w6lqO99jY/vce9Rjziddm7BFWMvZj7cT3oP529YNMnPcy7Bb6Wosy9EowCvkkVZrwVOVM7c0Oqvfe5Gr14Xoy9MH7mvTo9dDyZgSY+v1uTvRqaBzwLgGU9MOg5PXFrCL30TpI9p+QFPp0KbzzsGzs8ugoBO+pCOD7XTDO8cN8+vZrizz37st88Zfm7PYGwwbsY8qC94D/MuzXP/j0GGJA9XDwmPtk+Nr4nxh0943bHuwCwn72OuHM8M8mDPXmdbL3oQCe9Q0ZWPhnwj71HHAg91coXPiPTHrwNoBq8uB6nPaUJIj19Pl++Vwwxvc4mGT1tdQw+da+nPWi4u7t85Rm+lAiSvm02KTs6eye9tg0DPebKoL4yLDo9KV21vlyNlb00Ayu9lhBfuq2wwb1742e92wNvPVQyir3XTe89Ety3PJCH470Z3vk9pzYuPpxJMr34s/C8ipGhvQ2W1L3vUyk+3Q8ZvY8yPD1RWp69jUEaPgc/Hz4oc5c8ACN7PVhfzD0zRBW+TcykPeCK2Lz/r1Q8kAeJvZVkgj2iM9Y9gTBNPSYeEj5hLzg9FcSOvNY+YL6Flua9vFUbPWRChD0n/Uy+bQXJvQtJ8j3BPh86ip4vPfsIhz0vZeC9","xfuCPf9Ncz1/VU2+xuGXPcLv6jsgaPq7hJuTvDqOzT0slb49YFBtPt9zhz17UKI9FaTYPMrIbz2vZ7C9fI4OPp0KILzIb5Q7XpvPPQgoOb6oOFw8lPsKPs/VLj2rdDI+pX7cPSiAcz11okI+SHwiuuTp9b2mjJ86N/o6PeRSXT6956Q9Oe0FvFvyx7w2Q2g8xpTBPWZm3r2ONcU9rNCmO3kzeTw7qxm+nfHovF3X9bwgooq9DYGAvTpDwD1IZKI9HjVwvo3gqL08RmO9g+KKPUtCJbwygw0+Q2O8PY8DgL1rtyE+GaskPeD7c72DTQY8RnK0PWPSEj2NQh2+NM2LPaHe5by65NQ8gQ6xPDf2SL2JWiu+gF0hvcaucbzBOdc9utowvPUllb2ibQM9fwiEvWMVJr4aG5W93SkmvUI91jxDrQ692IEIPv4orb1Z6eg9lOJAvpiBpL0j/Ny8dQVSvAYfeb1j1w2+ONlfvdxeeb265ws+TSHbPD8cDb3Nq6+9tDv0vGGFuLtCIIC9ESRUPRKX3DykSP89D+byPBDy1TzFE6A9Ac0yPomj8739pJY9A7sAPN7p37u6kDU8JNMLPHqm0j1pozA9VLEvPhIxAr7Ha3+91zS8vftiSL1vigO9/c8APBdtqbyyPWY9RbXJPPS+Qb6QcgC9vH2dPU5hgj1zQjY950qoveCNxLwNRp89FaRNvSDjrD070U+9vGu+Pd1kI71iBdo9AVzePS9aej26IpI9Vo1SvTvppz3BKq88iDSrPeIoFr0QbIg9MQODvgHdeL0JaOc9L55ZvTV1bz21Veo82RTIPE5/0bwGyNK7t0mBvfBShLxYLwE+eD84PreX8rtqve48SbvEPZzdl70ocpI83T7FPY2vrz25uQM8DtEIPXSBh72ZYs49CmmYPHUsKb6V/6A8LVHUPYGQxD0+h2e+mM3Wu+9bqz0lKfE99m6gPRG2lb3f+Bg941W/PT53oj2bPPM9Q49QvaX5pb2q4Dc+v0IsPV8VEz5sa5+9","kBCNvCsvDr7626y9VS2UO4vVlLyndDy8QwRDvVQczb2Zv969GSTLPUzwhb0hbng8uwu3Pc660r1PhFQ9P2+8vWnuFL1Ns3m+pQrMPfIztb0YqFA9SssVvS/RDr3jd8k9MwElvpmhqL3LeBQ9cH4lvptst71sBra9qGybvGzdTb2Fkmk8cI8nvUahGL36Xli9lHmuvTyTHr4ZMPa9PGVgvPP8hj0XJxu+pNWFvFXBCb0dlW+9S8STPHOEgLxZkvS8R1jOvbV0tD01r0I89/K4vTPGnr2ucoO8UmN/vQ8f672YtSE8Q+vQPZ5kcb57Hri9I1guvo7FK71S8PO8P/sVPfH+J77HagS+Zh3ZPdsvlr4ePCw+YMkEvm3sTD7beVY+GjvPPDSbgL7t9Xy+fho8Oy22Gr51Kbs6Q4aDPa1KKD7BPV69AcvPvQU7Cz7m5IG4iERqvdelob1kLeS9CFtAvirjC77fQuG9CVqcPVGF4zwAp5y93y0dPQR+qr2gydO98tgKvnbIjj4rzlq8ywx5veiB0T2iFA694cyFPJ30hr2m7ZM9SCXHvHfPUr5tls29oTxoPCnflD32ze+4oSVGvkFAizyVHY++xC2mvqaXAz1Znr69uJ6WPfmzlb6M2O+9e6rYPdggWj3oBZY+oKN1vfiBgb1TKtO9cINTPUohT74iw5k9+jaKvcsP+zyJSvU92XolvV/uOb2SenK9erOgvXV6tbzIaK69C5IEvfZov725UP+7dd75O5cPwDxI9hu+GBDMO/1zVD04IQi9y/BzPXHyr70L2ae8B/T8PI/zkr1OqXy90wndvSiaej3tYXQ8arUJu0NSZL3B93y8lAClPQnEDjzwoIG916pHveNKmT1aFPg8hUevPQLunruIB409FM1bPdk6v7zSeEA9sqEqPBHF37wBJg49AlX2vB0p/T1hHh0+xUkAPsM9pz3H/La9Lt0mPXeO072aUHm8f46APSn4Tr2asQa9XnH5PV1u2D0lZRe90oAmPLHIkbxEkRk9","r2L9PHMpkL7AOqW9qUK+PADuJz4B7bs8VvMEvSuJVj66Wte9mA1bu9kf2b028gO+GoSjPTgQtjxFLY69dKscPUl0Lr2eJie8enXuO7vx5TtarJk9bpFrPhb0Pz7Vd8u8+UUTvC26/7wTXoK9spe/PT59/zvuO5s9di8XPdizJr52IYs9zZLJPAkpjzpfD9G7gPkOPs/9Nr2uikc+PYwQvRg+27158Iu+hpgzvYSw8D0QL/A8oqaEPId9xLzvITM95DPUvbpN9T3O0YM9V4IMPv/dPb2A/Ho9l5Q7PvQYpb3tp/U9IgsOvd43Zj0JZJe8wqAfPBcCarxwey69z0SIvay1Rr06mQU9Gwmgvg/LKT3bisa82QDmPPme4Ly5m5A9NzU/vIPAOT7fi+c9F0G3PYSRj7xapzC+/UQ8PYwnh70a3KI9r6kUPFsuSD62Q409KTkcPkvN3z36bJi+rx5cPR+YEr0iCjQ+REqYO+0Q+72adzw96690PRat2jzR/Xs9BP1QPumrxTtARYo9KEQRveOtXb7M9WC+YTKLPVTZb708XNC94GxoPU1OXr2Q7fq9Nqo2PR9tiL3VQxG9vp0SvhOsIr51zTy8RdlDvd8p4Dz1eP09ATZhPYpfojz+Ewq+ihfePepX9j2K0vG87d/FvTjT/jxMDYS9jlGivmvkQz1MntY91wF3vvypJD4sJkq9oBiEvBsEFD5CaBO931zLvJoGKr2PtSu9bR6SPVyafL5vaLg8JBAWPr9RRz26GZQ9KWMcPa6Jgz09Iai926cTPE4wxj27IRC+915ePkfhc70fos49ePDvO8lGE77CWCA7qwVIPr56CL7JKUY9u5eFvppVtLmPW6w8zxJFvT/f+ruuSSa6qoTJPbU62TyOnJ49pRMlu11QGb5eygW8TeGGPaDxrrzCid89kRVFPqFHqj3t1/U8AJLAvJgrCT7G4RE9kPRwPBLQ8rz7AYA98PT9OwdWNz7vcr+9TlwFuvBJ0DwpEDQ8e4zWvYS24j0lo208","ZkoRPZY7PLx+MvI8jDM7vomWCL3p9gK+AF12vltA7D1HcgQ+8n/CPUl2A72Tcx4+PyvkPADzzbwg1ro8wN+2vT0MTb0xOLE9AsnFPcAlTL2SdPM8KVOGPF7VVb67Rp09febzvSO7i7zXnRo9rJcivktAz72zB4i+ngDQvTLK7r1siDY9++Y3vLMMlbx2GNc9m8YMPUdSwb3W1ji964mmvagR+L0Y8429gvkAvrBJu7zGo1s9RDD+vckPpDxfLFK+8cy5vVDW9z18s8S9b6YfvbxcK719h689o0HjPJTd07xckC89gUw/PKRAD713Upm9LxA0Oz9C6b2v6QG+1Mi4PJ71Ur3hxgw+dBcuPlLYyD2UsSy+K2GpPL2LZj42PR8+fyjBPWOjjb39cP29dVOzvZMpAb5mo3A8kTARPjm/gj742/w9g6e5PXP4c74NGI09ntePvttXmzvITc295UztvdaaT76o84C94OoiPfx2nTnRciy8m4EhPMPc+b1Y5Ve8TM3kva1oobtZtGY9eQBGvaivhLxVQh69sRYEvaxUOj1F+vA9WBJHPfafk70pCrs9T4pkPQNWjjuKrha8XrZEPdFkbD7hTxW7x0ZlPT8eMj5Qow27jVhlvlKWqb6mSO89QZsKvsoA5DwcG549XLOTPBjBvr2wV3m8hWOWPtJncrsHo7u9DpNlvSmq97xOppg8vgHhPJ2bkDzKsHE5mU74PJqf471zu3K9OkkRvnUhdr3GdOO8QIpyPTx4Lj1upwI9loldPQtWdjwtzF+8GrrRvXzUPb3wFzq+cvhyPZvzObzWDgG+Au4kvfgoybu7vhw91oSNuYJltL31F1O9W6D4vRsflDw0aVU9o7JmvEwC270S2k27cLlou1t88b0tPUi9DyF4PUGi073tyES9K17avHqJM715HCy+9yLDPbtkfztKmMi8VqkQvYKfiL2b4R0811pfvXF6Ir3DnUG7H7vPvDy8rL3GSI68+Y+RvH0iwb1M+bu9kiYvva6SET6aA5i9","YRAgPqGx3Lwr+g284jVbPMZmx7z0WH681Ev9PUkpOD1If2c7pUEcPqEpHDw7W8g9O+ocPenK7bxCsI89Pq0RPRN8sbwdJB++fC/BPSE7l71jDs29q+KwPZ7AorxcqyK8pXIKPYRTWzz99/S85GuvPUQT4L04Xj28/U/rPby+9D2ROc48SHtCPrNkgL1FFqs9lzHfPU26hTyUldw9WiGyvcK15jrceau9/j5DvseEAj5+76O9JVaIPWkXujxm4Em81k4tvYNJ7r26zVm9QGMfvgAnuD0WriE+AFbnPQLuFj1cKnM8fs1TPdWP9D0o5jw9wIyCPTLLN73DrmM82iKQvcWgrjzg1f09Qlgbvu2razx3xwo+V/jcPXMvSL1YAuI9Oa2SPa2IJz5hWeA8Hq91PmKDaj31e489TkAkPkj2g72nsP89XpwjvUjL2D3QZcG8RuVbPfYbEjwviS2+O1QdPv0hFL7KukO+UyG1PY6m5L0rDZO8yWFVvKnWHD4ZuBy9rtACvpWSQT1+Peg8CosdPjmsNT71kOe9RaB1vZ6Qar392aO9odUKviDTObyBtoA9ybjtPW72K76qHls9LAF9vuSMpjs+CYE9KnTTvWXsJ75N/Mk9CAEDPu6wU72cgO06X+CAvYJGxD22aje89nwHPRsiBD5ZSDO+QlfJPBt8+LrXbLe8PfySPeeGurzT6Qa+demRPuo3EL0L/KY8m3CNPiE7Yzsx5Fs+AiJvvPbZizswiEI9jiijPX2RiD3NjRE9RSGoveeY/j1IkrA9BiyqPM3Qv71QefG9xm3MvQPiKr5X+5G+BStYPdVfRD64Jpm9vf+Zvsj8tD0yZ0W9DxepPA7D67qyUpw9fN++vDOlML1L6VG9adJkvre13z0Vvhu+vgaZPcyYND74Nbi9TNJHO2T//r2hZTe9buhEvc7rJ74j94G8yQAYviFGRr4U7kw9ZdeVPXu6DD3hZDK+lWYWvsmOQb1whLY9RWCAPQfi170hSjw9ZsJhvgVlfL5rPYU8","bPHlObiCED1lLQc+6oZFvVnCE77Qtj883C2uPSWTsD0ldCi9ijiCO4AfBzwYEZ29UIYcvmwE5T1kLCu96PeYPdFb4ry2cxY+sX/rvZECgT3ITN29dzDRvFFCuT0hrCq+gU6Avd4H6LyIBCW+RWkEPsG/1T16A5M9TeYwvXe2bLx4CH+9paK0vYl42L0nEr09+rLkOmScID05znQ8tg/JPe2Doj0nevs9ljKNuu7Vyj0e2z09sERJPUi/6D2x6yk+00sgPjDE2T2iu/M9rkGbPcFf4TyMc+G9LNbvvRTOIb1rhfy936MyvZ8ELD6H8QM+fP4Yu9LgND3YPzQ9HY85u2msVb1PiGY9MR7GvaqCHj0Y8Ok94ZfQvGxEdj6DCc895iMtvUzrwz2jR2U+V4ozPiYLgT3R7g09kuWMPoNuAr3ZbJ0+wT3UO8J/8zvkRl89x8GEunNiQjzbma296vtJvRhRSLtCbZ89GRP3OoCM7LsPP5i6MP9mPLAIYz6pGaU96k3IvvA2f77PLno+MehFu24Cn70UoVc9FXq1vHdNtD1EbKE9ZGf1PZ2SbD5qjnA8wzpNvXiFxDzUdK49MJvzPS+lHbygnoG9EsT5PGslPT73pky+/zd8PSbxE75jSwW90mx7PBTouj2PVgY96R1dPNBWyb223go+y5ccvNUXlz0CisU94BxSvL+gIL5ehYE8Y5OJu6UHQLx9gqo9lo3iPEZ2oT2GoFs+UzUNPmCZGz0oDxy8rSnwvZ/R6z34Mjc8L9UyvBxHtzySyMo9TugRPhSYFz1yF+c9RVoxvl2eMj4YZTS58SckPdwV2TzDfqw7kCMTPjnour0Tz4E9RHssOnjhAT5PP308anlYPSF/9z1x+CC9n0xMvrYeqj1jS4e8bZatvbWZ7D2oWoM59EFIvdGX3jvBYay9xt8JvOKYjr0beK+9z6obvSXBjDzRGXC9EFRBPhrObjwFhRw+V/V/PXoptLyCx1U95B2WPBGLDD5+WjM+utXbvSwLXr7k2y0+","1l24vcFXDT0QbSS+hM/uPLkJWz2KcDy9Mn5bvfdr6z245ZE9vZ0lPlBzRLyb/708CTUvPAZfaDy6uJg8Rz+QvVE9e70d7RK9drT/PX+cFT1BJo08cLR6PI6yD77b8To9GSZHPZ/egbzrep49GopdvXbRm71e9a29KL9DPeDF4Dw4chY+cjA5vafSdj3YOGc9f4Byu0mTwb1V1qA63vezvZAh/73PrLu7GQzuvJS/tb3p7SI8OFOLvW4Uvj0JZay9d5kSvkx2Z73jQFO+vZHBvJUjvLyRn0k9Z/qFvcg5wb1gfmW9YvypvFelYL0o9Nq7ax6pvfwp0r1/kkO9dkgYvAvZZLyD/GS8RUUTvja07b2aBAY+RV+VvblsEj5bNuY9BSk2Po35aT0qsO09fqfGvCbRrT0jAWW9+DswPurzVD0vDGM+4QETPUZxJD4ZY7g8l19YPX+OBb0wNky+m52UvT+rDTiIBEc9jLDUPUSYhbtRPRS+9WgNPg5AcD1ZV9Q9Wn8tvhYTAr7Q86k9ry9LvX5ULrzrx+O8Fb7dPQTjNTyWnAm8LY2aPXTJHjudsIO9vZ14vm0/mz2NKqk9cEZXvViki73Hh3a++QgWvqB2Dj5Ccha+IXXRvH1EbL44GES9hjQuPSsWRD4ZTBE+b1rSvRohFjxGW6a9ocA8vtODVbyy4RI+I8d+vgx96jzERdq90J25PAN9mL1eizi9ILOSvYKuAj5Q34m9Ya++vY6jPD6vz1S99TCfPHfSrb0CajC+1yyevGOQgz1YCwU9cckaPkyz1z35T0C+qK4sPcnQ5r3XaKU9aDsOPhMF6ryMNs46oJ+8PWr2ujxOiei7X9Y9vgedhT3+0qI90wo3PuhzYLyb7SU+NDurPD42N73ltRe9sghmPZGztb1uVPm82KBwvf/M7j15Eug8u95jvvnbuj0hRC8+BhCCPP7RxD3urMo9k5l4PWBsMj4/kUE9RKHqPWF6xrz9cY69vP3evFrqwT2zG849Ws3gu2qubz1ugJe+","/pnvPe49FT0xmtW8fY98vrBOgj4YDSc8K0/oPetFYz6NJJq9jsHIPXc5Bb6FuFM9PuQYPo5N472Z21k9/DHTPcgGgr0/spA972LcPIFMqDzwiOW8RVPmPD0nFjwHPDM9/kzSPIB/zTwSY4Y8kmatPDpP0z3HeBg9FOMJPWeR/L31A+09kTwePpuR0bzXqzU+JIr4PXVcyrzTMZE9G4GevVFo9r0wceS9oZ2EPeFRaL3lj4O9BNQ+vkCqm7skkd29ahnwvc242T2RQ8y8lE0UPbFAQT0b9bw9Wc8NPYgVu71ke8e82ESXPFQawD0Wlz2+CTL3PIYqcL6X36O+8Nn4vaOwFT4idwu+f337Pbc1h77+ci++Uz+avj4+hj4o/Vc+Qa8aPpfGmroQr1y+oTNhPnvAarr27F+7L2otPpUgIjzcrHw9sgoZvhz8072QK2m9s/asvRKRsrwV3FI+s7olvlx1VL2ZlSE8ZgU1PrugET57Ze68eMmKPnLqRr6fWi29zyGePBxkFD4Hf108L2wgvmXmSD6rmUE9ib4bPfX8JL0ZDbc8VVEjvqF8Bj38cO28x80rPAKl3j1U7mS8kR7ZO36G6jwQpB29jlimvTbGrL2+fAY+lRDuPeK5DL5+O9Y9e6QmPq90vLy2QSq9MD0cvvFevr5Oqbq9qPSPvfJjZb0Mppi6ZPw4PaBsrz20c6q9U7OSvrIL3Lxicx++BlQSvnxCML2FaFq+ZAmNvTH93b2Tw1E9BkfuPSoA1T254nq8sed1Pfjj9ryIsD++HE+WvRmRdL2zN9O9tZpFPcYMQrz7Fo08J8LSvcukzr3jwjM6BMkLPt4dMb6beCq+RoEPvtUrkDxILSI9PbnRvaSFYD1Ibn894ZYsPjST9Txh8us9CWkvPmfa4r2ZxK08oG52PeQbfT1qKFs7AzmguwiHHj17EWU+v58CPfhwZD2aCgC+08uPu7mOLb6Tr8e9C/CAPfHxYT0aNle+KXooPTvCIz3tutW9YrJIvYk6Jz7Artu9","oY0vPc65Vb6cSKO9twPXveWue77x4za7TzcGvIWx1z2Q6CY8+lf8u9uB/Lx4ORu954iePcZrhL008JI9iVX0PbDxCT0IErC+pc2BPIvuprysTv075VakPZx7qD2ZmBS9uC1RPng2fT6/opc9FdHmPTmMdL1h0Xk8rrgmvkr/k77aVpm9ajoHPvuArbxhgnc9RrcfPRD+WD2qnic+LC+VPQvXEjxE0/c71vZwPAoaPj0R/HU+b9WvvCrauL7oUCA7uRPsO0n0tztE2sE94hKKPM1HKL59O629Fw0kvKIwJ70yopA8jnWKPWEVMr5/UP88QmuWvdzUoLz/Cpa8mzXuPUM8gz2rGqO+zoABvnhLsT2Rjje+Miu9Pbk0pzqL16s7euipPRmcuT0TOw69ssldPYjXij084aq9BOB9PFXSMj7Giaw9H/+WviNj2DwzoTy9bPz2vEv9mz2nAjS9ugzCPS+ZJb2hItC9ex9tPQwxeT1Go+27YkvsuwhkMLwCcZk9LzifvOz7XTyG8dY7QbyzvYCdHj11d0+9jW7nvDNkhz1C27A9+m98vrBCez2IFbI9QtMCPP2qEjwe/Q2+FW0LPhtKBL5SOqM9nlDNPY6BiD3f3ZK85nDZPZGmar0J6Ze8FUkHPqkf0T3OUCq9PAHMvd5Mpr3xhQu+WT/TvCLjBr6k6Tg9b95uvpax8z2Sazu9wfeePNsU2zwc55u6SPEePluRID2M4w6+9OxZvYihhT3ThaW82JeWvKBYRT0A47M9e6t/PLzU3b0IErW9at6LPXeXNz4twaG98xS7PZl2UT2AxeW8whiVPTlNDD197tc9eKNouj1sEj3wTwW9jLinvJzcVT3g8Q0+SfCKPbNRFz3kDaW8lJBdOuPkcz2DG229jWBOPvWWh76VmB69sZOOOtkThj2A8ik9BD98vtojpDzTOGY5gDYjPcMngjyFdC494BJMPWdKYb1N5tm9uQExvRkUr7tUeue9jQdtPX6BSb0Zlz+9fLo/vSK1lb3kYVy+","MMzjvfvh8D2rpV281qkyvbkEr7z+AOS7Vc6xvWERJ72/k6i90VHxvRaXbj3IeMG71Yakvbo9Nzx3Fqo9TXxfvHLnIj45pP+9QuYrPJaCI702f329MC8FvoEKB73vRTA8IG2hPOnsAb20O8e9zXT7vc+drrueDpy9VCd1vUu+B75k+Ka9SvzSvDsL6j1zrvy8HygcvrOelD2UcoK+hyQsPRYy9z2MAyY9xX6BPSC1lLtoOga9a0uHvFo5pL17YLO8FT2WPd+H0zxLvum8g3ovvlRyKr2k+Y87U8LSvUOMhLzjdIO9uowRPGs9bb0xf0i8EE/DvXJ2670LFby9vgRVvUIIuD16Kpm+v7i9vFmRljwgEUe8RdtIPD3sc706PxO+28Y7uuLaDb7TO828tizWPDZ2zbqWCtQ804vDvFFgOz2hrcW8YPdMvtbcOTzvOsM9F82BPVE+ETzY8eS8sAdtPTkPQ76ZVLg9dpL5PBH65D31eds8Zz2SvXf/T7ycIJe95A0ePcX9r7wn1RQ9cr1Evm0sSr0aJgq75dYQPrJ3bj3axLY9GZRsvsJwrLx8EVS9Qw3OPSRfZT3kLgC7v2PxPUPMsr2s4Ac977UAOzRgFrxRtP28nZNePY1uxD2KoH69wkATPvcMob33GwC+bmRwvb8pvb256hc9T+okvdZMBL7fesg9FMLyudn/FD6ZtUU9aNjDvYgVir1PrGi+fGHgvca/sbqVWYC+2HRBPapfP72jBnG8hNonPj7Xtj1w5qa9CTSjvKpGgz3DYIS+e4i+PaXDmz0A8gI9o0xaPSmVOL5nFFW9C6gOvZ3FXb4cZm09GiWCPLEKCL5Q9Kq93XmDvauwpDxAnDo8iIEivCEdzDxhXl29oxo4PjyPHb0QRY28BDf2PXQjD77flCe9xqrdvBQcEDxSrKg8VdtpvuIlBT1HRQU+lKWaPWyDzj0fIfq6R+cavRYAprmrVZs9R1LIPQFsUz3A+kK+rGycvbwVMDwJb9084lftvUC4JD7HQaq9","nSVQPQL1071v89a9Nq5DvjZOmD0d/Z48y+nJPX4isD6EHlO+FcNQvi2MZr7IL4G+OYVLPugk2j3SU3m9EsfAvQRJrjw8kwA++CvNN5G9kr5Wgb28oIszPvNrRz61efq9nlWYPuyzDj6Jyj++Je+HPAuJmz3CcJa9E0UvPq/z275+Pn28HYDKPeEchL3hrwc+OFk0vstZ1D22bKI8ZbxiPGpMSL3W8Su86oUuPVbfPjvsExO+WLsOPO1ecT6rolA8QQgzPPZJGDxgjb+8h08DPYJVhz0dk5O9aLyXvTVPgr7YkZu9nYHvvIs4W7682Lm9UKdWPawmx735Nae9e/LkPHpUKT7lJ7g9wVcfPaPti71wSQ29y7IQPRAxeDwKPMI9OgAFPVDDsz3NbV690CnVvWCM5zuC0mC9onC3vUSOb7vdz/G9AbbaPfKZgb0TeNo7HKu2vsyIxz2ahn4937S8PW8nIz5hRxS9O51KPvL1N72Tmxk+tFujPZXavb1Z6S+9fyfDPbmTtz1T2Bu9deMZPdZAUT4vpqu9jNnMPaueyDxGgr29CyAYPZ1fhr2uaPw9VyFkvbiRuD2uC7k9jMn7ugzwkDyCcKa9CnDPPTYm0bwNUws77gFZPQY1Bj5dPwE+rLC5vUbTjj3YOCE9F8gbPRDjHz5F6bO8+KpDvYjp/DzA4DU92d2oPS+PID0nFgA+JzQVvfPt5r3i10M+fUIhPNtUmj19EKW9PVh8PsTjQLxiTQE+8fJ2Pb12ZD7urCG9cMe1PsL/Zj0katq98kCPPU8kRz2JG3C9nO0GPbIfarzZTiK9PvNUvRKNzTtVsSI+81yzvcbXqTyNom093ADzPatBdr6Upoq+gvQDPmlToz00fTA9CVvkPX2NUD0eTyI9DhLpPA9A0zyLrpU+WqxaPXyla74QlpM9/O4zu3rdqj13FAa9BydBPEj6iT2P7mE9oHskvk+o4j3cwCm9sUadPOHMh71kLWq9zkUmvb8SFD7ONA29aP2jPQGvsz20PUi9","f94vPvXkLr67u3S9u/xHvZ56PD0dNuC9g20mPYxkv7v5ypc9hqIdPqvzvr1xWvc9uIzKPce6Ir6P3DM+zANwPGZeIT3vfu6+NBMcPlJV6zw/U/M8TEn8PXfSvrxVMh87tM+APS2VsDwItXu8dcfePPCFAD0nYvw8+EVQPVAiR7x2emE8gn2dPXn0eTtGEgE+d2IEPRz+Br780SI+p1noPRTNH7x/+mS9X4GQvG6fWD1OnT09MkfGvXV+Zr6N0Uq9Xq0OPUw3Oj2Jknm90c+wvVbAOL0Plmy8THguPSwoIL2jGmg9t0AbPtfPFr1mUAs9rxI6PXCSez2uzPK9amXkPeQjmbsLfJ69QgCVvd2eEb4FkQS9+oRyPWMrer2koRU+jGEGPoB7HD4CN0K90DzRusJ56L3lRAm+7mGbvRlH3j0i8aW9bdnbvW4nlT0IK4e9/a9JvccdCj73wHe+pGQuPrbzgDuYjQk9Ejp3PZqcMb0tG0i+xMcyPSKWML6LnF47jS0UPrBd1z1W6rw7V8/LPWDpBj7tJlW9OCODPmYSi71Nm+480aYXPY3Qa769rA+9+9bhPWVEQL7sWMa9gMcMvvN3F7y4Oda92DMAPle+Jr6vPlM9LQ5wPTRNjT0xvsm9CwKYPbHkIb11xIq90hB7PZ6K0D3ATMW9IO7IPTrxnDxw94i8KHBQPSJA/T0UEgO+dPvZPL/jIz0BQpK90rcGvZJKWrwba869YJPsvfNZq71S5am9Y90XPXg26b3Af5S9hOWjvT7asDwudye+ojEqPaI5470oWyc9MbPOPUpBxL1wima7HAOxvHX6DL6r3jo8VHs6PqGq370e2w++ba6Eu3VGa73AgNI7sxkivi/+wbpWDIU95vwzPtAigzwwkpE89S53PUZifj2/pL6912KEPeiKiTz/Kzw9oICKvBd/4rvluiQ+vreVPfyVZL031WE8/KXeO8sAAr4xlTq951WcPX0QRb02NSS+0vQiPdwUGD4NIwM9tdCEvGnZLz52/746","05b9OyqQAb7NAEU9hAcGvY6FqT2zcLY93S/HPYZmZj0znFy9kHTlvQJb0712Lom9JYkIPXqBy7yxlN+94D+7vDHmHL68Z/+9AXwLvva9Hz10aPY9Uh+EvQgLdj15GaW9ubwCvXJlyT0lFbm9K6AnO8hWRDyKPVO89zUdvoLvCD1je6E9abXrPfr/8jzNJFI+1PhjvemiEzzXeEM+k1MOPXIG4T0F4Ie9LdqEPUxNiz0KjUw9Y4ZuO1kcGL1wYS29MSX0u14wOz5SfNS9CEj0PYrfwj1fA3S+IcijPCTxiz1ymfI7GNdjvpwu1r2SvfI9iKpnvVTzbDt5YcE87TblPaxIeTtRXo09gbVtvhOQVL13hb49VFQCvN9ru718a4w9JXsnPtQXKz5wp7+8kw4IPrWS1T095u29hPsGvXMO1TormJO95dv4PL5gDz4K2DI9chTtPQ8OrjzdcRa+U8GyPfX0Trs1W1A9PHmqPXFlezpwtFy+oLatvHN0ETxst2y9sFC8PaY0LbsDmQK+UOJcPcBIoj0kH1y+aiKnun2DD74oSTK9abCFO8BbwL1+7729h29WPTdeLr6nhQU90E8cu3D8M75bmwO932tCvrN2Ib1RjRy9/NaEPV2DTT2jrpe9E86ovXWMeD3mRew8fhefvbTw+jyFNdO9MWqEvh6cvzzB0Xw9BVPmPOssDz56cFi9qn0ZvkxPbr0uBJ++Ni1IvhnMer3RfZC9JJQbvgdZNryuubW9MuUGPs9gED1IKhG+TAatveBcJr1MnT69JWN1vQaSursfG9e9lP5bu31alL3FydG85Yc+PZ+T9b2rI4I9v771PZReX75iW569KEebviIL570yTFg8TBpYvmoT/LtMpAa+7pxKPvCSlDuTNZi8tiLWPWIzH704Ju+9JmmCvQYVy70Zhq28WFlbvUjEpLt4Pwg+dDEDvarC7bz4VQG93jqCvDxKBL02PCa9vIq6PeXWjD11lA++1RHDvGASlTyn97Y8lNZjvbuZ/j1otty9","NX2OvO+XBL0VVJ47DamdPZj6MD05tjC9CmsePAOYizx0c6s7FZjLvbts5T1DTKc9Yo2hPUkWjD1rdRA+iRPhuupGlD5YoYc8bJW8vZakFzy7/Nc8QTWSvR7ktj2Bun29j0YbPOFjlr00nS09SsQ9PUUXYz2eSvm7nIwqvVTjN71h/TU9lSm+vZoWEz4SQ+08AmMKvdvYQD7O8AO+NdVxvbg49D0pB3s9z5iGPuFbmbykc469Wm4RvYUS3jxMa+A9OAbmPHcNBzzcMbE9Ji0xvN5Qlj1qxBg66Y9zvYTItDzkmki9wmrEvEwMsT3Qw0Q9UOhCvr7BlzyKVHs99c7bvXl8wL3jAKm9MJ8IPaG4RT30Lmg9lThevR4B6j2Tt3S9MrtWvjghlT3ObZi9v4vePAd00j1JjQS8MSpwPCTTRL09tSC9H0EBvjJXKT1tjjU8E30MPtKPMb5N054994HKPSFFQr2uS9k9kZ6ZvN3wjLz2p6s9SY2FPXGgt70VST++DHVSPJcVCj65pNW8X1P9PTV04jsCQUY9gOvGvVvuiL2eLu+8zE8DvlIl+zzZGoU7UGAJPae9KL59rP29Et03vvp62Txdy5I8J0YLPIANrjzO5JY9hxQ0PUgpHb1bMsu9jrgCvcVNM73+l8k9ZMAevfr6B77IRlO+FIEfvivfU72jqRI+xKl/vk6wqjydV0s9X2kEvXTdtTsl2KS9Mc7mPL/ihDyXex29luUvPed+rb0E8c09gKAPvJicdbwPDmU9g7AwPSM7Ar2wmiy+IyffPbHueD22b0Q9gO3lPVSXer0TJXE91ulVvuQNfr2AEno9VJuiPfvyi73shKu9Ne2rvRm/jjtJIce6x/pTPS7xFb0FNWA9BniwPR26Ez6vPzE+5/0bPQ+wkb4y67m9tV/3O2so1b1mGxU9s9Y5vr/yrjzPh4M8mFcpPvK8mT290809KCGKu3k6t73ZbjM+Big4PrQPWz66tfS7YVXPvOgpCz5kh0o9AoAwvmY5yD2RKaC9","SkSlPeNF5r2Hpp+9IKH4Pda8WD3XwWq6QYs4vn7wvD1Z1CY9fWWYPNOzlb3gc5o9n7sHPBzbpz1IcQO+dOWBvaoUMr2IIfG8i+OYPSVRzb0JMhA9HlKqPd2VNrua2AE+Ufs3O7Rlzj1ly5E9wbO/PeyZRL0Z8CC9OqW8PAMo2j1+CgY+9/dXvbx/nTxUNKu9cXIDPX7Yl71yFZg8LAHtvfjD67wLYQ++9jRKO+3xqr2XOZC9mjkHvV0Hh739x6y8yTgBvo0ZnT3k7L88ABTXPelvSb2870M9eIM7PgYGTD0+hrg9QuolPYur9b0iRjQ+t9lSPDi1xj3NdWe9C5B6PRxvWb2KTis+yFUBvgulhb3lKq88DvcCvXh4cL2WryA9d5EJvknSMz0x2DG8ggoBvdqPg73VCrG643RkvX7bpj3XeCa8UqqjPYHYoz3Yxno+cdpgvcb117zTLYa9JQQCPri5rb1S8GU9ycOKPehKj73WF6E9841PvbeZJz0Hgwi+svBpvcs7xL1u/QM9ECQ1PoRdab1Jg508oDguPYovIT76y5q8TS0RPjWC+jvg7LU9rcTpPezvv70Bbm89O/LcPRt0mTyVKSi9X/gDPnNDEb6vdEI9yJ5jvblttD1BZ9q9t8jhvSFlsz2DYo49ws/9Pdk3Bby0rNo8MtwkvRX22j3JVDc9Z1XqOwhK8z1s0Tu9GOROPKeC6r0Sa5o9/zXDPN7D+7xIsPM8ieT7vVEWUj1qvM+8IbwxPaBN4r2tmX29TSeVvevvvjwgk/u8g3LwPeQkYj0sB048j+btPBfq2by/ncK8DYD6PTinDjumLz67PtwlPlv5RD4BU/k8ZABDvVsnNb1W3A4+aT8PPHdcPT710ck9TtGaPTobGT5uXPs9hR33Ozt4Lz42sVC9CgKevedTGzwUkJC9mkZOPf5O3D2nX3I94fVuvL3xCT4/hM29aLM1Pt2UVDz4Emk9/5cFPXVZHr6fKrY8pDWWPbDkIj7ysqa9UF4nPmuZaD0ux7Y9","o10vPQWzP73/Dj09Q77jPDNqLTyKiju9irKCPP+h3r1x5cA9YAJAvpLyFTxL7Xw9HM6+PA1E8z2+26A9+sfBOzbbAD4EwIU9btVevTsKnDwQu9k7bsJ3PdxJhD1KwLI82g3iOt7Uyjy4tgi9s5YaPqNBvj2Gni09qi4cvcwx67vOEJ88R5aevAENbT02vUE7uc8uPd3zuj37NY+8QYiyvRCGWT69j5G9SDrpPYBxcT2pQrQ90d6ovV+der17l0S9grSBPcUDlDv09B49IyxsvJLOsj3HeDG9im6SPfIAmr2Y0bI6N6Fpu9LZYb0UREA9IeKkvRa3ab2rU8o9EQwdvmjYKD1+Ju67zMoJvlU+pT2dd8U9adqkvfkp0L0eNzq8t6fDum6ZBT4mtP07rfulPXxpsDzRj3K9R4DDvA8raz2skjW9NGbsvRwm2z02dVK97kgUPkszv7uOh2292KHCPbtTrb1XCW89u7COPYwk2zzsnT++X92ovHLCx7092oe8Er6uu654ljkt/D29QVFmPSe2NT01Hz++ie4dPXU8hb0ObJy9g5/Wvb5GA778S6G8P55BPIIauL0l2xa9pR8yvaxbg72A5qk8bqXhvbwUKD12Ggy9Rk/zPbX2oT1WgFS+49HZvdHZ5T22Z1Q9WONBvQCxjT01+Aq9pncMvnaZRDxOV4++J4UTPs/NsLv07ra9vDoJPkyxsD0QAA+9drffPYVa1r3cmjs9CsNbPl9kTr7hr1+8h3dEviPXdT2GoGU991Y0PratvT31+R690ZYzvQ3yRL4SJ0E8gvPNvbvsvz3m+QY+AtiPPSzkwj2M3eM8mvj9PMgjabykIFk9RiC0PfI68j0eDQm+aQgkPsRjJT5zp6u+FPfevbGvwT17/ck9TfOlPPrXBj4xTLE91KEhPcPK/z0/fFQ8XDBnPlk0BD3Uqq09IlhjPPWMYr2/hQM+HccVvufK+TsaZqi8bKTMPIOHYb6Y3i28VRNNPfYgrTznnsg9mA3jPf2k5LwRE14+","gHm9PUTZhj3q3O09C82WPCctdjw5M5e9j0AHvaOunryzJt89XiO/vaDbubob2Q6+FuiWvZ+Ymz111VI9K7qDvWKeAr2X/QC9HdxJvWilHr6ywCw9rEvWPfsyCj60N/A8Xn5Yu7NgK72VvGi9l1ULPsvVWb32GD+8rixSvTGEHzxiaC6+nUSxu55p7L0d4pe+JYHBPPTpBjrOdB0+uQlFvram6ryc7hU8n6SLvRnoBblQaZu9k+IxPo1wCT5rgrU9i/MdvtiYYD0moAo9yhPuPWFsAr4yWkS9/qNHvCJ9yD3eRDo+ULWcvaY8A70JcDA9RL/QPKwRND5c4tE9TJXgvSa/Pj0KQYM9sMHQvSPBLbzaUic+eK0HPt1FMb2vg5Y9l8SAPBdS+j2zhX287uwZPjsf/D2eZw+++irxvWGA8T2AIwy+AKPfPQKgWD58XR49Q/tWPt7udj1h1K+9SUQNPhLlFb6XpEU9Yr7ZPSZ8pLyCMxa9VpesvVD1mb18F6q6XFofPhDL4z3stP+97n2tPUm6yT2kMIG+xcujPTptwb3cs5a9h0o1vA42E77Ndio9ppkmPUQ8t7zEavk8F1f8vK7dq7vh9OI9v9+3upYoK73h8QA90sMZPlQXCD4wZgO+dy4kPUqIqD0m8D+9VrW+vBybCTwLPAO+vMnrvYqIrj3v4CK8+WeSu1WDQr7mmXc8RmgNO3t+kj16qgW+kBbQOAHcMj0fuCQ+eVfQvKmrBT6IW0c+w0oLvjgPG76M05C8rEA7vhbTIb7o31E+LzxkvbPjvj1ILYk9RSVyPWf34DwlKiO+Sm7XPalkNz3Cb5c9xE5AvhUqiD1IO4K8a9YBPeLlxj0fHve94fkgvXxg4L3wtJa9R9czvn8PiTsCZ9W9agMLPjG1q70YbYK96lWUPO5JxD12VQs8YW3Yu4OlP7wgVws9Xh8nO0puJr4jwo48f55lvrWwAD4Bn+49F/jrvRtpCj5i1tA9bmgzvVFBiTwzW2G9bqTnPEXjvr3T2Uk6","J1CYvmWOtD3Td5u9MzGPvb4oKj6ktYU8rUQcvRE42D1VVWK9Bvx9PZ3YRD64BLe5Z/XhPYVLur1AsZo9eQmFvv5ryT07nS28xh3cPWnatD1jQko9aNdpPVVFk73EJXk9XogHvIh2xD1Q2sc9EKSvvbmpMb2w2gI8RIHKPSMVsz0DkWM9hluAvaavsT1bfY89Z1FqvlMB2bwgRcg9lYIwvrSjc71cUCQ9F+5FvW9/kb67iCY9URUovQeuDT7sCAS9+LhMvXvUKL31PMu90RcXvFWmpry3yqU94wUoPR5Po738RbE8yWy3PD5V271BaRq9ppgBvJVgjLsDvX+9nSGVPcTX5z18Dxc9QvUDvVGFJjzCOZC9m+i2PfkwAL6ZYSM9ZNT8PWozUTy9qYk9o3sJvnZJ3r2CcrQ8UddGvtvIcbxsohC+eY3rPCJLCb67/NM9YqiXvm5xtj3Yvya7TY/4Pcj0Ir7i1E+9Ma2JPZvHDD3V1hw+xyUivv8pRL634aU89fANPj/LMjwl3PC8TRUZPe3W7j3Pwbk8OAF7PVfGRj5N1q89ZkGvPZ1N6r3Mchk+aA0HPQhl2j0FY1+9ruIlPqo51LtqZoK90KdAPgbBKL7AsK09FBdAvdnvRj71hBA+YqbgvOt0lTxEQI674xchPsYp/zyMvQo9cuLWPDSDdz135xI+P36GvIq2rz1u+so9IFM2voB/1LtLz5E98YwTPpLiST7/LVU97ctovSDEH71jelS9ROzRPY+Zor1aTAI9uqyWvfIvfTwXNIA9+3CSvlvQYb4Dgp0997llPQq8Tj0R4TY+kpOROyJhfj0IBMm7IV+gvZStmb3J1Mq8zn2vu22GGj6VMu49Wq9pvR2IjL2M9xc+kI4fvNzlZj6dDoO9q3gUvhHKFb1f7dC9kggTPsAlqb1qBRo+Cge/PY3jjD7XujO+dEs0vsYAXj1wjKw9K5blvbfaYL3vcVO89jXBPThq8T0gSt48gOqtPdgUDb2kJE4+fluCPbh62T2D2hO8","7eEbPdJMBL0lSbM9h+J7PGpWOb1cQTq9hJiNvTMMXL0M74w94HlyvvtYdb0OE4K9EAChvSlGJD2wvW29hb9hvSd0n7zfui0+5pRsvmsQsbx+MMA9VB8gvtIBBT4885+9Jy2jvSvhDbwjo0i+iHjWPcDHJT67/g++lipevfk2R75bh+68ppLRPCtolTy5Mi49YdESvkHpVj6YCoE9zKeTvUWeNT5BkFw8mn1MPHv1Mj2wfs09L+cfPWHd4b2gpfw8DRT1PY3LAD6vRX471uc2Pakymj1XayK9wadvPEShAbyYpoA8cKc1vil1SL0bxgY+Pl/XvXDyWL3kBjU9jYf6vUJlWj7XFQW+ZckOPvcSZz3grBS+j2sxPQxlWT3Bh+O9JW7iPcKpi7ww/O89ExHivVXKvj0GMHw9HdyCvZ39Vj3tOb09CslLvQkA3L7slSI9iCJ8PGJ6BbyrJWQ9fRmrvUFxrLw1FlG9Xd2FvcyHXbtVS1w9kRvhvXpnLT2ZYhy9CjNtPpHxBD6R+7g9sm8lPVMGHT5VLB29dPo2PVzZzD00U1K7E9Vavm51lD0HkP49hIgMvhk/jj3+ZZe9r8wXPr5A37wdIV69q3/pPURunTxbhAa95wO6vVZxDz4HG44+A/slPpenS77Itl481KkqPt/Lob0lDuy9j5FRPrkl9b2h4BU85jbVPHih9T2eTUU9UDpZvR5Cnj1XQ929bChPvZ+Ldbssq/O9vIluPZfXvr24TpW9jiqPvJb6pj3Zbpi9nuxKPrB20T01Cf29/YAPPuvrgT2YO5O9uxWfPTeABT15B5c8pMmpvdHLFL1NcV+9US4cPiH/RL2mS3G9Qkn+Pd6HiTprIjG9RiuoPGKnHz0RfA89QLcPPoTl4zyo8jA98qFbPRNDJr3BRZw+Hpa7vJg+vr01rAG9RjYGPYEPH72JI4E9ipvPPKfJMr3ShQe+VLGDPd/2Gzz40bO9vDHTvBAn1b26EYC9Oc64vWt43zpT1yK+v+thvd1D+DyRajI7","ZectPuLdsbzCkZw9S6+PPAWRxr37Eyq9G0AEvoEeMr11fHo8e/ZnvpTlQb6iUP49B9JcPTvsbD6IXCu+ieotvgFwhL7PNVA9Ud0QPIEBBD1mQmA9EJAEvRDE1j3AdHK9Nw9JvbGbHbz4rCm+G0ABPhPS5j0rZgu++VIqvrnMGL6NIVC9xCslvaPKmb3RMWa+m68SvjR2sj3S83g8GWy/vcr/fz29wq29pYRyvozdCr0bJcK9kduUPKU88b1Yp8I9FtmkPb0TxzzmpX896xmjPSAqQLwxWfs9tFxLPewUAT0kiwI++7oivop6ur2Fv3U9JIEvPPxrdr2KGH49s8qevc6h6z1Q02w8g5grPu1DAr7t3TC+dbGDviyNsD3/Xrs8t5imvXSE7zxrj7S8ynAXPPMKF71boPM9TBXFO9I6lL2w4zE8NxshPtl3YL72Tea5Mtk0vesHcjzrrSA+gZ96vj3scz4FMvi9ptEevq5P8jukNoe8te5SPb+EOj0TeO+9/BCfvUcgCb2zlJu95vH1PaYkRj71VKM+clldvQOQNT4hzKw9qjiyPdOsGj1spNo9gIiqPVNEHz1vkB0+GSPRPQnS+z0auI874PgBPt9+Kb1BkTC9EdgJvauO8L07Sfm8ndeMOwNIojz7s/U9cMuiPGoaI74BGnE9wGEwPr1pXr1+cue92X5ru7k83L3nv7s9ry4NPhkmUD0WOlU97hx3PV5dVb05Cws+Rc4kPddiWjy18Ag92lOfPMxuGz5DQTU95HQ7PjYZRz173jM+JsYDvXIu0bu/eDU9jzsRvdEfGjzt4hK+gv8CPrF32D2Ycr69+mCkvfjlRj3Kw+U9MLtMPdu/4Trna2Q9K7mkPV9YRj03Wgs8zDoIvhJoG7yE+cg8V6+Mvfi7r70wtlA9S/xovYtgxTxpboG9nODcuxEb9bveayq96/QJPHW1CLvUUQ68THAUPClhlzx2hJI8qCq0uyqDmzzZPqU9/sOmO0qebryUi3E9WgyJveZbmr2oJzY9","Rl6dvCgKYj0pQF8+XHtlvH8BkL2/TFG+UNECPrB0/Dswe22+EX6nvYfG3r3q9xW+8hbLvPW05T1Rjoc9ia7WveAfYL1vNXw9pCNEvrWUu7yWjIg9lwqHvYyPgj5YI9+9XTvKvI3MR706Hge90OTTPexJhT2sqoc9H/K+vYZNAL7tbwO+HBgNvQBZgr184ok9JO2EPADZXT64JZk9Sc08vbKzuzyOTR4+yOiMvCXdeT18TqI9CkOwvPkYnD3FQ4A9O+PYPdMRHb2U2/M93cfTPX0sGL0V1QO8Gau9vSi+cLzlFIQ94s3HPPl11D0ioJg8o65LvkCsPz5YPss98aaTvQgdpzxwjEw99bPZvdoxpjwNkIU9Y9a1PR4Buz3VTws8SlkXOww5bD3W9hS9PJcMPZ7NNj2KQwO9OrK8PDie9D3Hpaw8zcZWvDc15LxrN907koIXPiLNiT0hXZK9ZMLXu9mVO75+Ki297QNrvRBDhz0pDwa9hzvxPABxOz2kRr+9+5MHPV+ma7wlM1Y91Tv1vAEznzycZAK+5GYcPoum/bv86kC9G6povsJcub1xklw7Nf0JO50/CD1W8Ja9V40TPeU4kDwcnek9eGnJPPqKqD2h9ki936PIPATlCT5EAxs7+jHHPcPIIz1q3pG93ESdvZ6coLxKZDu51L8avQ40tb0plZg87tfmPQ4DGD2rrCw9lyUSvqrsvL4d5TY9SfHZvHRdfbw8wGe9xjfOPUiH7b0xGVW8mVbIvbOEZz6taQA+fKaCPurNTL5nzDs7M/ZSPTXhCj3dGUQ8vUSGPL2U0j28mrE7BOotvQpRrTzMPsM9UNSfvU/OEr1ChHW9WWqnPSWetL52yno8mF/vPdRRZ73RvqI9G/B2PYsPx73RG6w90M2cvbUEgTsSa0s+S/QAPg20Mr0a8aC9CWIFvl+qSL3BEyy9DkP2O4tLED4ENu29oHubvNUCizwGPDq9goC2OwUeQr2xwM498twBvQ3r8T3ZXKO9nCvkO0e5ZD0aVMS9","vgmTveCU4r1hcRY9B0QrvLoOpD0WyeU9Gssgu4CCCr2py3+70aMFPd4HTb18/0c+RuHNvBdq77w39oa9Rb13vM01qr2Fz5O8yRqkPSV2Yz0IrkU+/SuQvRNV8zwB0qs9PDPcvKTUpj3BkbA8N53IPNPX0zz5eS8+slMivZBY+j0CXrQ9RM+tPGOHT7294yw9XxjePUyGir19xj0+8MsVvkF0Oz2EWAO+j5SNvTxVkb0i/y09Er+HvZthgL23AMk9XDTevPwJjL0Edqc9gCyZO+4VET6AdZ49CFPtPQ+cqjxeada9HvuQPUBoA7zj0hg+UjvMPAPCLrpNOl69+IifvLPQzr0BRL67i7SzPJOz5bynvRY+BWgtvcb21jzuKfk9nqDQuwC+vT2/Z4C9Cx5HPZ2PljtcaNe9aHrFvQIZrr2zzaK9tAmYvQ3faj27oIA9oBBXPvR25b2Htgo9DlOcve2+IL1iJyK83YESPp66arwrmWo9rzvluwhUCr7Bh3E9Usr2PSFA5T3Y6La9dWAlPQbbHT2xid48lhJIPTQRBL6cxZi8mSXpvS9UpL1MVKu92T6GveFjb7ysB9W9ddEWvhMsHDuykUS9d7KDvYeblr2xBpU98/VhPZifQb1s5O286FmRPKAfFb0AI6w9sh0KvXf8Sr0Ehii+vdYMvnFe+jyGldY9lsh+PE3EkL26Ap26JCniPBLsRL3Uk1y+2v6NPVRkID4raCE+UbRhvb+6Ej0dktE8XwQ/veJK+r2UPIk96n/pvd9WwT1WKyc9IF7Xvcy0j71zgQU+bjzvuw23Jz4YMUo8NzuFvF1siT7Z0gm9kKqwvfS7Ur4bZR29YrsfPgyk/D15sfq9NwQUvm4/QL2v74Y9FiQbvqvStj0lhdu9v4+TvZ7aP76kXw2+w5lOPZIJnTz/pbe8ZNW7PaUWnz2STVu+sS00vdHYIL4cxTK9vKlvvQ8niz6OTu49pictvnrj6j3AMSI8mpRiOqIrPL3v44Q9SFe0PMTH2L3yBkW9","BbEOviEu5ztdF6C9X92YvW8LAz53j4G8NzhoPF23ED5TchW+TXeuPWygcbzaJYm+LiQyvIxllb1OcBO9QKKKO5dlar34pc09tMgAvjH3Db6lS0S9KaW3vNvIw7yjYdw9LbOFvS9ydz3pCSg9XE7APJzxnDwrJpg8bQOWvfS2hz0fZau99G6WPZYJOz0e48o7y/K1vmfCuz1h0C8+e+fYvL8Zpj3DuRE+MU0pvvpF+j0OXsk8oOvYPeQjQz5mDis+kmetPdsHQDscEdQ9Z0kiPpwZBL5jxQm9QqqTu3LOZb2igkO+tTGAvZL6ozw/rQm9xgENPTkxbT4DYoY9uO1TPuP8/Dw++zy+OuF4vRjwxD2TSgI9oCUoPVvKTrsNuBm9AxmHvkiW4r0fuik+Xs+Cuw6RJL4FtYs9zF4IPSE/hb0t4Eg+yGCBvgQ+Zr3jRjS9HO+APq2uBz0rmgC+t/FIPcNSRbya4jw+TGyPPJFhIj4CZHk7GmaTOgnsRLxDFD89A759vSXUO74zmem6zdVyvq04oT0xEZs8PfH7PbOhlL6A1Y67aVDxvTw8/D2GUVs9Xl5JPURyJrxbWZ+9kaYVPUrJw7mZwy66GfKrvCVF6z0csu29fTEAvZMQMj4nJyQ8RSIVPqbHjL2h5k6+x1G5PTn/8L2/+ao92JmcPR3tg71cQna95Wikvgt3uT3MwN295z+ePHpWor1a7LC6+U3cvKiGib0s8Da9EmIFvkjGfL1CbZ28szviPNfNJ75+p4w9Gw3pvXciL742l+68YgGove+XYzwvbUW93/0SvD9vdL11Xcw9igaSvXmLL70is8o73p6ivAfvD77161a9XbogvgmZDT2PJ7w9t/JFveI4qztpHci9WcwkvVJcED5JBD69SfPAvSqtUTzOYYi9FEjbuvg+Sj0H35W9eaSMPQOJCbz2/t299RIrPUIO170H1ay919BdvRECu70IYJa9e6HLvZKIU7xMXRO9+TFovpf2Lz7rjV69LmUgPR47Hb3WcA89","QGcXvFCBFb4Tk7m8DwVHvnayOT0OR5S9SwFJPkl0lz7DtQy9Dfgjvg58Rb7u1j29uHzJu2AgAz6XVAe9iPtWPsO4qb0jfv+9yi4rvW1JwL0hiz28228pPQosNT30RlC+w89fPvOnED7BtqK9g4bgOnjflrzsqkE9FkcbvW4VSr4fTgW9a1mPPTYgbryjO4o+UnwFvg2K0jpREMQ+JRdBPixYgbw/5pw8OFMcvfAeeT7Qu6c9BgsBO8JNx7yqAUK+px9vPKvWOD6DXeQ8FK+OPP1fYLxIyg2+gGZovrNe873sKMg6S4DcvXZACD1vDDo8rXclvqSYhD1usL+6KT4kPeyTDT7HK3E9PDhwPcFYSrxD0i++EOyBvCX4CL4eHRS++h9bPCrnFzzg5ZM8GYPKvf0lNL3JCQk+ya2YvFn43Lw0oxI9ce1RPd4cHL4wIjA8X8RFvYqDyDr/q/W8cH2IvTsIAr12cjO9ikyOvQe65z3R58M9Wa8QvsTmbr043fk74m7iPEPdlz1wMJi8Kr4dPQPiSD2nuLM95rTXPdfqrD09Ips8XUHEPfi5KD5OSmI9v80EPRU9BT1xTQW9w9uJPYVc1j2lNeM9p/5EPiB2470FDms9/zABvr2ftjylpuI9KMTHPB/0SrxWQY09aHvMuqi/G74ppWO8fq20Pb4t6b0="],"bias":["q9uPPQPRdrvGPse93NStPbzsv728Nyk9T5otvaTAuDwVyAw+q4YAvZVWhDztgag8/ZJFPQaaf72ItuQ9SQUkPT3LyT0ChaW9EmiJvU5nAj4e7pk9O6pCPUzlcD1KVw0+OmRYvptxHj7yPzQ9N0+YPb8azT0MSJy9O7mNOdjHXb1P81c9LyGEPMsxzj1qVww8PTtQPKy09b3zmc09E6ARPpsUOz1Pwv88JTfgPa/8rz1NNOQ9booKPR+JYL2LQE+9iM0cuxyYOj59bhs+dVgZPpNUAj7/6YU9Q0JuPhsDLb0ItPA9PZP3vIk9Xb09AyM97bmSPYf+cTxvrCC9WMLwPQ=="]},"dense_6":{"weights":["13TXvZ9kmDxKIyO++k6VvFl8VL31s/C9wg+LvZ+D4L2v2su818IOvWcWST2sTFG96bcRPt9oPr2mNx+9iRfBPasRCj1TM+u6a41SPLhZEz4Ayh896EeOvccUCb2Aifc9CCu6vRYVB76uHYm7nl5RvCNVBz5Q4Ik9PuuOPKOatT0DnwU9BEl4vpTOIb0CkL09AWS8vJfQWr3VLxS8/y4RvlCZvL2gItg9ZHvqPKAMTj1lJ/e9efFMPmzevz2fxzA9gOdpvYyRDT6XIcA9u6ifvjy1dL3u64M8JQ1yPfNAL7xOa5O9QseCvYZkNr0GEMk9aboZvLGPLT1kahm+JhzlPUF7PL2FNAY837NEPRAurbzJ3ak9QFNKvZUuUb0E83m7Hj+hPHSo3r0hD2E+rAwoPWMfaryJYLA9kMy7Pbj9n70qYTA+dT4YPr3YBT2Iqj4+7KbLvIvGFb4sZTq9i2o9vuY0gD0lOIW9K2AavTAFgbyKmgQ9SC3bvfYNDj26Clg8X41RPblCqzwL1QW+sZEQPf83sTzzsj69S1eKvoFOzbx6gBa9xpNoPtm/4r3Tet693I9Fu2MxGz5axZ09LS73vd7CILxcxaE7+1D0vfS2Wb6CA4y9FN6YPZBJEbzEA3O+3ehUO0gynzyyEY09rqynPQ2ID74BsQ2+ybJPvfwfjb7K978916jrPOea8zxOMAY9CZsfvTHhR71b5No83BvMPXcqGj5zf4678NMZvtSZND6m7HC9xM8jvctJFr59Zcq735mhvceLBT3KI7C90PwvvkTDTrwRJ6a9l7r0vODdzj1Pf708j524PQDI4z3754k8E2MhPPv7s73jRsw9w4jtvBD7Tz070l++YPHLvVPTyroETIE9al3uvZzUDj7XHf+7P5UnvDiuiD3VdJE9XrXou84m+j1+/fi7PDbvvFFkYj4uk8w9GXfavYRRkj3gSKw9B4U3vWnTJD3Kzgm+CrKZvcA6Fzm7wJA95MXFPZR1Dr5WMBA8EoJYPbsYqTsivQW+","xopFvt/1F772xyK9tU8qPbYDFz1oBHU71M1jvVa9F75vOsS7SV7JPHzNyTwoHgg+Pn8vPY/vbr0wS0k9D7glvVah4jxUWR2+oy13vMO9hLy2HcK8IZ1HveoZBrtSoo4+Co0Fvt1He70NYPc9n9bVvVcxwrwwVMw7tvVAvt3+xDoMdKY9u1kTvYdi0rzbUO09BUg9PYIvoj3xQIy+1yLxPPGkrTyunE2+EReWO1O3XL7opwA8oZ+MvQ7QBT6wSgu95Lu8viqkbb0xTWu845WYvA6Vz705tog9z9wzvZ6nOD07esE8PY4svpt+LLxNjzO93D46PSEA2DxOXCq8CEgOPhxixD11JLk8ELKgPaRJ2D0LKY87XC1DPa6/kz189zu91uY/vXSMYD1R4608q3mgvZrOzrx2Jq69WFs3PQMbFL5u6HM96Gu4vZMlxj2Krik+DTw0PQ1GFb1uTmC9Y9ZoumUTx7yVcDw9RAi0PZVoxL33WcE9HpzlvfSB+Ty8HjU+eKKiPTEg3T32tma9mLW2vVS17j0ZGZe6C6SCPkO/vb3A0Z+9E+bfPGUcrTuyzG+9AszbPdGpCb1qJKS9mDhKPV8okToRcQ2+0LB7PCBfUr4Gqn89/ipyPbnjwr1zvtu8uknFvZRB0TwPc6c9LZmTvB2DtLyYEaA9zP2lvZ4YYr7XGxi8ftDKu/kpibs560+99amQvVuGJz1UiRq83/b9OyPpqD7sf40+kS+EPaQtvDzFN5Y9heU2vWKVob7uzwO9AaTWvQhlxj3dYdm9hdjIvikg3D1I1ZO97gKKvTZq6b6MICi9XjAsPml7PL0+IkU+arstvhBHDT3j+w6+PSYFPRzA172E5Qe+Xu3EvVxD0L3FJh89CDHcvadq577WwR29sckyPu+AB732YgC+nVwvvcfY270rxF++fFCKPN8gND5ct8q9sAEivIkKCT7xV1c+UlAovmNR3z0BL8w8HJKyvbLl2L1yAUc7rtNlPYlThb258F29bV46vrO1Ez4sswI/","0mnOPf0uIT2bsoq8CuMguvj6Cb0VboA9uCnlva5BSD3/aV69DAq1vDxgxb3Ns2C6+RKTPEIdzjvE67w92MjNPGQTBr3tbLc9EkDVPAfnJT6x/oc9LqG2vTGPOb00hkC97cCVPfV9r72SGVS8vQniPQWltr2GCc68pYOOvbQj4L3RpPm8h3a9PT8x9L3rOBg+a84APvD1mj1F8sC9m+IVPWKkQL0d0Oc9Egp0PTZWc71Sfiq9vQwEvqd7AD2KOdU9TEoRvYZxYz321nY9yVTSvQ+0gT3E+GI9YrW0OydPgz3DVFa9LTYfPC/ijD0aFIE9H9TkPPL1CT6fFJw9f8RpvuHO0D2i/j299ziNvcGc0jnBkLw8XLAAPWIpS71jtq+9BhmZPQ94J7w3GZI9Hri2PbdcFLyA/Bc9omrbu49ujT3vonQ8IKo0PRNc+7w2gyQ+I+Jyvb4sIr0YxWI9zAWyPW2zVbw261I8M0dfvPUkGT7llkO8TfdrPRkbBD5MvxC+PjhIPF/tDr4AgjC+NGkrvRnT5j3jFXo9qXwIPtZKBr7R4BU9esBRvdjr+7y0ZIM9Evm1vP8k3r0lPRc+QFCXvOX9Ab1NAbo9/YEavUR1NjwR1Q840/zWuz4sWL4JhCC+BPuhPfR1cz0Z3pG91OK6vUFVob1/Etw82cULPUEMm726fYS8FajvvAqUgD2SI6u9I9xkvSEbQb3Jl4y7ebSAvRSKsr0q3vC99BYWPi53gz1r08w9qHflvYbVF73EoKw9FjtUvT3cQ74u30C9EVIUPqmeAL2rpkG9PTudPNMMDT7L8RA8NiJ9PdK9XL3BMcI9HYYVvep1d7rjtYg9WwHUvRAQyr033BK+482DPSxxMr3ffB29jo4dPdETkTwWeoo95+mEvVaoqjwyZRg+hJSKvbYnOb57VhY+CwmgPXnlcz0esek9UqcWug2NRz6qnng+gWBVPTjRzz1FeYU9r4MAvQqnhD2Kt+y9sXrCvMGD7TsRR4e9jcWlPBuikL6sSyc+","SDeiPbALt70/5Yk97J4hveKRij2OfY050imGvI85hTzhy8E9V1qVvGMzTbyoZqE9OAH2PMCdJz1kUR09SJ6qPNYskj3Kk7O8HfifOyv4rbuYRSK97Sh7u86pt7u4oAm+IFtdPMrLkr0cCmy98g3jO+qrrrxycfE8IZebvYa9Q75ZrAk8900VvUwvJrpBtZC8/8y1vPQ/GzyJAc+8rKmpPSGPi73ADuy8QV+DPTXgAT7hbd+8liKGPXn6Ebsft9C92D3zPFCNfjz/TXQ92A2ovcwQ5b2GGbM8jhE9PbnbKL07x4c9J+TcPI2SpD3DKoo8Rk10POMItb09cwe+nhiNPbQEyD1LQ969XtU8Oz61jj2Ssym9YqvmvG/mdzzan6O8tMu2vNeZHD0o2tG9IW6SPfWfiTub+yI9TgcauESytj3JxxY9bjr8PE98TD1T3qA9VU1dPeR+Rjytib88diOCvG85zb0wWWi9Lww3Pjtem77IF5u9vUrqvBkRvb0N/xy+kEodPRcg0T2Nv/I9GbI7vZlGpL2qlP490gw0vQOGb70rZ5e9BruGPYBygr0VJ447KNbfuwwgXD1vAW89mucCPiIjhztg7n09S4X3vb1vkTwB/4C9KcVAPW0Xhb0rdJa8Ic6qPWXb672kqMu8iTg6veX1Cb47/Fg8tnRZvWmSFTpIh4q+qZeVvSZnBb7/oOu9ZuS4PSJUB73ERME9p1aOPMbq37v9gQC9U37XPVnuGj2LELa8/Xd4velDAzx+I5i9MB7bvRpuZD51yoW9PyQMvq0j4DxiXbK7PeKmO4Zbvr3g/ia9ZsgmvXCkaL6kgwE8kyDBPPDg2bwOVQq9UWnjPYSSUT1veg89O7Aqu7BRujxbnIM9aT2zPFSZe71Y2pG9Qpm2vYcTfr18cce94V+VPRzuNj1h6TE8x0LcvHsUmj1iVCA+k/jbPQH2dT3s8To+0CqovRpQkT2ZSLc8pZ7MPYkKu70G/iY+cuhpvUrK/T0E0R+9otmSvbEeFrzATVU+","oSK7PXz6iT2MNDC9a94Kvb/JPTyUcgU9sXnPPFgwHL4LN0C9u0CRPXtNYTuvmkg9r/gXPZiy0r02m3K8wKczvACR4rwJ+/U8cUyRvEITTT0Y8ow9d6jEvDHbzDw8JBy+OHnFvd0dGLxTWBe8Y4haPczTDb7cltu80KqnvX1qjr3VpJC9jvHWvF2H97xmVJo8ObS2PXK2JD0pt5e9te/LPMfTHj26Dem8heozvRaSwD0JQMo8teTYvM5Ihj1D10a9ynwhvh2WTr0Ue0C9jI3IPZW3vr19aVQ9P7XsvH3l7LdFIZA9nBfPPRMVjrylghA9MkHrvM2WY73cG3o9b6ZCvqtp+Tw6WhG98mOLvZlVBzwJ+Te9ND0yvVgUzD0ZFNw9eY8QvaqPlrtyno693M7nPA1g9L0dYAE8ETebPdOepr1Gbp09KWRZvdAJLDzRqXu+q+UOvQfyjLxd9g6+OzoTvjCA9z1ossE9MlsVvktPjD2gKRk8p0PoPYN/Z73Dlgy9HPa0vDINrD48PL29L1N6PeWWIT38HJe9RcYRvhMPaL36YYA9JM2Kvc5hpr0+aQg+HkFjPcY2wLxcOAg+i3mMvGvGRj3eMfs9EcOxPEQ8t71VUo+9+0ylPCBNH70Q4QG+oyaevK7JNr39vsI8e8GMvHx65r1NPA89nfPnvGJMJL7g3cQ7lE4ou05JU73EpoE9eiddPMm+EL1PBDy8bGqoPVqQ37onyM+9tn3fvBZ6ZLxkeOY8b8nPO6qHSD5auLU9g/5uOwezHr7WXOw9qvtVvsaLYz2Nnmy6d3QXPFWJUj69EME9Qo+8O51zEr5rD2W91O4EPfdinL3qhls+xV8hPXsxgD28z4C9rIS+PMpX9LzcuY89IyVEva8JmL243Kw9Vkm8PegL/D3qHo4+95GkvQZbRb0dlgQ9NBHMOw4GhryNU/090JelPaJLIj328d88hRyhPfXnuDyxRXc9siYoPcctrD3D9Pa8btJPvfkrkLs9tlY91WibPQt6BjyY7CC7","0gXTvQLAi72fF828QB6OvcofRz0zH0u8jfPlPWJalj1xxPi72XSXO7WpBj6X6Sa+pq62vKxOcjy5NM09eGi9PQJheb3yR6Q+q4KRPcYs8j3qU6O9r+9YvIGeBjuYRSs9Uz4ZPkspvTyHNt09RQvYPXgOyb2qwFI9KVSxPSSFj72Y5hw9/pX3PWXQwT1DC5q96JJKPXhwn71sq509NA1NPLMHjrxTgFi8JJyPvM7fGb17KxG9zMTiva3EOr1bGRa80gDzPJ0Hcz2JNra8SygUvWFJ7Dw9AG49LO7fvefH9zxPtlm89I/aPZjUvr2daSc7kY7fPJhKpbzcYSq8iPxMvvQPgrregD+8BEfKvFYD3D08h4U8htnrvAUPOb0QUim8v6hYPZR8972Q08A9FPLqO1GXez3pVGe8hlZBPXpLJb124k29YdcbPdD/kT0r9Lc9in65vGXrez2b/4y9cqa8PbYr8TyZq6i8nxGsu2IDeD3CfhW9PZvKPMykID3dx9k8b3p4vczLA75cEZG9OICYOzYMrb0mqVM9qp4pvTHdBT6otro9royZvdhvqj3qhB88ay+JPZ9Iwz2ECx2+ZL3TvAZwbz3hvMG9XTrEuiXkmz2oMY+9CUV1vQGHsz1hWoO9L2QyvGgQrTy9qoU91XqCvNdPMz0yTw4++OIDvZbCLz2TQCY9vOPZvVwrRbsj6BU9CFtxPeGCT71+LMQ9eb5rPXY4gz3jjIk91ZOnOrrgtzsH+vq6oUARva++ZDyCQYA8/PgYPrMJhz4Kn4I99vGVPjlXUr2ivSO+Ueq3vAwBEb4/ML68ofTcPAERcz24Ek4+vvGDvcM+27woGLW9kchAvRT6Mzw5tR4+sl40vNefybwojjG8y31MvC3wsT0j7LY9oU2XuxeYMT4UnW47bf9nPYpOzrywl/M9Mi+rvZw0Zb11jJe8qW1Tve1/Or1Jeyu+5mlZPWyDWz25kZO9DIENvABTcTzbgyw66c2YPdb7iL1Riri99HksvbJvjDxY3za+","SxOqPYJMrToU8iG9J4AwvcNnxLuZdGM9e6EvvX8gGL6+Ieg9PVSpPKloTT2P1yA8YpTJPWY3Pb3qmG89NFjiPVaORLxrd629R4/avEYDDb4IPfa8xOORveUyoDypBQ++WuHqPBRmoj120IO9QXYZvJ+urDpNaOc8nD0AviEWG75sSDY9gAmpPRXmd7ySGh49AKxuveBseD0TiO458ZaFPSWtPLxKorq9nIwlPOMukD1rCDg97I51vbSWNj0L+WY9PQPIvSEkt71PTx88mg8RvCuFPL3ma6Y8dUgFPUWgoL2T1D89lSTTPee6vj2uwnK9FuwmvdO5or2TCIQ8cWoXPUilwDwLjOu9mIziPcGnsD1JwIE9xpsPvWcDLr5unVU9j4XlugOgqD0gvMi8L2GjO6XyNj5vXB490BltvS3ciL3UDBe+fI+dvaLyTT1J7ZO9DyKjvTUVCz6FMQ++wGpavufXprz51zo9g31uvf/g0jsSQGQ9u/SyPLprvb3sjco47Xfxvdmdqj5ebh09vGsGPovekL25O008/LYavshThbxxiYC8cEBMO6halj0iMMS9/E48Pptq4Lyjo9O9J0W/vNz0xr1ZAdW83mc4vggWhD24YAG9cvqnvWeKcD2tbuo93huoPcjrsb2i6dM9tVfWvaay0r0N5Ju9hI41vTBTWT4tZAy+GeqJPYFaE75+gXe9olhFPfQjpj33RLQ9dGwDvkGlob33uII9mbqUvSfJyT3ahO+9c/zGvFi8lr1h0829lC9KPIxE8bxsnnY95WJvvkUkm738s7e8oWuCvYGj9zpyIzW9V1yFPUG65buO3X68pX86PYo0a73D+CI8YfY7O82//buATNU8O1CFvdblMT2dCYo8zfSdvJd+lzxE8um9SKZpPds65rubgZ49KbdxPTnlMj3s3ym+Zfepva+FAb4vHYG9AptTO3HP7L3IczK90ZIbvZ78IT3w7YI8eqknPdPe1L0JwPo7HJnnPfAAkL0uzIo8xgA/PYxpFD7BT5m8","bUEuPWe8nDzX5OQ9idmMPdHQQj1rUOI8Ao/QPbIdIT237BS+4poovhmzx7wTufM74PuevYDNnr0bkpa9QuSJPDDIVT1Pwi889dctvbkdOj7IvJm934+bvfr3rjyydyU9eb9Yvf79or2Ox4I9gtEcvgnA5zxahoA9pki3vViyi7wgyk09jKOJvrNgbD3PDyU9shi1PeIMUD2gm/i8yK/7PJMLZ72wKaQ9qnHtOpUP0T3J6Ui7ncfgveYAOT2Sga+9fzJGvV/XvjwXiUE9jQ4tvb/Avr0PZQa9XChRPGnd6ryzH5G95bsYvRK+DL7WirO9myDwPToyJb0lyr89GdSYvpDtAD5NOKA9oB+tvKg2kb1tiSu99iXnvaPZGzzems+9DROwPXxCLr71hYs940UjvSDd2r27gl09nF8lvVgGH70Wdrg9L/p7PTlaATpjeFM+FCQ7PVakjDzZ2pQ8ACskvXRs0TwDdZU988CbO1qZ5j34GoG7sUbSPXXMrT2UvL89p6WKvTgZnLy2tjS9c1utveHWB75agWM9+vGNPU4FSD3NXBS9A1zPPW58vr3GYR4+u6nnPHU6q7znC7y9SOEAvRR0AD6puSE+5bqoPeqkvj3sbOG9VXf6vVrVYz10PpK8nLGuPS7mub1i/jS+qOAlPnQIGr5Ywgi+TTxKvAvFg75R6pI9lu62PdW23T1UvKm9Q2QEvQO/I717eAc9H9aKvUpz9r0x5hy+eAXNPL6icr0B72I92L0hvS9BG7631rM97/2JO11IFb4pqsU7q/4mvaadpb1UBdW8FNg8vcfloz1X/UG+bVBWvJiOIr65BZo9C8LZvC84uzzGGiU9BsFFviEr+Dt1VJe9QYkHPUMtPj0RLxQ9/FXqvQQppL5q1Yk9jmNmvOc1iz2JjUe9MOo1PBCyAb0i2T495WaOPc7cOj0HqAy9aSKYO5bcaz2FOSU+hHCovcy+lTtoqA89vuavu5UOhb2iqjI9ryZSPUSRhb3yVhg9LQEfPSjvLb1uAQc8","UUVYPvVkAb4HImG88oyzPNTiXb2qotG7pp9qvfv4pb32Rzc9OuzNPC+zXTwJKTe98ZKBPvBWBLzsXd+8TgjYu6hZH77wWXY97nmjveoBFT75Bus8IH1BPC8xcb0JH1w+5ToQPVDjBr3hcuk9HpNbPBdb8jx2jGk9tWXlOz8SPr0jdqQ96HU9vhMTOL21VRA+qQ9RPsy5Uj2R/jA8J7S+PVfVibxieiY99LGhvdhbxz28CSa+j68XvQOji72iByq+UIAMPYYGDD3Z++Q8i0haPhI6IL4YgIk9OzgJvTFq0T11Cu+8ti4Dvrc2l7xZUo496GQGvqkmyr0XnVI7cSGVPVq3TTy79ak9AMMLPnnTtDteFr+9X6aIvEWRXL3Nigg649FzPeBWeT3YTTu+TEwRPW5bPLoTEKU8uGE+veIhgbtHcji90jodPoHm7r3jJAS8M3G6OxNmzzt8S328luSPPA3Gqzx5oys6rN/tPSotLT7Zd+u6btCtvafH7DxnSAq8rpvbvcMLZz3kdrI89v9MPSarAjzDoBK9JY2qvWTzljvk4kE9ijviPLzw+T09Zs+9ix8cvUDL6ruGcyu9M82DvELrpb1a2nK9hcexvTcnMjwkg4q84A+7vJWVqD2ar5o9yKgjuxvF2zzhifg9qF7rPeU8Az3kXFE92DEPPhaRAT79BtM8PAY7vah8cj2vYg++oE1+vd1Sc73BJ0s8iV2JPAHyBD5vVe48lE+0vecS9LrpZ0s9foK5vZ7mhb3ntfE9oXO7vSyAHD7AeXG9ToaJPoUtpj2S+wS8nbmaPG1PkDxYtU09aOk5vUiqET5F7m+8xOXUvSuCK7qqLbO9rWYDvcFKTDw1xVq+8nABPjdRpr2xB8i8YXqHPFZ8LT55nwa+7HAIvFOmeDx9hUC+qgSvvQwY4TzY3U491cIQvUMupTxSkBs8pawqvYFSTz0dpoE+JiHyvNex2L3q5N+9k6zzPdWduL3FyMy6XxfavZraZj0kBwc9qr5YPWohIbtMJiw+","WyIrPWmbDD7f2bY9w5QavW3Nh7wOxuC8lWIYPbLrjLxlX9g9q+i7vRwHX73ge+M8ozGjvUo17LwReVE9LtUWPntZ+jzyGxM88OimvCIdnz0WK0g9AQWOva7qGr26FyE8MfY8PSFItDxY7Li8Qb/FPJsX0ryX+5c8oSbhPRj6l73JQBm9xv7evIjsrj3XWXC6Y0W4vGoIgjowqve9z1C8PVbMTDtKkZM9W6IZPWVh9j3Xo0A8MXOsvOOQ1zzw5RW9XX85PG8wbLyZ2bQ8IcabPZvGKT3u5Yu9x2dFvc6jYL2copU9MZOtPAno0rzqoHy9B4twPfMXvD22APw7PUqMPURG0bwst2G9jqTHveagRT06YWQ9GVPRvdctiD04SL69icg5PhCIHD224Ic9rgRGvfMLtj1dK289vKjWvJ8JlT1EPuI7ygeAvrygmr1Q6VA8VXTOvezN4budEn+8riWYvUPOxT1cVN+9kSkivNxgKr2jPHk8uBk4vGFhIb5Auvm947mNPSlUMr4KbI+8Hw2lPCuXrr03Ia48ZwcYuqenmT3rdJO9KRiLvI9hK7whI/s8Kp9XvSE5or2IyTa+jAqWPUZOKj0aiBa9k1PqvG+JT7xoyIE9ilWzvf8AIz2M2ne7DcwfPIVmC771T5A959oTvinuU7twoA89eJ54vf3W5z0ik1e8SvxwPVcrnb3k9+M8Yi+sPa4O2rx4khI9nz0QPUE3JD6HmGG+pMKSvUBPmLxDuKO9rYl5vN08tz2QRZG8Qnq8vUvCe76ZgdY8HzQvPUaguL3CQsK94gEKPq8dQD64wQq+jpDKvgkTTL1tdbM9j8fYPXXMlL2g5UE9RMPWvaDJ8rq8rnU+sQgvPfz0gT2f6Os7vC9NPdAvhL5Ul849+r3UvE4oBD6waBU9WIMSPqTY8L0kdIM9Q+0bva6nET4kZdg9X5AXvnjSCLxCKQS+baIDPmWVkj0e1ka9fRSXvdwp3D3V8Aw9KJJvO99VvzvstKq9URa5vQk3rb1x6b6+","p2B8vS3p8T2o4Zc9pZUevYu1mTvZ1SI9SolbvNmc973o4Z28fAe2PrlZBD4Lsoy9NApNvYuVbb30z1c9NnyVO1+LYDyVn4y8TLDkOzH0174YrH694s7OPAzODL2T8V69EokIPdr6jD19W9C8mge1PX3blz3LINe7+qASvsYpsz2atn46YXyFPbRgQb07wam9Pe05vRwwkDxDfpG90HExvZQIXb3TtAe+qkRlvsmXrj00+vk8Eg4CPuUvBT7eAaS9hZ0ovq5iUjzfAaM9C9cHPvvXST03etu8iU1gPcOHNb1yzRK9nKGLPLmG9z2Ehxc+WLjkvfsU6L0IFJk9UgLfPLPT6z0axkO9i/DQPVG+NDyNdSk93tQcvYsbOb6yeHu9tEf2vct+0Tw9PE091ciEPKJ9hr2bs6+9cV3hvcC5Aj0bnBO+7u1+vrf16r287Ak+88bvPXYkgb3DvtK8BusUvpMZiD1uL2q9c586Ps6FB75rngK9JeaBOS43j7xNZ0q923BnvUxQU765Iks9162fPYGAt7y93NG7iq1HPoHctLzuOIK8iUUPvYVx8b0COKQ8l0dVvbkkyDsub/09sKUdvifPQ75Pkz09yb9rPSgg870YIaU9bxmQvVtr9LzViey8K1V0vVl3Nb6WH8o9jz0bPNWwZT0pIcE8Z7UyvWpEEj72+qw94h6TOzyaJ71Wevw9m4CNPBybyrsAESa9tmrsvU8PYbucldw8yxwKPgNu37sO7mk9FoD/PDdbaj2wfT48roSKvTspHL4Xd9+98V2PvCZfJb6txm69h0r/vExxnr605Ka7dJlMvOkHlbxchsQ8mpR/vD9oAb7UJF2+HpIePq5Osz3keu29SHE3vfs1Ib1isJe9+4g4PQGJLD5V3aw9sWM+vq3sxz1V3IS9tp69OeRRKr73AuO9voevPSBQQj3akS2+ksCgvcwDxL2heDm+hnMsvKswfr2b3XU9t8NyvP09lz2HEW+80PJgPTumqT2t1xu8yRGjvbXvob3Xyg6+","/B71vYIG6bsSNoG9ebQgPaPgND1k8628NqzdOzzRpTw//ea8Z2+qvX0zdb15Ty09Gri3veZRVz1sD6W9TbCYPfpsgL03F7u7ER9NPLCzsLyTcSW9J0jAvcFJrDo9f0I94ruzvcfLAr73jE68XCE3PiU68TxX+Fi9R8/dvZir+buiRfg9Mt+MvtjB9bzn0+O8cAwlvR1bmDypHWK9hJrbPWapxb3aRkq+YVNTPWzoMDvMi+i92dHduhcP/TvIIZ+9UwQ8PWgFFD3UZWg9O59nPdsnP70oFuW858+TPQJZgL3TTha8AMtdva3eb71L9Ha9mbXnPIH3dz1sINs9LiwDvijWG73/ETI+gN6yPYXzMb3ceSu9/z2RvICAFz26dwQ9WfV1PZIzkj1urBE8GhDQva//5rwxYYM9jwCsPfdvIjwVLR89Dqavvayptz1V3VC+fFiXPZab07y4rhq+i42FvbueaL0g2Dg9MIlfvRAvmzxxN/U9gxuLPRadkD1d1YS9nsugPV8g5T0kjt861bobPeN7NL0YnMm8fwvoPRTvQzzVzAy95RdMPW7eZj0l4349DWzGPLnhjD3lNvY8pFqzvcIST73P2bq8J7A8PmRKQj0XFLi8LNpUPcgKAb3p824992MPvTKPtr2lysw9tl4jPLWSlLzxz7e8Ww/lvU0XMr47YMC969cmvTePVr2KzPa8WgCkPXeFJbyxoqY8tRWjvc5HKj1dpKu9/AYHPfsQHD1l36o9DAwYvG+4nj2yydo6fb4aPQNYs72BzsS8JQE/vsvkA72pOzW+SGb4PJErM70tDcq9eg+1vAQmIjykZz28HXRsvEOdRjxubCS9EmoiPQDTCz1VOcc9eAc3ve4wwz27uKS98kvJPK3TS74ydlI9VGZKPZRuXT1j2AA9BRzzPSyiAj1NlQM8hsm0vV+M+r3wkTC8NpfEPFSi/Txw7BY+399+va76iz3Jara8l4vRvTL9Sj2Z9T+9m1tSvOm8C74vvcu7tQsavCl0RT69Mok9","+h0OOvUuHj31+0O+zBUAPOOA5Lpbo+e9jnIePdmjuL3BD2097p6CvRcHwLxn1RA8h6QOPoe/v73UTjQ+XXV9uyyfZb3npIc8kqKtPQMGoz0HDsq8eu1TPQkYMD59O8I9udq1vAcKoT2pvIU+zB0cPh6qi7wWT8k9M9z7PSaz/b0ICKA9J9i9PaaaWT1KaRg9W6GZPYN1qr30iSW9VuGUPJwbejxyvFo94tBNvaQMWT3ZB0G8Dl+APPIF4TyNgic+TRncPa90Zb2zOy0+vHC9vTHTUzxM17M94wqDPZIrXD2RYES9c6OVvR6V/r1EE4K9ZrfePY8WBT21WVI8CtFZPvopAz7r9DK6RICDPfaOizsCRye9bguDPcUPZjwGYgS6qJHQPS4/PD2k+rA8XnN4u2HbtL2WrCe9XPtrPczPSb47fHM9Qo8bPfA0kjzIYqa9JjygPUMHzbxqrRK9TrtWPSywlDzETMq6z6kwvn8E8z3ZC4C83ruZPdvNCb4/Pci99t2bPec9PD6t7LW9iwHmPfQnmz210Ae9d9SmObQ+Lb7FXZw9ZmRqvTmoG77MsZk9VJcZPpiLTL0bdI09crQ2PuokC734Xjs9f0sHO1o2Nr5YkFS9FFiMvehU6T3lmYA9E65DvE8bhT2Gl+49MwiAvqjXGz3JmOu8+rjxPJJ9RD5kAdo9YfbJvLw3CD5kE3G8jweUPQI0DT2hc2C8f0viPUeCvj1eE1U9GZgHuwxMaLzNK4y9KXgVvYzcYzzBS5g7ATqYPQJ2tz2qgB+9ntb7vPSMGbzY2z48Kg6+PVYyKL3kDiU9tNhvPR1AIj7zJD29xHkKPa1zvL1ebwU9sBoBPboRXz0Lt0Y+yrKIPfe7Wb0M0Ks8/p9aPbNbIj4hqQy9jjhivY0yu71y64y8k6Udve9p7TyXxti7+EexvHreQD0bq389J3ekveSVWj2zeoU9L5eFPD7i2TpTOz+91uhxPcOppTs0Kq28olu3PYsK2ju+3p+9Oc1iPHnFFr2exx++","3389PVzpGj1NF6A9VvedvB0Njj0+CwQ+IVuOvUA0sz0a0cO7k5mHvd+h8Dy4GgO+a5aTvaoPsLyahp+91bVBvKo1TT2tjsE84+UoPfWVuL4te8g8cIXLvUZyPL2f2h2+UOWjvayuiz3uty69WNngvevt5jzuNto9rfDvvWz1ojwkNCy+M6UIvgAvyb23Css84HtvvbedMT2UV/09U9N4vpns0Tw4SJM9h7A0vTIfhj30Q1+9G2eFO07rlj1dXw89NOUxPaIFDLzu4tc9iofqPQNQv7x7wK+9Zn3rvfiKD70lj1w95lIKPiIEIr3Kp0+9JcZSvXXhpbx4QOW9EAKUPaYmv71HxWE8lgaFu6clSL2a0Zi9RzqRvF4d2jytqRw9m0IPPZ2EBz1NAJI7k7+EvSsv4j2xy9e8kXK6PU9bjbwSkFm9yJJZvp6CU73Lnn49e2QhPa2ygT0G4aE8LwcDPmTFHb7QW789N11NPK7ZDL4q/dk8NlkjPSs2yT1F9Se9CGgQPcsoET5ZL5U94d5WPWtl4r2GaQM9fUkPviLRSD1E5lA9GC4IPbxJ273t4qK9oOubvQlJcz1j2Ta98BnZuunQQL2B/+S89FY/PccU7rz5zZo9XmHGvVfLmLwHw5c9O1UnvNixmr1+gZ+922oaPnvzJr0uy0a9jXePvdIYOD1ItKO9yO5Kvj73Fb4zNx69mfG6vL3ODz0NOsa9yocuPfC83r3KGDu9GigvPnlhA7xKis+8PqMZvVwfFbysaGk9SFKgPF4lMz1jhQc+Wx9TvmTpbz0TncS8E82BPIT1qL3/MZu9DdrVPDkxYD1J/me6XQSpPCWbVTqo/eG8FwqcPTsPlb19/Fo+eLwqvvOsd70jxZK9LBJ8Pa0GZ70KcNw9rkyRPB7e6j2lGj89cJaqPKTwCLyKj6+932ycvfL5er5CZYE6R0YZvr0kyD0CkPa9SwJeveo2zbz8bZi8k84VvsEx2b2pBzo9c3rBvG6NczqA9eq83J+OPS7lKj2SEIq8","u4IivbPh7z2w8yM+HAuZO1mU5DzsoDc9g6AtPbdkLb1XnqW8IJILPnPnNzypRxU8AzCpPCEYvLuwvrm8llQCu/KW1DwPYow9OQz2vF1QuD0YUpo9cX3MvHkKMb2TSii+MliPPdLMIT3DdSg9rNVXvWAajjyuZqQ7GYCuPRB9try8yyE98oJTPr8C0r1ZA+A9LaMMPL1jvjx8I7Y8HkdvvewvFz181u49C/rkPXeFtzwuhlc9nw4sPhVwQj3tMQ+9W7BXvcOJVT3n3q47yLjRPPTmPz3ee8u8+1jjPWIsCT2dZbk9nIp3vSuhAr3LjW48lYYxvOhiPz04avo9s+Fkvt4l8zx//D+9PqmxvUEz07072hi9gD/ava0QSLtxxCY9aLccPlzAhzy4Lcm9HgoKPUadZL01xsu83LoZvDxz8jzKb6e9VaMTvna+mT08Jaa9yrJNPAGQF70k9tE9qdeNPWxaiL1KWWY9DBqYvJjb9rzCyI+8LbmsPEuo3D2kr8G9MRQOvD0Qd77/GDg8q49XPDcTO70UaJk80pm9PeG+Az2llSU9aa2tvSccIT0Lyi4+jnSKvEYsCb4C8Xw9uUT0vQeli7x87p+9sHLfPUBm8z3Wwb69IweMPQLZ1Tt2M4i9d8YLvAi8rT2eqrK9bXSsvY7gz7yjE4w90CDNPZxXlb2L9ru911zJvZt+5b3VKpy8LW5KPVNs9buYY0W9y2qSPWRF5jy5sTO98rOzvYHyr7yLFFq9PkS2OFejyL2xkii7p47+u9KiAb361aq94W8GviRYbr1aA+69NcytPcC7WD7lrvK8kQo2Pb08lr05ygY98fbBvWl8ur1ZZQA+84qZPCcRxjw0UkA+7r0QvAa2nr0zYWK9YlEjvO22wr1eSD09x9hfPZ3gu70rTi695dkHvd2drj3f/tm9kh1Jvaf2CD4bwgU9AHKfvR3dib0au6I9gWCaPRl9qbzN1488K6X+PHTY1rxWyzO967UGvdMrhL3TPzW9mhwJvdEHJj27WdQ7","yt8FPjiWR76lTQ690LqtPTT+VL1EJNK8MJ4bvZ7Xm7xm0j68fqMTPowzqr0ztgQ9OULWvTTDl7xo0zS92qKGve1xYzz9zTI++umWvWBzkDy9itU8cHCRvZf0X76Ap9c8uQnjvfej1bop4QC90RjqvTHLU71ggMw9NMnOvXFVZr3B8uu9LG6FvtcC8L2Yz8o9RSgKvhV5Qr1K4ME9nwrZOg/apLx085M8B3wnPqoTwL2aAKO8seF3vW1NijwU18y9VeSYPZKzyD3eWke9bT6ZPXlprD3LRa49vBicPVj/ED09n5s9kUfbPGs5yLzImHm8YFrLvfE8pr1h2a+9OLMivqH/7jy3MAi9SbAJPlf0+L3gzXK9rT7jPLblA7ziAxS9H1CmvZKRpT1poYu9biPUvNZ/xTw6UZA9+BCxPV4fh71vDKw911MqvoRLnru9IZg9waqcPVgll73Ho8o9fx1Cvhsdcb0eVBg8vzZnPizlh71S7di9aaw9PWiKKL3zPlS97h2VPdiLSb6Xvx+9DSdAvWaOgrtFjJi9lA8LPsP0QryjgWq8ApahPSaPEL79cYW9sLbfvVNNRD2ljbI9j0c+Pn3Q1T3PYES97JG8vSYNEbzyJbW9vhDXvdYJEr5UsuG9elh4O2ctZz2NGYw9iXWzPcsllD0TN6M93p0LvYOuMr7f/ME9+MXpPA/52z1dLmK9eLq9vQqseb3N9Lw906FivYFWwDyGup68COjRPNdbDD10yW68LxEevazBMz1SssU9kXN4vcmsYb3hAm28NteMvuctJjxa/r27YjSTvFpc2L26ixc8S5KZvKdQFj7LjWU+6oxLPcLlJ73yGgu+XbC6Pe0UEbyAVIi+H6OlPbVJIT1VxEQ9OWgMvQASmj3HA/a9l2JxPRMUEj0h7169VnXsPXc0bD3oTWq8IgQ0vINpRL2hc/E9rBUbPOc0nL1fWyI8NhmBPHwziL1kyaG9lfaYO+wKvL3SkUQ9e+0RveL1Ubwg3L67FdVXPTy4cryXmAK+","CXq9PNcHKz2r+jc9OWMAPoDOPj23viC9hse4PcIWw72roIE934Dqu+6Gj70CI5m96RMDvgye+zyNVKq9ZT4avhuZLDyhgQ8+ZUPTPWoXuT1rKUi93biVvBnyXD2iNLo9m50jPfWyhb0yVkq+z7G9PZ7tMb3KpC69gsWAPPIVNL7SPBy70uUKverqZD1OObM85piIPWY3m73jhOG90dedPZXahr3Nf0S9ZqZaPdquDT2iShU+FfnfvVr5IzyiIHS9rJ0uPf/Yq7yejrq9nxDvPBWhnrww8F+9gfE9vVa9MzzxfOk8cjAmPbUC5r3/07E9Nw1uvd3YSLyT8ce90bTsvXlicj0VbqO99ZKdOw3Dib2e9n48jBa3uwaAvT3X5O+8GiYFPs7vAz22TIs8CYHsPE0e4zzIuWM9jQU8PEZ8vLy9s7K9t5guvuZFYj1u9vg86WrfPCenej1MnBq+EnMdvnLXBj2hugG9fvkAvZevBz58Y4y9O6lZu39exbnIzqg9rm6CvR/xBj3SJ769KtxPvcztrL0NFRG91eGYvfGpFL5omUA9E10tvEyN8T3ZQrM8E4WZPaD4vr2BmVG7oWuFvurvrj0TSa89y3eiO1xmxz30Ohi80KrevASh9jwh+iU9W08nPT6lor2fhqm96rIVPXd2Cr51see81z1bO1eAHT11wvi8HJ9+vW3QzL3PHi09igYrvcPYPr1mdbo9OG0ZPEOeiL2A2RA9MrL6Pf3HO72dDyo7AqYvvPszhL2wR2M9cdi4uRQciz7SCTc9cGvqPXnPKD0gHks9oysfPa0aib2e2567XBUMvTOj9TyKPIG8cuHZPEwOADyWDgA9RVPYPXNb/73hCG89PJWvPL1dKLxxrpG9NLmYPZS3fz2XYk69aJfsPLiQUb2Vpc69PqRiPT/hp71z5689bkNFvTzP5r2UD1K9ecViPcY0zL00Nbu9yEuKO7AlqzxfvaA8pvLrPHr0Uz1yAJc8gsS1vC2Tar4j5Om894OaPUunhD38/829","xNMrO6CSnLxPco+8+PIxvXg5Ybz83ny9PKt5PU6o+r0iZsY9VN5SPevCFL64xwS+Sn8kveTfmbnRg6y870qlPYOuDr3eR3s+56jRvU3V37zAK9I7PXZbPEgJub0gqt+95V4mPTLWyT3Jda89D1A8Plbjeb1bWYM9Ew20OuAwvL2hEIu9DzCKPveGD778GrS5N/gmPS6Xfbyldq89bfhJvbXyJjw37Vi+yicSvW7FND3MRp08inmXvTlFvT1uHBK+eMkkvVm3uT0xuRw80MtvPavnhz32s7W9nn0BPXylarsC/Bs+McETPrJTmr0CNAU8ek6gvcWYUL1MTPg8W3zCPVCvhrx4Lga+1r4YPuOVXLzrixa9zU/UvOkaE75OM389mIqXvL9+P75a6tG9nUQZvf+R8DqlXbi8XYjrvf3gjr3YfRi+iVWWvmohczzQSWg8OAe/vE7F2zx03KO91sAxvh8rUL0kZI48mGQavm/l5r2CtVQ8XJKXvYS9bL2TVVS9iKTUPbi/5TwneY89Q0D7vfnIQD1r3tG8ER/PPHZtrTx3y7+7JLPovdFDBD2jvBq9focRPk9pnj1BagI9Y+TnvWQy+D21m8w9ZwXHPcoojL55/aM9mpu9ugrFbT1nzBI+iGGTPeKu3r1q88+7gKUIvqiavbugFdK8TV8YvtYkpL1D7Y09DP6VPZOXzr0edje9uy+1PPT4iD1MOHc9/oBLPPHL1by0FsG9OV8pu0JUMj1uSV+9er3NvYF/JD7eWMk8zymPvLAc3DyqWOo9QakPPajUiDvSbdM9SBg3uVAd5zw0z0a9uyehvcLM6b2txAo+f58uvQhB8bvZZlE9Nkn9PfBD1zwahsk8dWagPVun971D5i69lhykPfTorb2w57Y98eNLPXSVm727NYi8Df00Pru5dr1xCsE8AcY1PRZ55ryeY208YNSCPd+j2DxWu04+21vkveyXNb2jXuA8LIx7Pcq0nz1CuXy8jG+kvadesr1qPWk9pDatvQ7TrT39VAO9","PZppvUVKgL06hmG996WcPR2ZRL0xFjC9Y95lvWT8Aj4lHYI9lV4GvjgHVT1YbRy94wItPfcpAD3ob/e8XLY7vMtusr3GASe9CLTqu6nUH71nGIq9g6PEOwtSnzz7E4W9ZHIjPYcMLL1Igws9u27rvT2CCr1teKS6TC9LPTtotbyovDc9nCmNPYxmXT0bEQC9YX0TPZ400Dx0h1i9JKa3Op/2wb2v95A8HvY3vGz7NrzSy6G9F3UOPt6dlj1BxYg9p/kDvqFOHb0AkcS9nizOPQw1or1UXZK8i6ARPQDiVDtwW349eUQ3vGLW6Lz8ycQ8hO2MPewwETw1UGy9PzQyvo/F6j3Vel09lrq4PRb2rT3N2Bk9ulQ9PVJr2z3/u907w853PVb1RT6it8k8y4NVvRpx+D0y5gO92IK4vTEEhTwXFgO9JvHgPZJh1zzlLmg9ejy2PUEmgTsJRdS9domivc5FDD1xOK+7N3MjPZZTJr0Y1gA9YqDAPa+ij7z/phm+oRczvrq7175Bt0y+7hQcPTRYGT40tJ25M7+YPkX187y0zls9zoe8Pb5/QD6fuQi+AUxnvef4DD2rJeU9B8AdPv/0bLygVZS9hkUdvcC9uz2vJak8C85ZPYenibxeBmQ9mDw9PRLR673x+lI9LjZGPs3l9buwk1m8a2IqPXrynD2RKeg8JImsvYd0fr31Fhu9M+TGvSIRlTy4x7+8nRypvfu5lbxj68Y90JsGPZhMTD2DHha9ZzLBPVuFAL0q25e7G1qGPSvIiL2aKwW9kLsQvIrn173fAiI9oVKCvSZkir4eUA47OfqHvbHOg715E8699p4UvsF9/T0BQNi92ohdvIAGFj2hXHS+L3ySvbPXyLtbMcE9uXgPPZ5rMDzZMAW+1TvmPCAzuD0B2Tu+v+nrvQElLr3cSK+9F/7CumOAmT1VzYY99uszu1Gu7b0C2P68yw2PPdcv6r04ZlG9G3z/vU7c9D172gC+t20fPO31dL0qMbw9VPa6PM2JtL1xC4M9","chlgveBClr1Meym8mKibu4QzBD4ORos9I2cIvQXGPL2Rk9A9puCSvbSltD3ecDg90V7ZvBbAjb01YBc9IeRhPK9PZz1a77k7oPWOPJ+ROz4Wjgq9EP6FPafRcz3pQOM8m1GsPf7H6b1pNa48BgPRPUks872CNIK8rJTqPAJkDb7muAw9xYkqvqhP0LyCmTW93PObPWkkAz05fBA9YG4mvDyn07y1tLy9bNYaPcaEp721TN68q03ZvZ+tQb3t1wq9dDoWPhHTcD2RGM09LvjSvf92yL1szKq87Ow5PWpRnD2VkZ69oIHcO14FkbuhwaM9pNxXPDpchT2gNCi++AkVvgcVkDxQD/W8DrScvdD71rzUXE89kYPsPF7Lvbt7EO89iFjcPJ7o4b3VYts8F3JmPUfW373HCgm9GbrgPVM8qjxToto921EVPpqIOztFRYS97dHQPUGIE7712KS9hiDHPbgjn72YAlw8i0e/PQlvnT3a0mG9zVw5vdu8Sj35j2m9sZ8xPbMRPT7Wula93VfEvFOCnjxKyrW8CR96vroYcT3PsnC93yIovT6uFL0k9gW+Eyv1PSoQlr1OlvM8yFlYPWrw7T37+S2+oXk4voBrSz4k5ZM9XihDvcGhK74vY3m9v/7CvV7trDzAkiW+Hs9ru+u+Jj34GYC98mqrPYd9VD6haBa896yXPcxlBb3sQKU6RvEkvcF4oD2hQj+918vPPNpcI7uQLkW73JQ0vh6TD75cWY48UPMdPX9mPruNPXu8OklnPLKvbr1VYDG7GP0evQ/nGz0jj5O9MWSPvXAyyrxDUgE+XAJuu41U/Tz1UDY9t1nGvF5tlTyioLA8G/yHvdb61zwsci+9F0QevT7LY71I+Vi9msrcvKeJlLwHajI9CXCdvOXvbzwtrKU9D71CPKQYmLxY5FW8lJoRvWpltD34KCs9GuWDu3yNAz3LdhG+c/qgvKb0L71WPgO89OKYvVzXoD3Nw8a9k3idvbENhjxd6dO9+LFPvVZuXD38MRi+","w2cePn8/Pz2zSLc9h69Ivp/UTb0u22A9xE61vW6r0r3oAqm9O7GAPm+p0T3UmSq9jK0uPRS9Ob0Kj0W9WK0HvK/cmjyuVlK99Gs4vfYwrL29HOY8RmeBvQM2g7137CU9FL0UvhcN+j2UdtU9wKuAOttsKL5kR4y9Sd6OPYLv+j1XW9+9jEDZvaqhCrwYS4o95gTyPRDHDT3RZwa/65E4u1136zyEfaI9CcywvpzwELqXYgO9oTIVPUmq6j0LVEe88GT6PF9VMr6Hg2w7GhXiPaWQs72rd/a8ZFD6vQchrz0e8NO9AUIKvvkkCL3IwC4+BZl6vexd/L0oqee99NosPopZWbwY9+66Gzh5Pa11x7yXm6S9EO73uwZbib0VxBq99ZSuvRHFPj4r4Jg94cPvOw0qhbzWOxM9lUQHPaGpez3Owf09L5FFPsv0wj0ra/g80O5zvaRpgryQF2A9mjJOOyQCpT3UsD67JlJEvfKwVj5YiAa++myyPJIOG7yM3Ic9CvLCvGSjX77QSg4+Sd00Pdx0JT2a4qy85j2MPD86Qr44PLQ8PCyCPAoEx72RtHs8klxwPQr9yz2IMvI8/ajgvmnyBj6l/sm9DBf2vXuaIb27I4K9pXg6viEOojuwcFo8UPeePazn671X6oG9aYB7PezwBT0HiZq8arMWPjTb6z3WnBk+HpWqPFr85j2yXfI87Lw6vfA9tT1fWIi7jooDvKcCrbzHgIm9ETH3ve19Lj0ODbi9IBrpvMifnLzHUTS9KUXRvHb1cL1QcDK9qfo7Pikva72R0oY9RhoRPrrhNr2vJtg8VG6bvf+CzL3owyE9poYzOhLPTbznv6y8E94YvUy+eb1lwp++Ai29u7gzcr3IUwM+s4+2ve/WbD5obwi+g+tSvGsK+b0LDAU8VoMHPnZ2jb2g0zK+DCByvY3wKz22K1e9FwDFvV5au70+CyY+ZTgWvmnIUz3xEFe9+ZcHPRdGoj1ubfw9eOmjvDGLgr3n+ak9MsFAPdVDMz3rZz2+","2+95Pc3FCLy29f68C79WPLNpdLxdTKg9VPJLvBWN87z5Tc+831frvNMmw72/RBc9Z8C8vcUR2zovL8m9N20rPVEb2jsvPui9rIZCPGqkLb6CxzC9pls+uwHCzLyhA+69zipCvHGz+L3l6C29F8rRPXtIpr1km/c7WQ89voHlRL1rmba7K96Avsy9KL130ky9XpgvPA6RaD33+Xo+1uNRvXK9N70TJL49dpUsPjdS371N+qa93+4lPGLUrbxEziy8+l6iPcsmFL1IqXW7x5H7Pcnrnj3FyKC9BTGIPf2eEz2/LqK9tLAuPHKaxjq9NNu9tG+wvSoGQL2rFKq9e2Aavg=="],"bias":["rwTovxJfID/TTYe+RDlPP+hlfr3gHO8+fvbOvkS1hT9esCzAXC01PxTXiL3e3k6+6q0aPxCKXz+XUa6+pQl1vwta9L47Uo299uRDvm4b6b64v98+KcwIQEzwNj7B0+0/aE7aPpCJOD+x6iC/5VkNwMTZrz/W6wW+lPIuPdyGHEBwZYC+Gf0MQI0njD8MSOk81L6VvubZG78oNGW9S4AdPxD98z4JsDy/7qsyPo981r8tIMQ+9enQPlf8k76R0jQ/ONSfvzseMD8iJNG+yXmRvzfAxz8gw6A/s+uMP+yJij9t4pK//BHsPiHBnD6bMqw/Lb+kP11EFr8icRQ/apVUQA=="]},"dense_7":{"weights":["DiRvvQ8eL74E2ri+ImZkPkEJVz9UmuO+CnzcPn1AZb5AIzS/D5ErPx9In75ymgY/N7qKPkTl+r6ysFy/fZ+vPTUQmD5uo+g+783SvRxEzL7/TOq+7uxyvv5TXD4YAxM/rQsov5V0YT4rLIm+O2sOvzG9cz8koce+AD71PqBodrwqoZ0+ubQhvmTwjz6Jduo+S01YPmS2CztAAIk+4IIov5dzMj+GY5Q+0/9ovvd7Br+SJWs+vpmHPhZAPb+LQEQ+ALSKPpe4kD5jwEm+T0COvQHxqL4+36e+rMEdvzsTcD4aW+Y+AWZ3PmxxQ764Kyq+ufsGvrowST5dtjW9C1wZP81lTb7P5wQ+OeXPvbE5Tj2bK1A/PjxUvuSfdT4yhVI+Wy2tvk7cFT+GgNy+4pnlPVRKir11gJ+/f7r7Pg5Avj5QIF6/Y5Y+vkmjpT4x8te7tT1EPnrHBL6Td3k/snb+vhL4zL7TN+M+jQJVP5JlXr9S5qY+6sPHvtV7fT5l+0G/ag2cvLJCRj3n7dO+TN2EPRmbNb94OvI+V7GUPnjqbb93JqM+Xc/wPtqA0j5vdSG//nD4PkD2xr47FF8+adxbvQyy7b6RuBO+pqRwvuxgpT0dvJq/x2uqvgp9Mj6TCt0+wuxvPk+OBb6p4tq+7a9Sv1u5QD7spIA9TfGsvjF1ET/bjQi+usGivlrkMz5GD1s+lfP3vfK+PT4F+RS+mn9xviHUzz6FyvY9ylG3PsvnCL8hr9E+Oea+vU0O8r6BYTG/idz6vcQ4tD+bYqg9W03TPr2HvD7n3LE+5EQPv8yIRT/3I789Mgxwvk9xH75qQ56/EaUSv7Pvkz5M7MW+u7Isv8xVYz6NM7E+d2/VPQ4W373AWgw++/i0PqWc4D1D1jU/sdUmvpjW/r61hhe9l6WpvVQnj76i2Wc+z1rOvhwyfT4sty4/M8k7PTuj/74mTAG+ZeEKv3LUEb9pW609W6n9PcX/vz5MvOe63RsVP/+PST1bdQo/QU13PejWT74CO+W8","PZ+LPsKYS7/lHUK/8///PY6iEb6Mduo+NjGtu+53FD/KCw4+D75/PtFdAjzeyAK/LhshPZsDBD8vgS4+TU5jPvk7x75cVpE74lG9PkxOez56pcW+dSRgvk/z074gEM09ddsOv+atpL485bK+IKsjv3VwZz7Q4AY//nCuvpqqW79m0/g9CZf0vlokDr5QNMg+NJFdP+922z7AmCC/joENPjCB274G8q6+MRxIvoF61D6Ar1U+ryaBviA+dj72eZQ+x5dPvy59n74TbCy/AKQ1Pnc04j8V6w++1pi+vmr3Ez6XiAy/ddx9Pm2HAD/2PoK+6/n2vP4nu71+oCE+GIYNvtyAAL8fRcY/PDY3PkW3E76DrSK/Vb8mvxtiJr/fRAO/fLyFPuKAqz0LTBi/uCnpPhJaB756tHQ+zA5GPg5jz79P4jU/3gsevznvo7yl8YE+031SP8CumT5xc7C9CH2Vvtr9eL2kQ+w+HPTNPlTEy75Nru2+2hpRvrDlpD0L0yA/UaWoPVVhLL93+s4+ebOQPSiZeL5fZWW9jlbdPpo8HT8ry1U+ug2/O6PvR78cwQC/4clCPohprL3LxU2/rz+dvl4hOj6BY3K/NOFkvy7E3L4QaXe/CRU2PyAKij7opVg+9ymAPkPJEb+9wse9tAZdvgHY775bEF2+s8CHPhI8SDtNE+A9vqHgPowXNj19pNm8dxUTPk2/6b5qL/++zfBrvokd1r7SPgA/kyhQvwvhvb3nsmq91SLyvhEVO798+IK+feb+vXantL4z1ls/YKHnvmJoRT4KfXA+3ZvmPm4XDT+mmdC+UQmUvg0vgr6uVbU/JpwOvpNUKz9KjD49gwUXvpx+xT75ThU/yPCqvnlh2D0QsY4+WjMJPpNsIb6g5jS/JgavvY6zqz7HHEA+KPu0PfwuFLwMXag+zkFXP56ex77/Ir2+OtuFv8u9wT45vnW+E4i7vX+Wnj4IdCa/Ykbtvh7ZL74IwKC+WCXZvq/1d7+47Ns9rZ+SPsSeM76EzIU/","x9HPPrJ/1L5jpV6+63GPvof/vb5qhAK+QkQXPvXYqz5023Y+vgTFPpJFoT6Q79++ewz4vUWm575Ig7Y+Sz2bPtzWE7/zugg/uubjva6+xL7PQOm+IXWHvbMnJj52VXs9ocm0PoJzp77Bm68+kMlwv65cmr6ZCUK+DRwPPynZJj8M+IQ+LN0sv6kMR7oq1Bm+9EM2P5hXFb+gxIi+BLM2v0YrTT9IskW+/1w+v6xnbj6IK/O9utfhPsORHj5KiFy++D9iv8ofQD5Ka/c97M0KPmBbNL8PSQW/ZH4xv8h19b7DMEi+VQgBv8eyBj/MHtG9bT2Kvh4KsL6/TJ4+np4yvcBlkb4462w/AlcAv6H3qr52uRC/euoov4ANtb5BpQ6+LuYpvjLd1j5crLi95OwmPlJzxD4WQ60+2Sx3Pd3NEz+N1aY+FcskPuGqgj58K+K+fwncPr4XDb9aAx0/2R6iPtk87D2njpu8YAOWPXJ9Y72jfHa+sqsHvmIpij33aBQ/Wxbjvp3Lbr6rWBQ/7FJIvhagnr7bahi/kH8Nvuixaj7HeVk+cvn1Po/dz75BSGW/5TgLv+gkuT57AFs+a5E8vfEWvL63xGe+PyE0v/A1Mb4TlO++lSJVPpHrsD6Ofja/tK6APpfZFz7Xe9o+CrxWP/PvrD0ceFm9KC8pPRZCHT7DE7i+G9+Mv9/YwjzR42C9MAjRPJ1EVL2ayMa+o3i4vs1Vhz3MelA+UeiqPk5E173D/mg9Jdo0vzwvvr4PKkE+gDnqPpRzmb6ibAw/JNDJPjCumbxsjuc+iZ8ePg7LHz/x3gA/nZxBvvuVKz+64JY/97mJPg0yjztAw+W8R8Pfu/1T4b3t0R2+e94GPjW6C79fHp296bAzPlhIzb556ca9h4CGP2wz4r7mZoe+C+YtvuMmob5J5pS+2YspvpF3uj45XF0+joSdPkdeej71sA8+UFBsP1Qb7T6KQtc/2rK6PoMUj77WeGG+Rh/fPtJhDT8T/76+busAPmfLQ77N5Bk/","CTh7vC/goj5qqXy+5ecQvsnvxT5JmKm932JQvgB3fD+5O5S+L8UAPkymy76MgUi+43a8Pj7PKLynpyg+FPrzvMNMp777cMe9qqv1vbIwGz5bZ22+jQM4Pg4GlT41eSQ/pF8Kv/VgED3zBRC+id4WP/KSnr6FHDI+8I7GvvaTJb4/vWm+9+QzujaVPL6ItTO+MQ6HvvNOD79/0s6+J8AbPsSyiD4w+XM9TK2/PhezNr+htM69OYk5vp/sVz06ILo+Yd79PXiAYr4xwFs+9K9VPVzUD8BQ1jU+21HhPtm00z3q4Pc8dPOqPiUvBr+FoBm/oRXZPTxVRD5T+/w+71E+PhKf1L2AgUm9oMNAvmscvb20mtI+BCocv3DMSz4uY3U+SJGDPo2RhL4mmSq+iV0Dv05fab2lOKW+7a8nPsYqfL5xqoe+0W0av4EtAD8Vc9o9B3aaPyIinT7hgb4+3akuvvRysj4g2gS/mcbbPikmgLwaMiC+ixQfPr429D32sv6+27/HPsAzbD9+CJK+HiaWvchXqz44QaA+6NwDP6OS5z5nfjg/IJWgvtI4gD6ELzo9/8I8u9qLej1atJq7oUOivrqGTD5ABb0+CzcPv9UMsD7/GUq+qIgJPojOwD1vidU+Tut5veDrdr7zmOK+qe6tPiONjj6o2pQ+i6tkPpK+r767l+M+QjyjPj3WOjzusPA8ctfMPjRLDj/5HFW+NoQLO7x5PD5W7HA+YCc1vmagUj7ZnvK+7lO9vvEIDr+lP2S+Kv5ev8FeX79/8gS/w28PPmgddz8UNyy+70shP8BLQT+SHp++3zvavpEDBz/mNEs/iT/ivh6fID77p2g9R3KYvtBI/jz/C4U/7Ca0vgZcHD+hdHE+Lh+OvK13OT7yfBa/HpqfPgI2p7305iY+al8oP1XD1z6nFuE+N6GEvsQL7z14Qi8/qN6vPReeKL+WKKc9mKJ8v09R+b1AgLe9aU8UPoNMQ77BXcm+WY/HPus4YT9GW5w+lgl0PkMxi74Vvre+","Z9TTPlTjqL06gX2+LfGvPZw7wD43n7Y+bN51vpWEIT/abWo9QAe7PTXMlr4xIO4+31adPk0r9T2UBhm/eggiv2wuTb5O2Xq+JSgfvxGtwL3XDbQ/MispPvFCDr+btZ4+fzXPvvWTqD7NiyY9mw8jPwGidj5wwMC+oe1Fva28575TSoK+o859PlWQeT6Yxl88mw8VP625eb+NO4E9ZT89vqAF3z68s5m+2jbDvm679b7EGh4+dXVEvpo8Kr7s2Fk+60Z6P3wB7b64R+u+cr/AvtO6tz/CtTy+jXdsvWxIEj/MuQE+NR4KPiu+oj/T76Y/OJOgPtkK7zw9lDA+8VaxP0ZU5D5+Zhq9jcrwPREXJj/TLCw/6NMDvpR9Mj6/aT6+ojsgv2FEiz85xDu+rQL0vVgip76q62K+JbshvoW85L4phFa/vNKbv86QwT7iXFO+uQ2nPTuZyr5c0/w+eehEP4RjnL4i4wq+kzUrv5lPvT+GHbI+PBEiPXvFGb5szF4/sZTePk+XHDwaRdw+IpbgvhqQpb7yed89/VIQPrN9qT5CYRy/sHmBvuRrEr8pCg6/7kowPgnlX74XhLo+XvsBvy+56726fbG/E/UpPrGR5T1xlDu/VppOPiNvcz6ubDA94AuQvlqhT7+c4B2+zi7VvUBIdz3uVAM+GAIBvUdqVT72Xi++g6NSv28GsT4JatQ91BFBPkb4UT0QNJY+zE4kPzGF2j6YMAc/c71Yv8Es574cCUS97/pJvuaOMj7Ar1w+THFQv32cfz6qUDo+ONIqPkigy79TNM8+5KnlvEhVn77+dlY+zLqTvbGefjvHGFe+9wlJPglmKT7r26A+u0xjvSn5QD5+kwC/8m3aPi56gzz+wQ8+mOM8v8V+Lr5F5si+mVIkPhBJrz4arqA9US06vviBmb030AC+GYPCPlDOH78PNOi+rzrSPhSLELyMH5W96AeBPWOu9z7LWWe+cLRIv7OP0L5Ct9c9rLHgvg2/EL/6Mlm+tHNCP4K/mz7i45O9","3H7xviB0Ub8cpLE+rGMdvt8sAr8wwxA/CXhwPjsdZz7zfL6+OE6qPSEu7L681oy+15d+PhNTtb39j7o+Hnf8vv4WAb9Z4V+/b18mP2KEoL7O9No+CfmIPD5J9D1UVDE/vFzmvtP4Bz8Bhfy+I3+Yv81VE7457BI+HP4ivmNzHz8OP4e89+MXP7/mjb3lm+Y+7IYav6lQVL1LzE2+wy4kP8V+nr9T9/M+M0OIvjOAvL6ZvOo+03LPvr2DtT2c8QK/h7X+PicDO78/pcO+DTfKvormvr4fjxI/Tu7UPhJ4ej+c6wa9xg3lvVqaLD8qo12/iBXIvvsaQz+fsCe9GHk+vmqqsLzmosU9Axh/vjTpT75PxVO/fYMjv9yExj6uE2M+x+eNvtP2Uj7f1p6+1GGTPiMAQb5NjCQ/Sh2pvgVk5z7rYWS+bd0Zv57srr5qC3m+RLuZv8fFSb4Z44C9kvCrvq6VLrxpzRk/KKqkvjWgLD+Afeu8RtO5vUjqr74mFRQ/fKONvkZhsz5xLaa+tqZ/PtcGRj7jz+y+Iia4vp1Shz/nD4e/qiVfO0LLiL0SjgI/D7JHPSo6XT6Xf1M/bWtEvrykIz8fdVY/AVz3Pi6OGT6BJ4q/0bhWP4Ou9L4RT4Q+ZT2OvNH3gT9xZ4K+B2k1PiE+iz29rEA+CcXHvjLowDxxQTq+zelVv9Ezyr5ogt8+WwPUPmuK0T1eQh0+a4sCvwP0nD1pIxm/Sz11vv2DH741CUk+wN/ivnYjDDzyG2Q+BCxiP2EuPT+kXcu9AIzRvuEwZr9nZo4+IfsDP8sjQT52kQm/W7CDvohEE78ZuoQ/D8wlPjR2dD55KLq+JEdPvpnJ2T4+PEk+VMfQPgcENT54a4g+QR4ev5YorL7sbIo+rO5Jv0PXnbzLiE0+hzccvcSTd77grf4+gK3uPZC1wD3l/eK+k8oFP/2L+T67Uby+x4Bvvuf4k77ycik+4clPPjejeru/dzi+y3dqP++yu76uLJc+IAv+PSsFmT09Njg/","2Wa1Pkfi8z7m2Yy++VWxPqJOIT+1z4y+0c0fPWG4pTsWq3Y/rMnXPl6xE787sak++XuGvlnEKr/yEB4/u/PPPvpaY79sIVO+LIUlOxuvV772Ax0/zT2qPl8aCL+si0y+GVsmv+DHkr4hHAc/PqcsPym+hjx5/xG/jTaaPYOl4L6RUq09Ljx5vzqn3L0NxzS9hyVlviNUqD6xb/a+6tTiPSQhwr1wHwm/YdzZvh8Ivz6I7Ys87HB7vgh2SL1+mjq/sGh2PvWZar4Khtm82VhoPiEwmj/RiwK+Sq5XPmtoCj96yOU+2Em9Pnk1UD9AIIu/bH5JPuJ6r75rw9k9uNwXPxw8rD5M3AS+gtbNPj2XiT6CKZ0+JEK9PmouITwzz7S+MebVPgPNaj/iahM+5DxSP9Dqar0AIHm+RASbvmf6PT+GYSa/0D4wPnrs0L6KrwS/3cUBwJJvbj5BKoA/Xu8tPqIKGb+22/U+rfn2vgnTsL4kZBo+a/oBvpe33j1SLBm+0ngBPb7hZj5qwsk9EhXZPfC0IT4gtoA+BBIhPhpzg7/dRos9QqOYPkMYaL6zAem+iz//PeEyJb/1m4O+DAWzPhNH1bu9D4u+mK4fPxCLr7xd6N6+fp0mv29Ysj4j7wq/ZiCOvugipj7+wQ2/T46SPgjWg762WCg8CNKpPnM8Z78/nAU/OPKpPnlcRT2zBIs+w2Zdvldilj7pSr68nyMiPrstQD0g4s+9casCP/0xWj4H4hE+kjTlPtF+4btX/TM/MKGGvzAa8L6bE0K9S+MHOw/6N77Njty8uTp6PPKz3754pTA8iSDzPpL4rr6Ewpu/WEPWvnJDnb5G+UK9m1auvhgY1L17rwE+7ZiEPo69Qrw/87u8Z9cfv9NLmT4khQ2/vLkuPzpeJL6G3Tk/v4JnviAj7L7l8eQ+I57APZHvTbyPaMu9I5kwPdkSAL9b5gs/fa+FvgJHpT7aa4y/8VtVP7PNED5CQnm+KyHjvuCJwD50vII+OwqNvvyQFz5hV/U+","NUSJPrvzDT29+ha/JesHvk0jzD0j06I+wKvZPnpzkr+HJbg+UiMTPypEJr9oZxM+tzMUPzMX/z4EU8k8c9S4vr6dLb7HFxE/uQbJvpmfMb76fTw9fpxbPuHhVr4w36e+YhLEPrO+6j2hdbE+4vQsv3E1rT6cQcW+JU58vs6ukr9Liua+SDqOPTQQvL5wUkO+leyxPhYtmb5aS2k+E/tAvzU4ED5vGwm/BNBavjufBz6DbO6+ytULvwISLT5IikG/OQnPvltCq75y/uY+7trXPkP+Nb4DWVe+gBSQPjELFj+UjtE970fCvsj7LL3JTJM/tUYEv0RlAr8mmKU+3mEuvkJ7xb6Lhk2+g+rWPlfMFL9PRiI7yx3AvjA0677zkak9CXpBvzkWRT5WFIw9IurcPnnN9r2t3yk/dV/cPlkAMr9QaNg+HWBBv5U9ur74GwW+UReMPZ56hz4TBSm+PJ4ov4QRAT8weu4+N4nzvpnhrj9g6UM+cl3rPscPUj6J/xa9QSrHPhqmZj6Aetk9lpA7PsLOnT6DQJw+pik/PSigKz/zDGM9QIkbv1qrCb6mX0U/8tWDPq4fNj5dJyO/F3JTvWXZ6z2kR3i+o/aCPpp8Lj15Pew80p2Uvl8qjL5NqJS+QgAbv8OBlb55gMU8WHEtP9/eU74GQgq+R2jvvT7/Er/Adlm+EUr9PrF5Dj4/DGk+eROuPtXSOb8azjE9Jx3evsjOID9ZUoc+guOqvlPTT71Bo64+p+zQuSLPlL78wZG+AykaPxEWiz9wJQc+bGEHPQCqKz5OTfa9w9yBvRmOzj7tEtC+necNPvyekL63IAS/OKBXPpEbFD8yGTg94yFWvRyUzT4E8MO+lCWKPSzHJb/Io7o+MBrvvju+Pr5Uojk+OMGCu9Ln/Tw4EAU/oT7avlAKbL3gc/A+/I6yPoHNPL4KuHU+pOUAP++17b2ur5G+LIyGP8qUKb/a/CI+aEjoPulHiL73LtW9sozjvXYkJj/brkU71ba7Pi3UJb6vgPA9","ItOoPioHWz8gGis/kqkLvxw1oD2I9bS+dU+MPnd4WD/N2FG+EetCP7yVUj5fL7M9aWNqPrNeEj7MhPQ8D6cIvxF6eD7UhTs+C1ldPgIWhD7FchI/u3TmPnmtrj4TX5O9dyaUvipU4T7B+nI/0qzdPokvM77zQYA+FASRvtGR9z4TW8c+Tz6YvpTMNj+AaXE9aJqbPjsACL8xgZW9i0/fPUsPGL8D/Ye+WUzRvRuuGrvsZD+9NhFYPa8kNb7E8T0+gA58P4H80b78HQs/JE7Evr9Lvj+0sym/DoMKP5f/rz61siQ8WJwgPOQHBb96ToI/G1UMvfjj2b6ujc2+MIT1vsMZez1VXoM/VZI4vgcP5j6pMPw+BRfNPei5xLvhNT2/zDwhvnKXAb9MGk8/adaIvn2mkD77g/u+qeedvmqz6L0DI9e9pRsgP/Uzf77rzdm8c0CPPhYUEj+EC4e+gPODvowQDT8Ljp2+xfcWPwkEkL8540S/rDUYvbhAoD2coui9xb9zvkGxkT4ElVg9lv+Cvu/yiD2CR/K+7w1WvgaDzbuSNwq+ttqxvRjrEr/pvwk+LWxUPsnN0j6914U+hBcovrkTLT8BWU+9QpRKvsCLZ7wJFeK+JthAvvb+k750jza/ZPMSv76YXr1T88W9ZVFnvywPGz//kBW/9J0XP8HnvL5/fKO92+OYvuhKJL5xVAY/QrhMvR3qHT6NGhe9aujTvTZFR79ww8U+DdgvP37WBT8X1gu/+o78vgNkE76kyEs/MK+Qv/z9v74LJt8+MamOvp8gPL+jHx6/HfEevzH1Nb6qyAQ+EVxZPqS0SL8YJIm/Ro8cvbQkcj7kSOa+vY2HvtDOpj0cM4Q+RFuqPuXQUL1hplu+cFTuPmudNr8A2Go/Gt/TvmjDJbtkdPA+Kwb3PitL3L6TQ8k9XKcyvwZf/741FU6+6F42v5x6CrxYYhY+W1Sav37tgj59cRE/3LAbv4+kVz7rPgA+1teLPefVqj6pfMg+1BkHP89CNz7Zix8/","ickqvsJukT7cJkI/fp64vpcWAr91M9g822iRPgW7uT47DTo96YaaPhobhj4HOY4+eX7MvV8dMb/EVeQ+GUZOP7h6IL6oATm/hL4LvYeqiz4mqUs9Dg1lPsBYvj7plQc/vGwDP9TCuz7JyvY+xxMwPzIbpz5OrN2+DwljPPwbzT6Hw9A79m0ZvwascT5V2Fu+ffW4PiQNgj5LA7G9RRemvtG8Pr5+aj4/JwCDvlyNGb9h0J29INiTvqokhTx1Wvy+TYhFvyYXibvwcgq+0zOFvsGMGT8APyO98PfvPofe3j6FWIC9I6LYvn32vz0ZsVY98RDEvX7urD5bzIQ8+4ZuP5B/Ez5suJA/OtSqPkSVzL6RjBW+88S4Pj396r5E7UK/KV4XvmQijT4vZ5y+CPffPv5HKL4JwL6+FgfJPi+XBz+/K/0+AvaJPgwXG78CAMa+wWisv5iDE75+JDK/sgMOPx1yCr9InO29nuh1Psr7YT22EO4+qqoVPknfaLp3ii49gfGrvJ5C4b5r9jY/Dw9Cvnl+Pb5fuPY8a0AIv9B1lTw8Qma+mTZNvwfkKT5SGzu+7vW9vgZHBr6i2WM/Ed8gPupzmT1PDxu/EDnKvgAryz1KvuM+cvbHvvOZmb7czNM8HXlGvuOyAD/PSDQ/b5Y2PxYZGr5bbZg+xZbhPmPajD0BIGY+ivFEPqx7Dj+ATJm+98uJvJWelj3C3Gw9ic4gvjLbgL2IwFI+iOZ8PqfDwrxT0bW+Z6TqPs/gzb5KaAe/eXWTPAUxWT3moYO+7XpGPkGtFj+Fspg+U4abPgTN9729z9U9/JOePlY2Ar9XL7A/x9scvrNe2Dr4HNC9TBHjvVsfCb+yK1y+jECZvvyRAz40jAG/pCNovsKdGj91tQO/q8fFvoITIT5sp7M+F4tBvx2OgT8Zr8w979uzvdizEb7XxCy/0+W3Pjg/ar5J1yK+iKxZvsRMAb5J77Q+lOuzvQVU8j4RonM9n0Z4PjibL79jm+g7IpIBPqVLQr5nLUq/","6L4aP5N1ib4h6vA91UuSvgt5Or/NkU2/5v1Rvm8TqT2Qq4g+zMrzvX1KE765ELc+7L/Hvn7nMj9j0Bo+p2hzvgFafb9UQTI+3PIiPTNAGj/UZLc/5/hfveWRD79psmm+J43bvWbII7+n9yg+aOS3vp+0sD2c7RO/r2UKP9qeLj+vrMw+cUlkPt3RLz5Gq7m+ucfXvq3akj4LULG7GsgaPR9LHj4cRwA7kyLvvjbcJT+1PqM785/IPhJ8t74gl+O+ieGcPx3k1j6oQTq/7IusPXm5HL/3vvS+D7Gfvoi2Fr/xlXq+/i3IvZ7cjD792pQ/yLaFPZ7B/L4LKlk+LDD8vlap6L54VYo/IaWzPiwXLb7IozS+KsivvturZD4ou5c+9S2mvVyrlL6fbH0+FRSTPSZDKb70DVS+Xa3Mvn6YoD7Ollc/tZEGP1aMjb3KRTw+XZwEP8m+4z3nx1E/lCr8vQbYZT6bg4g+a2FTvdp6LL8igLo+2JynPESIxL5orrA+0iCwPdHNC79rBQQ/V4xiPptLVj7YNvO+bqQqPimOCD+ww2u+HMj8Pua8JT/D4eo+n747vlMbHz/mkvi+2NLFPm8Aoj+OVBi/UQN2PegX975LWPQ+i2nBvTeJHL6jOPU+gYkjPx6g371XmNE+87tFP7aZIL6VhN8+3pLIPa726z4="],"bias":["+8yOv7dWR796dtS/dBTNPriyyr9D3UFAJoyCP+b4+j5s3Nc9vG86wNT79z8ZI9A+aHk6PklDcD8q5Ss/0OfKP9eosD9ysyBAshyRP+VAcD/hHog/LcgLvqWK8b93TCjAEAfJP6u6/L2RksO/efECwJUPF7/+/z2/+DMRv8dFpb8="]},"dense_8":{"weights":["0L0NPi/uAL0zbsU9XcOpPdOMLj4eBLw9MpwrPN5pfD10n7U9GccRPllUXT7AHGs+FIS9PUmszz0Xiwy+88+YPShvoz2bD+c51fc5vZb7Or4Goaa87uepO0lbEL7Kni0+iVxePXBE7z1FUvk8DSGMPXT+fb4B2bE99/bePVki2j0gJiu+5qY7PrrYaL1+klM99/sevdkWaT2Bvwa+tGebOwHzljvtFoo8n793PXlkxzrolG49sDQuvtEj7bwpuve8cLcfvQKPFj7vU/M9cSYMPUE4PT5Yugg+6T0dPlt0ET4VzAE+FTHVPcvZDD5Wk3a7sqEUPmtheLtioDs+b9OIPavtNbyot449++QLPCmZHr5rnBE9ZmEivn8vXb7fOmC9PS8SPkRWSj5yDMu9HbalPv/NBb6x0/o7ZB8iPmdwwr5pj9A7uLPLvR7HXz3Qrku9AZXtPN1koj0RGOy953PdvPbDRr7cg7k90CWIPtZUSr0h7Ge+zftMPrV/IT5FU4s95mfdPfWws71/gWi8No5lvaTqND1aNhG9TU2HPveXur2GP6w8CgX8vdlzFj7smZY9veDmvNl+bLqtc3Y8TUAWPpbkrL3sLAk9IVOhvWURqb5aC0a9AveTvVA7EDtgtYI7iOzMPVW6Nb4SeSa9d6HJPZIKkTueCgg8FayHvfq2wD3/+Ak8BuWgvBpx8b2R5ra9A6YGPuow273Q8NE9/t4SvGc1fTzoPdA828bOvZbPrb2E1ag9OCJKPcZforzNZ2w9B+fSPDkzFD7rSw2+SKtyvi+N+71fAmc8rOSJvG0HALwrlXG9eJAgPla9L70pxYs9IaKKPY/ZBz45wXo+Kmf6PUZYeju51Ka8e/hWvQYXcT3YtBM+XL0GPUuZvT0vGNc91wimvWOS7TxUEZA9z8rxPeljSL6hCPq97KeJvbTrET5ejns+/NilPcxRRD3+xg69eYGCPl/Xkb2Syt88EaHUPYSjbj2W7AY+CWm4PTpv6Dyf3e89BojDPQonv7yFyic+","fwjBvHdkhz4qNlo9D/a0PQV3Vr4ktRw+OeZDPnYY9b3GUSa++BCxPcVIFL0Mpvs9EWQfPpkWWz4o61o9jOozPnTWWb0CLBc+H7n8PaqS0zzmKWk8p8TjPA12uT1BdOs9bf+JPXspeT13dnC87lC1vVnsY72q5fA9r0BNPSnD9z1kG1G9rctfPsKI1L1MZlQ91LuCPcf3Db2DusA9yBzCPWjOg72WCIE9ruCjPeAd+70EaBw8MomFPQ0aaT0p/mM+WjzqPUn7tD2Euzg+eN9bPlBquLs4gg++REMCPpFTPj2n6bk8pva+vQK4kz10LqA9SBk/Ppjhmz3KulI+KlS0vUfAjT0xBcA9RTAZPNH6sDsC++Q9nCPOvSY1Bb2b/Ok6L8GUPfHuQD2W9rc9XxFFvakuLj4f3GM92ALVvQB7Wj0bLrA9Mz3aPakxQj0qxIY7IL6ePFs6Gr5bxpo9wJInPenPuDsP8jW8zKfrO3E52L3a/BW6b1RjuwRxXD61lqy88vIMPi+YET4ALUW+J34ouuCh6T1xWRS9Z+zfvUpm3T2WbMI94vcIPV4QKr7LEck9gx3GvcErFz5I8zs9EpPjvORmxb2Cr+c9MYGAvdjm+b2t9N+9lLMWPWKWeL1IhBa+Eokmvt/Q0T0TlbK9Gw01vew8MT6s2wQ9S2inPpklUD4vkM68cRicPXXdNb5g6Bi+alyXvQnBq7v+WOE9AuqPvH2xKLxYBsg9XlCpvRUsGj1xIKU+5GX0vBrg/T1YWwc+QDUNPVEHMT3106+8HonNPfHMyLxoiRe+0h4sPgH1wz1I7ii+Ch++PB3/kLymLY89VCmMPTeUM76RDqw9+YOVvTOL0j2GYFo+jT0NPT6UDz2N4nO9cCjAuwWB5zx4pb89k3DTPLyiVj2ckyg+XkQtvqeMCj70/d09dFZKvaPywT3gJ4k9fwzqPVLNT74FRni9v9cLPdDmRD6tUZW9KRjRvRorSTxvyQY9JwlJvTAT9T3V9go+AohNPXDcgD6abis+","CtGZPTiWjj38ec29hjWHvYMCnD3CIL+8EpsVvTm/DTu+zJw9dd4IPkFdAj4M4BC+we2OPXS8Lz4SP2K7vgWcPelOvz07Myc+as6OPWfo/r3v54q9p/vDvAQQFD2MKV09Le95Pevc+zz9TW08ob9wPXg9Ob1SyQy9t+ctPY1Ahz21+MY9b7h2vU+GIz7BX+K9JANKPhRcwr1r9/Q9TOmxPD0coz1V/yk+pxRAvT4/mT4Zf5i9hA0jPnQX4r2j34y+g6J0Pf/xVT3cUw+9UvUVPCBPS75SPIG8ZG6ZvZXPFD4lEBW9yiasuzXQpz2ca8O9/dpkvSf+9D1CZtM8N4iuvean6r3hKqI+LsnGPPsoS73OTk++mbGWPcRaoz3I15O9IfyMPYdz7z2Pq509qQkzPj+NCD9hIP49+4YWvbIJQT2y6ws+MJ+DPa4TAT0AMGg996wAPqMPYDtfXyk+k/f5PSUPyLysacc9OT/oPcVdmTw1NiC+FFGRvcgLrT1mc6S9USaqPUmwFL5rbIO+MqUWPn98Ij2V95A9TRO7PTk2NLzqMJY9Ywtnvdl3wj0Ts2q91S5RPXC7yjynbsK8fYVsPuyyID6i1ow979jvPSVPnDsK2RU+5ev7vZ/cWT2sxQK9CBwVvWxgFb66IIO9hSS3vBWn8b200jg+eRCNPC9EED79kt+9G2yUPiO2yr2M5nq+2LmJvgRMfj36/EM+Gj5NvjZOwjy8Xjk989fJPPtpMT7xGvg+rjmUPY7urjk1aIM+rusOvAyMMj4WXyK8QTjkPcCY0jzRwAW+hLorPbo2oz2d5Qe+1csxvLZ6Kj4LGFa9wNDPve2LKL4e47U81CLwveoVoT2oOiw+aYCqPQzMnr1FY4g9q8BGu2Idq73XeAa9oHWJvLBh8zwRkMC9Z1BjPZvXxT3fTAW+/LH9vSX8JD6kkwS8X4KyvQ+vDj7uIzw+Ax7mu8zFhjqVGhs+ODUSPvzpFb3gNPc94eb6PZGx7z2pI7a9Cy9EPjToND6UO429","iBbvPZbjQryV+0q8R2tGvtGEBT4mB4a+RpAzvnoSyj1LBFE+sJC5vSr0Nb6H9gi/ZIbCu4PsuD3wMiI+RbjhPT7c6LyXVUW9dP24veTlTL59z+o9sPMAPbQJaL3R8IK9zbvtve7hFj2yBBw9+9SmPZbngT1NmIQ8LimgPlEtQj5jVH48Jf2XveqGhLw0qzA8YfqKvO9DEj6q+Xq8gokRPZTao73MLiY+R5KMPJL3Uj4Z/tw7iqVFPVAxbT6l35w9JSh9vbyrVD0mlRY+AummvT3BUz49QwC+zFlzPIr0kz1JKh89Q+dZPppY0z20OVg9kZgQPUP1NT4xvOS9HBCuPZJLQD798Y+80F9SPj/nVT38hNA9wDjvO3PRvD3LMsI8rr69Pe7G6z1NLhg9N8zFvcmDPb2nThM+PftZPQ7xnLyqbiU+xcwlPZwBOr7jJ2u+h4QRvfTjuj25+xS+xAD1PYg/vT1FBQQ+cm9ZO4jhoT0LeI08KTNJPQ+lEb5o41g+hfkbPSsr07ybF2K+DMHtOw0zbj6hcDi9OCQYvAIJ/TwcE+K8t7wBPXRlQz1FEnO+44uaPRxkcj3H2o+8nQeiPZyCWD5+IEE+oIWxvKC+l70eFgs9eDdRPYgTUjweqUK9cgG4vG0jBr4ot7o981UcPgB/BD65JRk+RJOyPWVyMj6hEyM+rNOFPbufNj5BDpc9XbUbPo0kvbwe3C0+EbPwPU6L5zz9EjY9U3pZvHs9YD6Bk8g8zj5iPPYD6r126MS94X1KPW1AF77bLRU8TAwQPeqAHb4UxVC9AwU0PTYQsLwZakI9/BytPF+T1bpixuq81ZaRvpO9Jj68RMC9PgYAvmRiAj5R3V08lAI+vpjTZTsud0o+p72YPMTakz3OLtg9cB6LveLvULt9P9m8jCTOvfNDMzzrycG87LCgvJ+C7j08IFm9IkZyPgwDir3UFO68c1otPqKQsTzN2pC8aNJovdk7ET5sN+y8jg6JvEC9ID1Yfak8NJqCPajQOz5L2L08","sWUjPhVfL70SYuK8hFQPPhNpaz4pkTM+0U5OPcXIrL2Hqxe+yYdPPT5xnrwP4rK8/J65vQMagb709yu9hc8cPGvE6bwhNJU9p3V1Pj+ytT5FC4a+2W4gPttshD3C5L49SDZMPs8Iyz18ucE9zXh8vYtsNz4uPKm+UD0JPtkoqzzv14g8bW4JurqbAD6/oGo+7FrbPLNceD4WPl8+wPoBPq6Sxr1cVUU9BYNgPR/f4r32GSI85BlJPZ017jrgm7g7j5QMPnUACD5aZt28q9u2PVxK6r2shdM9c/A9Ppc3WD0YYEU7OODxOn3efr1RYAY8CoxdPtGtuTzQnj69yVlFPnr1pDwrTww+pjXJvTQ86730f608GDS3PNVblTyKWyY+5x4EPsuXAry5ezu+p92YvXMjzryvzM89gg/XPZ/LMT6e3eU9/0mlPcFKIL3DfnC9/CZ3ve3U0rvnABA+QRcpvlK1yL1MinM9TxPSvdiKajtbMmI+fJBPOz9T9z279k0+FtQrPkUUnDwP8ty8XeQdPrBHVb2Lfxw+uidsPrxehz2nYri83H34PdlevryUnVc+lcXyPTWVer1Q1pu8a6sFPlH2IT3yY9U8w/ecPREiJr2o1um83X7EPVVXo7wC3io+bzzhOaEpUz0lcDk9YBHJPPjx4zxT2xk+xvGmO3cIHzvFMSq9zzdGvuKeVj4h3J49rmrxPazaCT64I849rJyFPas9ob1GuNu9pvIZPARSjD6mMdG9t5jCvb7BDT5diUy9LMrqvWI9Pr1f4VM+aPBpPSMfuj3Jgby6FRemPY1wAT4xndE9mdrsPagv+D2KiiE+Jv2WvYx9lz2nZhO9h0WSvfvtfD3dH4I9n+kRvqGpzDx5CWa9xbZCPgravL3laPk7I4NyvdykqL3D7gA9Zj4Gvq81KL3260c9yyyyPJjORT56hu48tMgXPmRdMj4SSL09oQrsvuFkCD5ZfZI83V+0vSdNdL0ebNy7f9gEvsR+MDy1DLc9SSHxPfbRFj6+UgE+","huwBPiiBaj6zAAi+kJfHvbSbw707IXO92wKTPbNgv71G8Ri9A/sDvmAfiz0oE6S9kR48PndMTD5L46O90IlGPXdPFzzIDAI+LHEJPmCWhb4515a8VRSjvePa9j0zXi2+DOINveiYCj4XQvw87bDiPYRYlz1Vwae8QuO5vclO9TwUKzo7F5VNPaM54r4ew4w7ElS4PcGldrxfjcc9QyIDvDhyy7w4jgs+D8LIvV/Dlr5fFRq9Y1nhPQVLHj4pmyY+V6fmvZOKjT1hFMs9tQcAvkUeE75rrly8wC95PR36bL24GGM7rmqMPd4cHb6QSYE8rntlPhnBG768+nO9t34ePsfxEz7I2me89FmcvWs8kz36ZBk+M4m0PKxqRbzVdWE+7tz1vTCZKT4flaW9eD7UPHi+AL5CZD29A0/kvfdz1bxLIVc9N+bHPZlBf70fqFQ7kBHgPFDrfTxyifE826ySPYWQqzzJKTw8AlZJPhA9Cj4wsHe9UOcYPcb6AD4VvAQ+H9Q+PvpeiT2cBaA9CHo3Ph0LtT3BpA4+WnTdPKXEEz6BE9y9AfCbvS0QLT51l08+IX0Pvt57kD3h05A95Ex7vCUv8rzAMEK909uovM9GuLxRgZe9SvuePT4/C71PyaC9fiYjPj7l0L3q1EE+Twj4vXJNy723fVE+2o86PtSpoz0wh509z7sPPq/hOD4KXWU9KQgYvTiqSj3YziE+ISASvhL/mb2aXr68vJLWPfvUDD2kKoA+oRIKPpoIqD0D+hE+qd+RPe2Qxj1m5YY9Rn4xvebLOTydjmO99r4IPmpMmz0rAcq8s8zVPV7ogj3758U91CCTPSAgoztT2lG9sZEoOyJQfb1JH2k+D5fTPexsET6xGA4+aKAkPtCjuL1khP09AhY7vtdmr7zU6gi7JHfjPWvgPz1baYA9t4MPPgZHOz6ylb48IOiwOmo4/7pUr1I+dxHFPL1gjDyYMh89kD9EPVeRDT4dC9g9OhQQPh8s/T21SsM9MZdiPWbeDD6AN049","QFCJPqpLGz7hVLo8fEmrPGQmWz0nTlW9zWUHPSBnmL24KCM98DRwPDigDz6vDRM+FSllPi5+1j3/jws9tAWbPQppVT5mu3C9qx+6vWmPHb5OQgS+c7TmvIpP2r27daq8s1gYPonmIr3FfSA+NpSGvYuC4b1aMpY96UvUvP6RTj4zVKA84aaFPYkkPT5eZNe9LVGrvdsIOz15LO48nU/PvESXVj76qBg88WVbvmhOvD6DdBy905+9PXMBK72ywwC/ZiCivX9sVb6Gfl690/QPvIwJv724uOO8VLCDvdtVnD31i/i8eYkyPmHeJT6Jf+Q8b9k5vj1qi7vuGPo97FdXPWArlz3ImEq9wwJNu5OA4T3PHUg+3C4CPmrFKD5DGxy8tx/1PaopFrx7Xbw9ORsnPa09Rb2kSf89GCewPRWSHT6ol/W6eKPbPRZq3j1zF2493gmZPZnN6LwWvyM+jz3GPRDp8j0lU/K91MTKO6bK87uMSJQ9nS1EvQoqKj37Fyk+3+ECvmJnQzyKLaI80R5ePT6Riz3KHEu9eoA9vVhHpT0VOIA+jR65PNz1Lz4PBYc+cd1uPX288T045gC7MaBAvJo8lT3r9wK8S4kLPpW0nT25kJE8gwSFPUIzwLsRgAE+tCRlPTnPJj0hRg0+t1a9vUWvV76z0xw+S9mHPdmLFzzySMC9vGY3vFE0Fj5wEaw9n/6fPQUuFL37p749L2ylPU2VCj7lm5Q9xWwuPjcYhz730Qg+QFYRPkKadz1Ins09/gpDPQF43b2RrII9LLlQPRl4vD3jmOg9NkOnPWydSb1HWjm8bvvJPYBpND4asWg9WahKvrp2DD74sPu7P3QNPnD7Q76MSLi9aiMyPqgg5j0H4V+9ozEcPvdP0L19uU29WWHpvV4Rx73Y/xE94eguPmuEAj58vBC+uAlBPYb0vT1xksi9x+I1Piy1MD4Rd3w+msVaPs8ZkD3vN8m7KJoUPG0VCz3jcAE+5tKkPfuBQT4bWao9qQV/vSoMuD3fO5A4","rYKvPZiTZb3R4bO90l3OPaVhsr3WDAA7cCN3Pj2vP7459ek9PvIuPQ/kgL0E1hg+OHKpPlEQDT7cOA0+NM3nPYXOxb2wxMU9LFp5vUiXHz3ay6i9aUzqvSUWCT2gxVM+JMDqvZOMCzsbmQw+mbQ0vsVSXr5PP8k8qV8KvuColL4YhPO8uwfOukWbBD6KhAI+6YR3PewfVTyqu/49vM+sPVfT4jxriJY7+UW+PHpu3z3r1d49BcqxPVeNbz1M9/Y8oChqPRlUBz53Hes8PZMUPaChvT06X/Y9lBcXvU2ldD3xaLE9+opWvcL7VT6KkzW9EsosvjyS37z/n6k9i7dSvChGzjzDKH49sQ6zvMbPTz65cSY+ssSwPS3NsLx+KIQ8Hjy7vOVwnrvbqkI9RuhIvTz+r72Wgx28YXOfPWgyDT5qNTs+kbLEPdDh+D1ION49R5hqPr+B4DwSbeA9jF0XvLYqPD5rCpA9h/YHvLB7zD1FA1Q9QFq3PVLm/zwpSho+PncNPJf1z71AMZe7B/zevS2xlT0p/fU69LZ8PeLgkr0xl5i2eWDXPTUfz723eFg+EX6DPfd3Br2VZ4++to88vjRnQr3DNBS+aDwJvi+PB7705P48FlQLPYF2eTxE2Kw9rt/GPI7Nl70e6Tg8xQdtvaPDnb4zKv67jD6wPIy8ur0toJm9981RPBaFv70dyR89x3fvPc4ZGz2y5e09h8m1O7Nvhr2U16M99mMZvh8jw72WxRY9rR2ePcJE2T1hkYA+2liLPA0aQz1Eni0+xYBEPkBXvz0BUgq7yLtiPp8RsTug3Jc9OLK3PZIASr3m5tw9JcdjPum69D0nkV28SwUHPvdglT0BEvq9iHhGPgkwAr2sfoy9gQPsuxptYT4++I+9Xdn3PWogRb3FnP28YRl1PmZtmj1T78A8IumxvY1FuLxNiYc98bDtPdtdDr1uEb49trLePCf/Cz7VHoE++/Q0Prkps7yciW49IbQYPl3uOL1CloM94V/aPUz82T1grSu9","zWrTvTdazDuMlyM+U3DdPYoKnz0jmPI9w4kevgCHQrxyCIE+qQQqPmuP3jyBwbA+1o7DvT3gPD1iMz8+M9rVvcIBI77xXCu+G7pkPYdu/jz9NSU+R80BPquq2Txr1au8xnokvUZMPz68h4E98N7GvQGBbr7Crlw+IvkSPkbBgj19VwE+JISzPUVW6T0M7sw9SAD1vO6+pLx2Aw0+wLTQPTeZcjwzaIY8Sz/JPY2TUD4+7q69sNmkvFoexj0nOTK9LPkRPvuzBz7UWqu9gaVKvu6v/z1Vn5S9NCRRvYKXLD42IlE+6kULPXrP0T30O908T1q8vaEQ/j1F0Ce8u+CBPSFyNj0MXhy+bDkQPqA0Uj4cT8k7gUvQPNfrPb2fAY099xp+PYqqg71cuEY+urGgPshpnDsNeDe7kOUgPRKDFT5JXJs9VNUnviffkT2EDEI+L7HpuZDeAD5fw2w+Ym2gPRACqT3DjQw+/ENiPhXb6T2ZuIu9d2cgPgBkg7wwN2k8wPj5O8LBGzyI+Gc9/BTxPW+9PT3B8xM+jpV6Pv5Cij3lkMs9Eg7JPeZC9rs+RJI97ykkPqDinLySFb69lW2tPd5PwLx8KCw+JFgjPt2GCL7sx8W9jZ1oPfqBkD4QH4g9Gz94O5we1r2NNN69Y+F1vSR9Fb4fNeW8rt4wvpHymr3t+cg8YYGOPLJDAT7P7QC9b/0hPfAVID401Mq9sZiiPfZCfr13/gc+Dfc+Pnjeyj2/ZJy9LDGju7LcEz60WA08MK+HvVGLGz5wjY67IOLGPWAl9zw7lCG92wnJvb42E76MqjY+Gi8LPh/7Sr3gHz08nEEovnQpBT4Dlyk+q6X+PTnzPLznz/M86dHWvZ2wLT0jD7a84acbviAVCz0yorM9kOxCPnjY6z3gSES9gSREvuLgvj2zuiQ9A2cGPU3qGD6KyZI9Gxz8PZN0mr2UW8O6RwDuvVxEFj4FOPQ9/V8ZvV/hYL3HQaY9jOe7PFZkFj4mjCu9aujmPTNAPj5zM3Q9","qPvKPdeUI73H/0C8kEerPTbxFD6I/Iw+wMswPmcrjD3+BLw7V2SwPUS7bT4Xv+q9iLwdvUiJQb2kPRi9i8wMPqwRND58w0w+DkwuPDb3cb0wvXi9/ce8Pc7Okz1CQ1k+ZBI2PkpSqzzaG+C9+9YPPrR/cL34tbc9Q+cUPnQvdzxucYY9Qd+CvJg9UD4ibRu+UUC5vDUCjr2gaNm9NS2UvaEJEj4PZyI+SNWyvVD9jz7iqyi9pV0DO3ipAb1Fg7i+zlSbvif147y1Zza9vWcbPQTlkD3Wm1G96p2LvR29vT0Hs7G90KEmPiAF+T0e2f295d6TvW7NMb1hfE4+toslu+isFj6V+V08vC3ave716z2rw6E9dDKjO8CTvz10pzU9eUl7Pi7uCz7bdB4+klxRvSuNKrwZbDW9UcHEvVQvubx8x8k9E+rJPYyRPL2CqW6+SjQQPt9cJb6+TQ6+3V01PkvA/byTYgW++/1QvL3PS72EZsU91ON8PSPaD7wSJ5Y9rehpPfhUHz5nMGm9MDpyvXNLcz5qJOq9d5slPI0yB71CLCc9NKtzvY/mZ72+rxG+VIuJvfWnED6/8wE+72NUvcQ23L1LYT0+76g0vRYfLb0OJDQ9FKSFPBQyJr3ahh2+iUbGvZtgqLzNVjO+OMo1PcT1tD3rVem9xSAqPvZUJT7hvdS9v+wNPrlBALymfAc9FxfTPWrcJz4QSWm923C7vUbmXbyhcow9Y83/PVVOuDxEKMs9lECSPSrx6D1sLzk+TJKEvbCMUDwi2hY+HVwZPhwLQT2GOru9BaMMPgr4NDzD6dk8ycMBPNA9d70xaqE8Di8wvmO3Jj2L52E9WdW3Pa536jyJGxi8Y7UbPjC6/T1GPIm84mViPl175D3CEb695U6ZvJNZAL1Kuzg+0ReMPrJEtbigYlE9iPwBPj/N5D0oZC+9mJ7ePTCmCT1BnH89srOUPaH5mj2Fz/288Z9MPCSaADwBeaE9IWsGPci//z3gDwe9MitNuw2x1z2dSai8","4srFvAEgzT0IzRO+JvO6vERUtj17vA2+tKGKPGoVPj4Bmdq9FiH1PGSG0r2Wm+O8tbekPWF60b2kk6y9msUoPRwcljzsgKc8F5XhvCa/O72ShVk9bLaXvQgH9Lzzm6K9zZQYPnKPWzyVofc95G2JPdswIz2X6Vy9g6tHPupi1T3BQCu+DtZmvnzvKz3ulTI+IEZxPS3rzzz+b30+/IBGPYSxHj0SPRo8YTONvcU5GD0Gbl49XVWHvde0Qr2B9Zw7cRkjvX7Q5rs9F3q9Y1PxvSi1nT1wOiY+8W1jPpW3gT7292s9LOLpPQVFNT7wL4k8wOBaPSUX+TyoShI8A5LLvQGu8TxLoBS8FTL4Pfq2T76qoTA90ykKvmfuFr4Obms9Qu0MPu4ZmLvqEZq9nDWiPp56Kr19WOI8dErGvTQyFr/D2LC9jxx8viGcJ75mSsW9B7v+PW7ksj3igP+8ZvOPvI+kZL5IPi8+8YQrPkQDGrxX5Wi+J+YWPjT7oT4NOuY9YrCYvXQZ4T3o7Z+92xiVvMb/ND40WCc+ahSlvVxqr7wbcw2+FIOtPS2RYb0XMWy9HBQ3vQ3/5L2IW6E9t56bu4mMr72wdgc+CipyPZsiLj42fCI+TN2QvW0oAz4R/BU+fP/iPZKDBj09wQW+P8woPoGWYz6Xe2i798/0PUcWrz0="],"bias":["m6omvSbZ2T07JEA8244NPhXizD3K5Ks8cjEQPc21RjzqpXO84Vneu9DwKj3D74e7ZoRmvZHWaj31dii9eW5hPk/lHT5zYhs9D9Rave+/nLzd7t09NQfOPa4C6T17SCY+vuFSPWm8mL0SYh696Xc2Pc0hpT1LkTK8Xph8vLjkwj0="]},"dense_9":{"weights":["13ENPiwJNT55I6A9eevOPfYK+j1hQ+g95GaZvOn2873cCiO+RAGGvWNKRD3kLxe+iOWCvshuDj0dyHY8KZjTv+CDiz0ayBS8gDdsv79hQ774zM885wDfPZfAAb4r6nq+M3GbviPujr5IUSC+tj+TPtiStj1Znic+9m86vXRrEL6NSgO/mj0xPh9PJL5SGgM+vDbWvNNoxL3xlMS92UI1vh42S75CkbQ9qA0qvSIzSj500Yq+JVAEvq5SLD5itf89xaE2vqZXiz5dx0S+UYXKvZsGMz6qt+S7TlohvtlGjrzozU89PBv8vZmwJj6Ktds96IenvjxDR74KBTk+CJ1hvfOfUr4/ClM+ZJNvPYu/qj3iU3g9iaVAPfRRHr7XZCY+KOIXvhsURr0lbos+gFdhvskjlr4r9mu+DLvdu7+m8b2T0LE89Er0PWeCFD4FvgC+ZeYvvPnonTzR+Qq9CO4gvpx1KT4/2X4+PPT0PNumdj33C2S+Qe8kPsF0z72/nBM+W3eqPOU1Uz0Rl7m9sngIvrSLhL4H7wM+NoryvXIC9r0sPa69prlZvnVt2D3YgdI9lu5rPh1DN74E4Qm+/jS9u5Uger6zTyM8y1BMviLsGr0cKZ09hPjDvQloer5oQYq+xCWEvUSR4r1OWVS9ZtRtPtCxvj1RTdW82OeNvQ49sb1NZ0o+DqGdvmIxmr7zp5u+slPQvYrKNz1Ug2U9OYCfvXDTKr6TDzE+WYVFvmNneT4Uppy+xKzovU0mPz493n48gsghvtnSIz6OJAG+zM9CvjNLTL6hffs9f/cHPmytlz3NeYA+M/iZvXmQ671o/ua9ihjCvYlWmz1/q9+8NEYvPkD9R7suxXi862ZOvsTMFL7efOC8x2iUvorPSD6t8Me+HvTbPXuKtT0B9J2+gAjLv5712zy2CzU+3VLqvWnnhL1xkUe+1m+jvTVv9zwdFn6+DUrQv7G6qbyTQSU+H0ocPgFdhD5OfNS9nkWJvWVaHr5xrsg9HHbnvSzzHj3BjhS+","B2Gpvld8ML1kOzW9wDPUPV2mAT4n0Og9cSOJPq3a4TyHSJS9p5mYPcSetbyPSDK+Mb/ZvFd9Hj5ZtVc+IfNyvtOeIT7FF1g9Rx/QO3LekD1abyw+2gWiPPLC77tw3Qi9rMJsvNnPEr6ViKE9uqc6vrhZJj4ta1Y+YlmOv4nKur/oC869389MvWFIgL/mFR0+x4gcPo0WcT6ey9Y9Wm+Sv4Jr/D3uSZm9BG2mvDO3Qj5a+zQ+utqTv0LctD3pWF6+LxCRv0QvPz7mEZ++NXVcv5K+Ur5vKmq/9iP3Pf0wxz1r0Qw+YcYvvZMQmLzVkvG8FseqvcUfBT3gXTA+24l3v0Wof79qFVy/ArBQO1TI2r3TAJK8YImDPAowQT2fz4W+jzC7vdzxHb7KdFo8+VrOPTpFBL5SoGo9CXmZPXSMFj4oZkO/4D0gvcJlED3L+8u9oJ3zPTD1F76jWEG+yAeIPYAaIT0bBkK9Nky9O+aL0jqprnW+txAxvug2Qz6Wc4A+Lbm5vatalL22plY+euO2vYp/tz3c/MA66DyPvmxUwjxphHK+2nsxPtuRVjyivTO9PHyjPYkpdz1v2w69FOU0vgxAFbyht1C+l5mjvbD2ET7GRhG+KRQxvddWsb/PTYY9GNxkPj+VRr67/iI+7ns2PiPhg78cFOs9HL2UvSmhCz5ZNdO8LLwtvoa/gr90mCS9hkPXPCjFkb9ynMW8uKswPqPeYT0J+To9ycRtv1KYi74wiYO/y19XPStVHT5AvVq+L8z8PMG/lr02E+Y96bU7vPhBAL7IiJy8IH4cuw66KD1ip4k9IjbLvF46U74TcIM9IjUyvahxJzxUqR4+7YfFvnEBMb57vE29wsPNvpzXLj3V0lq/HXftPAF+U76lYrM968tcvgAf4Lx67J+84OYAvivIC74+s1a9eaWLPoU5r71bR+U9YkFWPbzQFD4LyXI+jryOvmTWj75K/ki+tCydPQrxYT1Q4js+bMfMPd1ySb6A5g0+n7lzPFR9hz3nvJC8","vPulvk5jMz7XT1A9AMgLvk6rKL7p7gW9oYFZPUK+hr/TPS6/m4QYvX7rGL3wuGi+4iWDPtlm+b0izxK9brFAvhijQbsTVO89JqkCPfrIGr6aMsW93hDSPYy3CL1nZ4M9c2FEvASQK74xkM49sWIevqWpnL8qg1e/eIovPnN05L4Df5u+4/GCPrP+rr5UYpe/Gp3AvFQElD1GdxI+egOPPIi/pT3LOcK8f9CWPYmXMr4u/DI9cWctvjexjj1wrDE+P7qzPVAwgb3C22w7mernPIdvH7/Btfq8WVHfPZtGhb98/KS/9C2Gvonvk70C00++a1bGvZTYCr7x64096x8APrzyFztlza08eFo8PUJ79T3I7UA+BE5rvoawTr5wn2i8wrhHPg17CT5+9Ii+tfuKvmUB5L2UilK94jkgPoMGgT2j5NY9PAkQvq0hcjxYc0O85RJBO74Gu72IlkI+nC2evQAFGDz6kxy9Jt5lPs2YvT2fzyG+nchIPZmqbb2IqYq/KkUhPsMNFj5dRe2+916hvixOqzvMfT0++gUHvvGy5jyJPP69/08Ivtwnoj27lg8+9Q2ivqKPSz7+aVU8/U9wviiL573h0Yi9CNwqvjIg1z1AuZW9M8jNPExQYb2ZQpy+0C57vn5YU77PjEg9KA80Pn19Q77UqaC9+dugPfaVLD4kKOG9+sakvb8oQr6dN3k+oUYWvmhbeb/cImW+eONSvjviUb+cs5u+C5NuvrK/6rsDbm0+YCU/vsUg3b02Y3G+HH7PPZ/AXb4Z82Q9ms77Ow4m9LoNqCG9erWwvNkBhD7yHPe9uAoZPtrKRT3GIxi+uhpcPpVXbzxrNgc+BkZZvC4Xob1WnAW+kNSrvXrkLj7Moiu8UXPzPTccVrw72q+9eShHvte1Gj4Cnxe++BmvPXu6JL1wI0i+ZrpmvjLDAb8PjRu+ORscPj6GTD1Lt/4812OCvqr5bL7JDSw+sVZNPkKhwbxnM+W8vghnPpykxb1IMoc9yuzEvS2vC70E3Sc+","q5djPjfQW7zBnMS8Z8+5PboMILzKYWY8Vp15vhg2X70/+Pi8vJEWvkGpa74doxq/Oh2RPWMYhb1lLYi+sCt9vZKQ4jy28QY+kTsdPqSAhb0Bjje/12nHvFmIrr2Acy2+/w8MPAG2dz6Be0u+yNg1vrtX9Lxz/pC8yDVFvlfBdL5hQde/M+p1vuK3OT0Hfji+icOSPX8Z0DzB44W7VPJdvv6OVr2W+jc+tOgtvcmD4L3qWVg+nbgAvl1x0L47mAM+0jPoPeL5U71yHbA9zXknvl4QED74AeK9uIAuPikqqD1W52e+Ocj9vMY3lj4UE/Q9IHyPPSj9aL5yoNa+M6yhPZaO5L1DhIW+fNguPs9u2L2W536871vcPWPOQz0dHAi+5mAFvZ5mwr5Kl0Q+NfMgPqnIXb48ACG+WRflPdMDhz2SJVw8RnU2PUR3N76Zhcc962qAPUitr7yM6fK9run1vZNo+zy8gQE+EV1YPBNFOL4EzT++w2mnvjQ+ub0cUYo+"],"bias":["9EUKPq1UJ70zJ8Q9algfvvODfD1oarM88q+BvTBgD72DNWE9OKMHPnQskbvK8+G8dL+jvc1gy7v3TuI92m2jPfz7xb2lcN89ouN4PZ3iAb4Qs0q7"]}},"hash":"184272a2eb952ec311a34648cc702f1336df530bc4d754c384e0141bbb8f7b9c"} \ No newline at end of file diff --git a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_decoder.ktn.model b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_decoder.ktn.model index 88bb21249f..e69de29bb2 100644 --- a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_decoder.ktn.model +++ b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_decoder.ktn.model @@ -1 +0,0 @@ -{"architecture":{"class_name":"Functional","config":{"name":"model_1","trainable":true,"layers":[{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,1],"dtype":"float32","sparse":false,"ragged":false,"name":"input_2"},"registered_name":null,"name":"input_2","inbound_nodes":[]},{"module":"keras.layers","class_name":"Embedding","config":{"name":"embedding","trainable":true,"dtype":"float32","batch_input_shape":[null,1],"input_dim":64,"output_dim":16,"embeddings_initializer":{"module":"keras.initializers","class_name":"RandomUniform","config":{"minval":-0.05,"maxval":0.05,"seed":null},"registered_name":null},"embeddings_regularizer":null,"activity_regularizer":null,"embeddings_constraint":null,"mask_zero":false,"input_length":1},"registered_name":null,"build_config":{"input_shape":[null,1]},"name":"embedding","inbound_nodes":[[["input_2",0,0,{}]]]},{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,64],"dtype":"float32","sparse":false,"ragged":false,"name":"input_3"},"registered_name":null,"name":"input_3","inbound_nodes":[]},{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,64],"dtype":"float32","sparse":false,"ragged":false,"name":"input_4"},"registered_name":null,"name":"input_4","inbound_nodes":[]},{"module":"keras.layers","class_name":"LSTM","config":{"name":"lstm_2","trainable":true,"dtype":"float32","return_sequences":true,"return_state":true,"go_backwards":false,"stateful":false,"unroll":false,"time_major":false,"units":64,"activation":"tanh","recurrent_activation":"sigmoid","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"recurrent_initializer":{"module":"keras.initializers","class_name":"Orthogonal","config":{"gain":1.0,"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"unit_forget_bias":true,"kernel_regularizer":null,"recurrent_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"recurrent_constraint":null,"bias_constraint":null,"dropout":0.0,"recurrent_dropout":0.0,"implementation":2},"registered_name":null,"build_config":{"input_shape":[[null,1,16],[null,64],[null,64]]},"name":"lstm_2","inbound_nodes":[[["embedding",0,0,{}],["input_3",0,0,{}],["input_4",0,0,{}]]]},{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,64],"dtype":"float32","sparse":false,"ragged":false,"name":"input_5"},"registered_name":null,"name":"input_5","inbound_nodes":[]},{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,64],"dtype":"float32","sparse":false,"ragged":false,"name":"input_6"},"registered_name":null,"name":"input_6","inbound_nodes":[]},{"module":"keras.layers","class_name":"LSTM","config":{"name":"lstm_3","trainable":true,"dtype":"float32","return_sequences":false,"return_state":true,"go_backwards":false,"stateful":false,"unroll":false,"time_major":false,"units":64,"activation":"tanh","recurrent_activation":"sigmoid","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"recurrent_initializer":{"module":"keras.initializers","class_name":"Orthogonal","config":{"gain":1.0,"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"unit_forget_bias":true,"kernel_regularizer":null,"recurrent_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"recurrent_constraint":null,"bias_constraint":null,"dropout":0.0,"recurrent_dropout":0.0,"implementation":2},"registered_name":null,"build_config":{"input_shape":[[null,1,64],[null,64],[null,64]]},"name":"lstm_3","inbound_nodes":[[["lstm_2",0,0,{}],["input_5",0,0,{}],["input_6",0,0,{}]]]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense_1","trainable":true,"dtype":"float32","units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,64]},"name":"dense_1","inbound_nodes":[[["lstm_3",0,0,{}]]]}],"input_layers":[["input_2",0,0],["input_3",0,0],["input_4",0,0],["input_5",0,0],["input_6",0,0]],"output_layers":[["dense_1",0,0],["lstm_2",0,1],["lstm_2",0,2],["lstm_3",0,1],["lstm_3",0,2]]},"keras_version":"2.17.0","backend":"tensorflow"},"image_data_format":"channels_last","input_shapes":[[1],[64],[64],[64],[64]],"output_shapes":[[64],[64],[64],[64],[64]],"tests":[{"inputs":[{"shape":[1],"values":["AAAUQg=="]},{"shape":[64],"values":["MWqKve+35T9k3jq/sVdmv4N+Y8C2GJk/P3ZUv22sAr+rRKW/wdlNvmbqzL8aMDI/lfiZvcW9db7SBoM9lbO3Pg/2Sz9+g5e/vXqEv6rYaD+AyZO+5wk3PBa/NL+FAak/iJO/vViJz7+RlUw+08nVP5EHnD8ZLvk+gJIpvk6pOb4TnMi+qsESP446vT/1kLe/IkHRvr5IoD4pAWe/CF2nPwA9Vz8D6Kk+iZmovzJ7875fXCG/s4EPwJH8sb8zXSM/KivhPQ6xH7/Xo4o/n+N5v58RR79LYQA+hV4pv5Wj174I25q9lSbavRKxlz88O6A+sXL/vsYafb+Pctg/fQHYvg=="]},{"shape":[64],"values":["l6y0P395dz6hDf4+ZtquvxC25DvFff0++SXZvqkLY8DzSDk/4LMKvxjd6L0lSfS9MX+QP2LLAL8exD8/6mzXPu2kTLwWkto/53KBP0/X2z/79HC/Rr2mPyQT1D1AX2i/G/rWP2QaOD857x49aOpvv4gzaT4ptty+bX2dPxwulL9e0ec+1Cu4vkgKoL51gyc+kV5avz4dib9aNAC/WDyePMTUXT5bxGM/5vjEPWLslT+8Gp0/yVCnv5O4ZT9xp3w+AazQPq2brr+j9TS/do2aPQGJmr5vem4//bqRviRlpr+MTOK9NsU3v/Q59j9Z0RK/1SWuPsjVdr9pRyq//0CTPA=="]},{"shape":[64],"values":["WmBEvwUTAD/Emv09AVuzPu04yr7DGLM+eDN9v0CZnL9VwHS/KsiKP5R0nD+ACiQ/JqWlvoxcxT5/gYm+1GI6P7G0l7+G2QO/GOIJPjpNET3p58A+6KOlv7usyz1lK/C/rT3Bv1cJBb8Daaa9DbCCv9CiVr77IZk/r1V9v/shUT/OQ3i/5vvDvnDeZD/NbUM/R6gYv75wDz8+I2G/eDnRv/VkXz+hqQJAsfSDv19WKz9PKEu/MJ0rvkc/4T/bFK0/BlvavxinVL7hzSo/1NGKv2hlbD8RluK+PmtPPTlQib727jI/QThOv4aQfj9Odws/FoOVvwYmJr+Dqpw+f3xHQA=="]},{"shape":[64],"values":["dE6GPwQYWz8cULc9wgOFP9/7Kr+0AbY/t+58vgGxlbzYqoq+os0RQGvLnj2tR4w/0ZmyvhvyPj7/UYs+PX+1vtrJL7/OXdC/EpQSP5jgID1Umkm/kvZBP4bqjj9o3hS/XYy7P7HFSb+QP6O/uRtXv5fzCb62NpS/+k6QP0LRm79F3vm/dbnyvthNtb7Zu96/WGQ5vwO6uz+Zvb+++zA3P7c+GEBPJOu9kcwAvkLh9z2yY2W+iuNjP3HQaz41E6a/nIuKP9zJ+j9m2jU/PyuRP4gCgj9ocSA/XE8/PiqOkT/c7Ps/nUMEQF322L+F7RW+hbilP53YsD4EZF+//dzgvQ=="]}],"outputs":[{"shape":[64],"values":["XrzZPmMfhr/2WQfAb+qGv/HoU784i3K/69SfwI9eSMAf74rAJs0+wG3gScBc25HApDkCQHbvTT83hSlASmlGQDRq/L+nsXK+84uaP5flw7/G3RlAU+iaQAEeiD9qB7E9TjmYP12GsD/R6aC/xFKdPhS297+++4o+RigjP8TrNMCv3nZAWJBzPm2xE0DOfh6+4rcFPprTGUBoaZVAfDxdQFedDz5zvRHAqUn9vzZ7mUC2fChAU6wqP9qaiUATxmNAkNXsP0jstr9EmAG/tYQNQMT3CkC4vFS+pLX1vldWYD9Z0Zs/APYJvGq7AMBeiQE/TpqePhNgPL/QemC8HRW2vg=="]},{"shape":[64],"values":["f19OPop+4ryf3IC6fxT1vedwx7veXEo90wU7vM3aPb/iFfw+ycAivfYmvD6vRrC+pQRpPrG99LxU5MA9SL3YPu5E075sfbg9gJ1JPxuFLjw+kUS9/Z2YPRP2o7zCxAC/0QSOPZ4HgT7prWS+qneUvRRxpTzRtho++SFJP4+NRr9Ixo093o8EvcZ/iry2dVA9jAYZvbaVzb0TuTi+GgEGO/FMXj1cky47a/QhvavlQT4K+no9nmAGv3vXID+UzWI+6ikBPiqzMr+g4xC9KS7+PfO3k705wwy8xq0Ev5rPCr7gEyC+2Oa6vfpahj2bapW9+u8OPI6A5LyGlkq8r9LxPA=="]},{"shape":[64],"values":["5nRSPlbJtr28dIe9qKpJvioDDbxh+NU+Q+OgvoyoHsAThQo/r6+pvt4N0D7aK8G+9ZtnP9e/Eb64sEE/FMnoPnCP574CbgZAG0qSP8+01D85IBm/xPSmP7hoqbycxw6/V+4QP/Ss2T7FApO+An9Rvwi3hD6gJKs+IISlP0E0hr8MCvo9SlKKvsgRib0snE0+V3FCvmj3bL9OjBW/snt0Pe85Tz40q8s+LB0jvT17ij5Y5sA/4dWsv5xXWj8sGs0+3hFlPuVnYr9gPou/zF89PhLIhr6d8eO+gEhpv/zJVr/KfNG++SEZv6Cewj+khJa+J7iBPt+vhL4yJsW+rnVWPQ=="]},{"shape":[64],"values":["RvNtPngrJD+7Ksm+4ycEPQkhPrx23Fk/e5FCvjjFg736OSw+mrG0Phuipr4G68Q94BqiPZ9Qrb16xgY+i92wvrKA3rwXpM2+gtB6PZ5rPzqqmdq+82PAPKNc1TwAysa99oMmPBxHqT6CKGy+SuU6vfnjWb3DGZ29naiMPfEqJz2QFdO+dfHhveKFHr9XVmi/S2aCvueiLD8odwK8YOJaPdxwGT9Fltm97S0hPjFw1DuH9b+8ceNAPE0QNz7uj3e/4LM3P0sryz6IpRI/2awjPnFMfrtbPEo9FfFLvPEZbz7OSGo+jvF2Piu+ib5ykZc7TKgMPhpV7zuNTwi97vyZPA=="]},{"shape":[64],"values":["jMF1P709Sj91cfW+2Ps/P9A4zr1E36Y/inxHviD/lL3+kTQ+E+TJP5nuuL6dyTg/uleDPsstVr7jIrI+vspQv5AAaL4iqLC/otyFPXNesz5CRTm/ep10Pwgfkz/6iD6+d/OjP0/gwz5xN1S/N2O6vgq1dr0aHAy/zH3sPc6+iT1Ih8W/xnVGv5PtbL/3h9q/T0Quv12nBkC8C4C9AOQrPhlrMj9TGyC/5lklPtqiRj4ipUe+S+j7Psh5uz7W4QLARsWqP85e/T9EsGU/B65KP0eUzb61MC4/eKXJvU9OMD/v4q0/dz0VQJV02r/VKYM+layDPzjVBT5b+YO/ruAPPw=="]}]}],"trainable_params":{"embedding":{"weights":["0N2vvH48hr3U/dC9S+8yvfe4rzxZuzi8QPnZvOua7Lyhr9G87kwoPkA1lL2LImu86L8SvqiQXD3EimO9zuycPAZC3L607ZQ9zZ9Xvrq9ST6xf58+0ghMvhOFiT6cZ1s+Sw+9PYspjT6SusC+KsxVPpF8vb5OCW0+UC9WvoCMiL6cRww/HxT3vY29RD7QKPy+B1PjvmhD8T4oDxC/cZnavtnwp74Bf9C+y/ERP9vvpL7yZd0+0fj3vkOatD4caZ4+tBG0PIuYD7+e23K+2nKRvsVvFD5pQ8Y+E98bvjF/Xr04nIA/avFrvowrOz5sEZi+8HSGPvBBxz0e7Vq8Ea1YPuSXjT5iDMM9nzEjPyRSZr4+64W+FYrhvEwDOr4DvXu+/bMwv5FTDr7hmuw+JygFvb6vyj4Vsp6+p3swPyNuCT5/Nhi+RBYSPwvku74uOOs+i4h+Pt9s+75Su3k+bye1Poe8Lb/dDtY+DzOdvkG24D5E98G+3CC0PlOapr7Ct6S+yGN4vhkA/T2C+Kg+VrTxPLRYfLxsctC9YnoqPoo9VT7eURw+n3aJPTPLDL5nLIQ+7/C5voL6db2mDh69gRqIvioqRz8pjK++CCbRPqtuC79SWYu+oxjSPreIYr8LjEG/kaBLv7On0b4MVaw+TFYZv9FAZL20Lzm/2ascPxMJ/j6jIOa+07yAPkm/g7+b1pQ+FSe1PvQwvb7lNsI+FvSLPon5ljxbSfU+oibKvvPOYz6kh7y+B1gjPymj874EhLG+9tbevluWP7+MoJg+ix2lPlbNKT4vg1w/EmtuP1hdKT5IwDA+CaqWvsvaDz/RMnU9ScVXPyMq0T4tfFE+nPDrPnrIP723LGe+gxsqP6STKL6YgQe/bk2wPtCetb7Rfda/Spyqvlg0Sr+ywEC9QoVOv+7Yoj4NSg2/6PChPlGBbT8ULyQ/8aOPPmWUlb6N5LA+lKaqPqbNRb5qRs6+5rITPxW/Nb6IFp477YovP6mAdT5ynOU+jpJaPs8pvL71bgG/","pX26PiBRCz97lre+6r4VP78XBT9tjLi++LwuPr8BET+fUV6/G1Z+PoszWj4XoQc/3B0GPRh+BD9ezMS+ylnavvTvSr8B3fC+NwCPvUu/oL7pkWE9/vMHPmjIGT6O4uq+IzvYPm6rQT0n4eG+DW3xvnKibr6LQAi/mUsxPlVBYz7lLjM+QZ8pPo0i7r22YcI9yrUFvE4lH75DVGS+Ub5GPvvtob66tYM+LpgcPeG4dz4j4aq+B64uPRpaq7zumFy+hmaRPjolJr50pCk/RqaUvotvHr+uX+8+sQQWvkYoiL6sXfI+unHvvvXW/D4WNti+L3r1Po/6s7757qg+ZQ7iPhNTyz7/pAU+FgOmvgjQBz9Ty/29aC1Yvn3RGr9Su94+0e7APKGhZr4anwU/sATMPqDuaz94Gkw/4r7FvhJD2D64Wds+zs26vbRrqr6B4gi/pmwev2CTA70FGUS/sdSovhLwLz/MrJW+k4ZwPipdFL9yRBE/h6zZvbKG970sjOg+wy9bPvpK0j1WV5q9ogJOPgLkCL31lxy+W0ILvrFxKD1KmT+9q7l1vZ9B9ztGpOM9NAhHPqhLej4mCQ++LV0uPmyVmr4oN0g9ORvBPUcbF76rGGs9ibNQPY0HrT4nykA9loT+O1x2lz7Weso93cQlPeTOPb5DniG+lQCyOykVRL73vg29oy4avEfMIL6NGxW93TpnPox2zL2WmKE9ItfRParDcL4x4MQ9KrYVvn0KFL16nea9mSmivP/nBD3fpuu91ar1PGxPZb7tmci+8y2tvdIoDz64i+O+PDMUvwG68r5kEdS+Wl4bPizTEb+Oc7y+lMkEvvngKT0vrvC9y7rWPm1EC7sRoRA+eWK5Piak+z0dQiM+rpKEPa3ghj0H2J28HcagvM9jhr0JnPy9wGi9O+S14Dy5iEK+nZhvvNZiJD0va7Y9lGoWPlVFxr78/8i8m0tlviyc2b3Gt7m8S/XpvPmPBL6pdJ0+PrrqvBR8tb3bmM+7q9McPpkRLz3ydiq+","p2GDPZXE6TyfkFw+2z4aPkVsGT1gqNI8QOh5PQpN4TzS1VY8bagSvrkBsL2/jEU9yM9AvSIZIb4YGxg9h80UveTPiL0mMrE9TOSmvi+wBbw9LTy9oaZovqz45r0CUZA8yicnPbBdCj7il/Y91FXsPW5/hj0Xlb09+r6VvAeEIL7af66+6cZ+vRwUhb1p0ow9eORDPquXBb7KQjs+iss4PSbfjb566zI+1AQ0vug2Fj4KS6W9I5CpPST8xzxgFkg9P7eyPgulTj4+3xw+77ghvatR/L2vzZY+rsP+vnZhIr5wNs8+lDw1vucWfD5fC3K+5WqnPhjzlL7Zyji9Oq/Rvmbzj73ZU129VEPrPWGhmr30/YO9BsHJvOFaID06AQK9NrG4PM+qX71Z4Cq89DDevXmIWzwhKF++FZGsPd2vzz3lDC09VVYuPjR1ir6wSJs+DdAyPm9J373vjTa+VaHDPcBKcb4suDk++eolvkkw4jzipfq6JGOVPReVeL5Ip9u+BM/IvTA/S74SjQW/YcqwPVMUtT5a7pA+1IgfvHIsgb6fDGq95jF6PuORs75Lg5m+Mo4FPXnpmb2d/NG91kAyvhlGrj5nNWU9TmS4PkUQAb0ym8u+7RNLPcRb7b1J9P472fzlPHo3gr5obok+6WNCvVWaSz5jEka+EQVnu82SCj06mrY/u9uFP1KhPb7YSEm/ydJlPEwmzL819Zu+KqluPhCP5z+tlka/SR3VP83kKD+fqrQ/mhazPwvuS7917Gc/QRAvv7w+jL9utL8+FdoCv2OLAz/HbQQ/r3HNPuBMTL6uhAE+d3HOPY/fnL0Sm7e+hSPSviKsH79JpFo/Lo6RPvkWbj9qk/c+JTC2vUYXar/UgDK/bo6vvqmrtr/02ly+ORp1P54CwL6BhFE/UttPv8R/ID/KvyC8eh7svj8MLb5pKbi+vOAhv4bZDb7MjQW/eX2fviHcsT6owTq+ObTzvlr3JD6df+i9gvvXvR1Vp75XWJc9rEJAvlo51j7lC+4+","5uCpPtYYzD47qAs+j6MSPxFPNj0R5mm+we1cPoC8cz51zKW+B8AaPv95470XtX8+iECdPQnZgz48nvq+u0fevvh3IL+Dt6u+fPBzvqINHT+dg4o+96GCvrAOVz62hvk+MNl/v1y8qD4xNgu/wv7FPn1Jr76FPc4+AdFavRXHWr5AXZk9tyKBP+9VCz72JBU/ba3mPInS5b+DwbA+dDctPzFmpLzt9g8+IMiuvjw6iD9+HJu/38cLQGn6Ub9pinE/8+gfPu7MKb9pysu+wvzcPZzE0L50kCu/Q8X+PsxrTz+TXNo+GnfCvn2wtj4RkmA/UosJPyOphz8SYWG/5nGIPxNpCz89HAg+MmOWPn9gpzwNBbu+meQ0P/gIHr9sEQ2+GHz+PUlzCL7F5xQ/Y6u6vgU4LT/5GYU9KpOgPolDEz2iQS8+j8DjvYQIhT7THaG+lPaDvNo6ST6zyJG+Z0uovhVWIz8jiqO+4Ce6Pvrllb6886U+9XMHv0LnHD6jSok9iByCvgXWDj+MUi0/XIW/Pl77ST50tyq/zHzGvWZ9wD5H70+/zFFKPkWfj76O3P4+GgD/vtK76750Swe/BkfVvrBx2b4pCIy+Xl8YvlyqjD59UP09Xi+4vZwvlj6JmcE9O9jvvl6PkT4Ptey+k6TLPd2kTb6Zo3I+x1Y2vq4ThL09L1s+NRqLPtVJcD6dJic+ZES4vjfMUr5vjcc9oMqKPb++0L7HjgM8yTt0Pt40QT5zEsc+oYqdPahVPL5QZSC+21pRv7w1Ob/zuOW+O1ZJvVh1R77rAH89uWqKvv1ErL7/Y06/nV6BPtlSt77BQoG+bGqBO+nZ7L7eUQU/YnY6P8dnizrM7Bg9SRfJPV4hxD0d9/u9yZWIvqvLcz3Q3/o9UhagPQH9nL08OC2+SxkFPLPuVz1lInU9zUBevomMwz3Fy8c9Ck8DP3lLvj6VZIQ+jnGsvls/rT3pB3s+htDCPh4XxD5b48i91bmWPkUdz7ugjRY/2KqhPRE4yL5F31e+","2IePPjO5Ub902Ci+ImD2PVUUA7/5U6U/TwXtPXOrqz3MooU/fVszvX45kT/9woK/fdTIP/+/Az+Z+wW+4aKiPcmupr6XBvc+qEynvuKkND8Gb/U9dO54PpLBWT7t3Fc/+0qBvpCL/j6hDoK81irXPr1Agz5W2pA+3QEbv3XJPr/ZTyc+rSdRPiw9AT4bXaI8LFHYvem5WL1Y/44+O8QLvnxSCz75jxa9nGdxO5rf1z1v2Bg+S8GYvbuwd74+vQk+EWG8vnnPSr8rwNO+xkwoP1yw/D1kQiS+wC0suxvVjT9dOl2+IWThPc0VtD5E01g/dj5NP+HVVj+x9G2/InuAPje3Nj6cDck+yOgAO3Xrmr42cCE+0Rcov/USIj6H5hE9dFyivglxsD1GTfK+woHcPiJFLr/+fiY/mcqMvryfxz5OQsu/W+2bv/X9NT8Kzpc/IDAUP78gBb83Vhe/OPwpQOtxCsAYz0c/osALP4tfuj737Lo/BotQP7ly5D9z9Jc/qLTyPrDpSj7X7eG+WXsEv0/oDr+Dr0fApNREvb+wCL9cmL0/oO6Kv8ozBr5ePLe9A6AgPle5A78/p/69VMZRPzJEzD5a/4k99ueeP2PD7b7lH5u/xjGrPixpF7/MlRK/XB59PwFAtL8dHQk/jpWXPuk4kT8Oty2+tWAnP/jlkz+UkAs/vTK8P2DWx75zoMO9cU+qvqGkHT/PF52+GPO5vJs9E748mIw+cav3PmGENr/pVGU+nYgGv1ctMT7MXai/6OgBv4Jx076GtW0+JvkCPy3gF7/ZPXO/lgqDP9gQIj5auT2/wyVEv8AgHb9bcyo/tbdNvxoMBj+pbo+/06hQP+/Du78SLwbAN0aBv9PNvT+zzaQ/V7G8vwvgvj9B+Mg/idf5viHoYT+HgMy/NX7NP8Z0P79Y0Xk/+0+Jv8PPjz902LW/JxAlwLeKt786jcc/heDUPy0v1r9U+K0/ZDuzP/0w0r6Getg+K2fWv4RUyT/7BHO/gSiaP7gvlb9sh+A/","ljJEv1evSL8e0le/WZteP5+QWj+oAoK/krA/PznlWD8Ax2G/8mlKP3dtWL8ivFA/4oUsv5G0Fz9WaiC/Yt8QPpvpUr/keaC/5CGfv8C9wj8Pur8/N/TxvyG9gj/HzXc/BFDRv8PjdD9Hu6S/Q4W3P3EhUr8vRwM/3TWVv6RZnT6gpT2/1DpWv5RiSL8zoF0/49xTPxncXr9b30o/ZEdBP8OpYb/dzUI/AatTv5xCVT+TIiy/JKgdP6KEKb/DCFc+oD3Auvo4DL2H1BI9QawpPeZ5nDzghqe8XSUBPaKs6zyAR4a8OimlPNDMMrtYbAg8k64XPea3zTxQTsK7o5I2PQ=="]},"lstm_2":{"weights":["VNlyvSvjdL591hG/kuJDPUPegz2t/Qa+6LDpPcy0Ar8CeFS+K/Aluysmk73uUvg9p6Mvvr8/Vj6McGW9iqvcveK1Wj40xQA/KmXhvmFkUbwIczM+B12cPg2UDD62Vt49LWwgPWvPpL6q71a+6JvUvcSaYj2Y9lK+SIAzvmD9J72BjZO+9duuPj7CYD29moo9LjayvJiLKL7Nb8Q88Gu5PnTNlb4ypT4+Uvb4PGLGwL6KgWc+QRErv936NT5Zbfa9+fDNPaXcJ71vDR0+axIWPh29DT9R5ZC+cCQQPrg1gD2vcAq+tEkBvlaUoD4y3vu+Hs40vjs0Lz46b6899+uQPuB3Wr4E4xs+6EYVPkY70D5UBru9BYzlPQubHD7SnNu+NdefvAGG5L5U2pG+O9E9PTotDr0sa9m+QnQwvO6h8by28Lo5wF+yvhQ2iz0TgkW+r0Unvs32y77hnTU9NIAJvfU+lL10lM48O9Q0PiE7MD1IwOQ+1ZFHP73gTLzfUbi9jpkAPz+6/T3xxQG92SKBPhLgKT4QIYi+yFQAP11nP78yelU+uRHaPfJRrb3llKs+qN2ZPr4A1b4yVRS/QYldPoNVbD15uUC8erpUvvZqJr5lAo++pCodvxua6L6iQoU/vcVHvu7Ehj4Pyxy/sIWnvS4lkr7GoC0+i2ApvwQIDL/ikow+PcuCvCE9hT65jgM/tyMDvlOsDL9GTl29sIk4vhDA5L2OBqS+jZafvtR9yj1mgoC7k38ePsbKNL1PbJU9m+FAvlKv87434Yu/FXYqvZ243b6wBoi9tQ8GPvOCXD7JmEq+/UODvnZshTwjWIo9Wd1KvlYVbz5aaTg+InuBPbY5JT94Ib0+vPMbPmNGNr+WNiw9JZfNviWQWb4VaAC/NkGwvX3whb6eacW9aSwZva4JCjwZTby9QWWwPvc06D7wSx2+EpMXPjsdAj+FCNk9/tPIPbagDb9SYAo//bP/PlIWOT2ZDaQ+a/YVP1Mftz4HCTA+gHjNPNNWkr5n8yq/","l+pCv2n2BD4Plwc/WCNdPTstL78LSDs/wzuNvRj68L7O2ZS/WF+PPliRe7+4fj6/oE+aPqFxTD7DpaY+Db1UvwkXYb+Ih5c+JSoyv1f2MDxZZJ8+h311vmEPsb90eGG/v98gP5W9Ib/JLEi/g4NHu0Aqmb4h9sC+ezSFv9bs4r4UEFs8Uv5HPlHPhr4Btcc+t/RzvqlYwr7csee+DGH1PpWu8r6IpQ0/ayDOv6Zjv77wPvY93hV+vgD3rL7xbTG80lwfvgMt6b6V8aY+39+KvnKb4L1hq/c+QcXTvigoZz6i9xc+c7wcPb1zrr2CbFS++rgQP3/Cqb1a4xa8E7NBvySU2z1jKA4/EqogPpUrEr4Mh5Y9LaX2vp8uer2INay+yNkRP/G6Ez2w7CG9m2XJPJ2ZJL4JGCc/23oiP/Km5TtoqyA/PiZiP99v0j4C0xC+qIhvPonQir4VKS4+A5N8vQ7sDr1dF5i9Pu2CPfXDlz6umAY+wmoUvylyyr46StO87epnvlqkrb0xkeO8BUaRvcYalDwiXMS+wZ8gPiuHmj5FBcQ9/74DPgLiE77TINA+jfNPPgCupD6zLS4/zMoKv8Wzhz32Obo+4A0mP0+6m77mIjY/mJFCP28BVD+fLf+9B9SKPa15Xb2QGy4+x1vMvmakEz9tWkq9wx1zPh1WED++f7c+2lCwPRgoMz+CubM9W7DDvvM5ST9a9tM+j3Ybvda4FT9mob6+YgeePvsGob5/kA4+HGX9PIJOEz9OEYA9g+nxPUMWGr5FrYY/zvYvP+sV0T2YcUs+CBO+PStv2b63wtM+eaRdPpYs8T1UEnQ+rad8vs4/JL9VCrM+n6YxPhce1zzmSY69TvzQPD5jvT5ovRu+8vydvskdYD+6n88+0TIHP9TSLj7bKsC8SOasvmmFDD7XBNC99ivJPhPoEr5T4C46v0pVvXOChz0+qKC5z2kyvuC44b5t9NQ93PtAP9iYvTxkHDc+91mnuwKjlLydWda9p9jPvoGq775C53E+","HiZVPuQ8Wj5xC5y+EwS4Pm+E3b3fc7W+3qZSP2/dqr7L+Py+4qbVvXoNqDpWGJa80S1BvlcCoD/VpNu9YdWLPo7Iu74YMgq/5cyoPnPhJz9AO5u94C6RPjCCizwFxiG+0/SFvnIEp71bHd48oKDmvX17GT9F03I+gxLDvg6asboi5sA+ITbEPtk9gT7ywuQ+UFWnPRhiar7A4yK/m/8Dvgec07618hY9XV0uvndU8r4cSwg/UXWhPi0sCz8Vsz4/2CnsPYzxbj7UuqY+Or8jvU/Xiz+vySS+q67KPhFjET3nJ6g91s9TvwSinz2myh4/haVkvPFJ9j2rF5y+7IQSv9xbJz+p75s+dEF/PkNsqz6GxAO/7V4zP93rVb018bU+EUrRvuwQBD9bf6m/71xfvztHBb+TG+I+b0bovg4zOD/+iq29f27iPfmf47yh/tw9sph6Pm5bOb/i0oG/3aMNv6jbej/gfxG++TyHvx1G/DyZuQi+ZEoBvwQrib9l3XU/Wfp8vtUKGj9fgOU9O2BFP7MyoL4rEq2/c/M5PT3+Oj9oaia/v24APy8hib9EINE+a2mrvmlIhr777Y48kc9Wv2zHYL5O5x8/9ntCv5J5Ar/ZuT0+MaOKPvMJzz8n+RU/oesTP01vMLt5DVq/cuEBvzh4DL+vI2q/pRX1vfwJLzxQ6By+7NF3vU6bTb6SRom77LwePUEkur3OFbM9Ua33vWxet70eahA+85a5u3yu4T25gtw8qq49uzMZpb32x/G9PYG/PdPjLT07UgU+6CvDvUvVir3B1eE9vawVvaq6Oj1m7zs9MY1bvpHMpb3fl4K+mjofvviCPD2r/Mg9BsEuvYl7Zj6/c50+JR6TPO/nsLzQ6ka+JSZZPlghFD81rh4+qn3MvaIrZL6tfL89YgFSvbYBW7xy2l++9sObPRwXhT7CUwy9FiXEPUJa9T3dZz4+xJBSvlb/Lr/IqRE+SEEEv4EvIj6Fm447e1HNPpCQxr1w0N4+3/FXvHA8bD57i40+","28rZvbdzNL8E2QU+tOdkvv8Y4zzp7648V+xUvUkE+b57MkK+cuQKvyqXlj58SGI9wcngvtz/Nj6PFni+BpNavrTogb0br4++zdKWPmklgb57qbO+xjsfPVlwoT2AD6q+OQRkPvF0KL7K7VQ+APOdvrlwsj79HCo/S1HwPRA09jtdArC+fcg1vhvfKrvgQkS+4kohu5nMC7wfLyO+nTqxvUTMsb2Eae09XKrUPZLt+ryBGt+99Q9OvgZQlD1XXgI/9WAhvPM27L1gVUY+QPllPuDHBb9Chpo+QURpPpK6jr+Rh9u9QRioPQQnbL5Ztm897ZmHPc+cGr4Ez/W9SetwvqE54TsXuJU/3sfaPlDdzr3Fc9k+eArSvg505L1Ind29AeM7vj/UNb6C9Gw+dMhxvXkhGz9EFLO83NwTPRT9jTxClbc9Ey5PPRg1fT4moWk8+FPFvoIqhb4Nrj090NofPnq5NrxWFeA9rFWmvmiCG7/+qh2/DzeDPll/wz3b0ye++RWVPv2Z6jzkjhI9fqiWvh49eT3IpTU+O3YxP5DOEjwk7Xw+igQJvM+3Mz2t+hi9RnkGvzodA772Gio+kFdjvWMPF737jom+/0I0PgTBa7uShRs/ct2cPmPSM74hUQS/fHwfPrUy6D4ZgS8+az0Rva8Vjb18daa9l66dvUq1wL5QpAi/sPRDPkAz4j6LujQ+2Cx8vfnVeL6eR4G9z8BcPqEEmL/HUBQ/1M6YvaOXzr5L2Pm9lgzHvqeVaD6lxJG/4BUxvypZoD55OC28jJmFvk70HD758s0+ZkP3vlg1jL+KIjA/usTCPgy1O7+5Wpg+bAZDvmFz0b5dVuu+3Sobv+ebR7++EIo95sOBvmllWL9v8Pe+0j8Tvp1h6r2xT1C+xtzGvirhqz2SNYK/Y3w+v9Shr76PUX++NC4SvoEAij5BMIQ+Q5dRvzrAWj7UTPc8W87vvs8lCj4nQas9g12ovd1Jmb5HLFy/toSSPY1VgL/GxLA+wkWzPe08DL1xhgG+","SIhJPtv9KD4BPcM+1gmHPrcPcr6tm6o+EZwEPqkF7j5xndw+yYNQvZo6wz3YqaO+vQoVvssouT1QCUo+ousKvmOn2D6tzUI/AlGlPg8rI705spA+llaIvh3tcb0ClRm95zwtvrEBmr23Iba8bFyxPuLuoj5xriM+OZ/ovLrWJz6ac6693x6Cvkve7L04I7o96ltDPozPo73oggy+FuKivjlHlj5ODIE9B5x3viuA7T5ysGc+LZi5PiLpjD5bei4+ZJsSPh+Muj56W5g+/3yfPPiGsz0YF+c+KxNwPgNRw72aFwq+uzRmPdwG6zps6EY96aQuPjnjob7h3Mu9W7J/PLKCiD6Nlv69yoc1P4U9wjymMYS+hHicvAzfaz5CSQ8+uyjPPijFCT/Ehwq+ZGIyvvY19T79lZE+S7X9vZKvKD7HTFw+o3DSPs557T7fi5s+N1RZPq+0Fj/3oEg8k2jmveZ+Mb3hVik+IrlrPTNMVz0a37K90kN5v4wJnD6PsCo94s2yvsXcGL44PZ49gNUTvaI9/jsyNvs86DUKPkb3Az/nL929pEkivegrF71hk207RcYdvSG3jj56Lbk+WB/uvvFbz73cGAa+8/4APxqSu70edlS+/VYgv2w4KD73fKy+fdCJPgugVL3sNS0/VcJZvnL/1T1bVAe+MVlVP7+rED7vpOq6NYkvPQVdAb/tvK4+IqdQvvYzND6795s+XLk6vECsBTwND7Q+N4zsvQwILr1bEwy+yzcVPu4SIb7lQn8+bA7xvUthEz1XCp29fpDVPrr2HT7rdwM+HtOvvXwO4722vka+5oUMvbiKXz2gCCO+NCUoPuFnVr6ABLG71c8LPjlwJz0GX7m+BMCrPodpLj9uuG08CJz4Pb3ccb6Vaa0+L2aSPVQTGD6Z/zg9srORvstOlT6yYFM9s5AWvYdbarzZ4pk9p3cbPpCFu74QnQi+bQ+1Pv6vdr+9wCQ+uNUjPn4chL2S4Oy+pZQevxlnxT5t7bu9dPorPqb9Hj/3a1I+","pI/APvW6q75BcNC+1sv0Ph+g6T0hA/g9kg+Nvhbtjj692g4/8/SlviLIOT4SSZE+wjGOPqQNeT0usxa/vRtTP4LsCD8w6B++/XUBPywYYj3pb1y+eyEUvafK8j39vUs/roAOPk3sYD6EZKg+AQ6Qvgp4Mz6xIzM+56GfPs8Vkz9hIlo+abCiPa+zmj7N8Sk/3a2bPvBn+72b1QO+0VCkPqs0nj7mRBi/ICshOaQ1GT84JZO8LkY9PoCYAj7jwc6+xqM3Pi5VZD8xH/++zd1jve+v/D0I4hG/qQe6PEWkBT9kNpc+iqoBP1h3lb5IEp0+FRu/vpJnJ78ynAC+pFunPjGqNT1Namw+312IPkUuj739rw6+Z/RePgxSqT3i1gA/9EJTPoVmm7xEBVg+FgUfvqm9Hb7gyAa+PNBNvaUgsD2VxRS+yAwzPiE/iT51ylg+cQAaPpakhr5RHjE+7dEqPXoqezvXRl4+2RSqPX/YIT5flXE9JV9SPuoO4zuET2492qK8Pd1Enb2kUPc93bCFPCvDAb4c14q+sjCevneEgL1ZQ3a9AAAWPgY+rbwTwZ09a9WqPOu4AT9cZyO+dkBHPktuxb0aCou9jIkBPlser72qB1s+HPkkP7fNZ77QnzI/al4+PTPuwT2W/jK+78ZNPWM5hr5Urqa9zwwPu9SNJb7YG7i91AwNP6i2Cr6A+Z6+kf7rPCQLND0boG49/rmkPu2y/z2HIcs+ZkcYvvI51j23OtI+whpdPpIZkD5oB7C7i1dvPablHz+ACxg8wMKWPruuTD7p38w+Eae/PO4Zuj24b6e+Z1EgPrkccb0FbD8+mq9EvpeYGL9Kioy+/xScvcNHWb/tQVM9fZ3PvVOAO77Ahqm9v4GRPnL6LD6cZLI+SdmEPTCRTb4Dcgm+g36hPbspM76LPGw+rNaEPhT4N7/sWhk+LGuCPTza8T6sM7K6AmapPf7iN76BDwU/xQQxP9kEcz7lnx6+MFSdPjGeG77NKLK9r57SvGuMf7pBdMg+","FkPAvDxk/r6TByK/ayoAvlwUpDzB0Y0+BxNzPnFAgz60adQ9QSHRPV2DwD21zZG+oAXzvvZwr77pKUw+Nq8nPMAHjjzP1Ts/zsjHvuNiET7NeeQ+RfsRPoaw/TzfMEQ9Cw7JvjQSxzyamwM+E6bZPlJypT7mcU2+onszvonpm77/TxC/gpl7PsNXdT5k3N0+sbWPPbTMTD5jMMY9ESN/Ps0bm73p6gc+fh+Pveeczrupysg9tOkfvgi7Zr6pROm992x2Pqd+hbx+Bt6+QjQxvsr3fbx5ERe/Sjpyv0HUUT+yBUY9ZTVSvpCd675meBS+vR38vAKNwD09STg8EwuoPoHn+D5Msze9eHFnvvlbjb5MPEW91Yn6vrZXdb1qjqK+K13WPomiLr89fvE+K+kfP58hiz5FYYc+vmuyvuHsQT8D0qs+diqkvmk2qz4K5Xq9/wYfvhVNgT4231I/oNYJPjqW5r6u0wq9t6FjP9x84L27lwk//kvEPpZyYD9E44k+V/B+Pmw9obw4FZ8+U6sEv09qbD4tTMo9FTMkPnSnkr5xr7M+lsTjvvIICj/ZI0M/iovUPp5YbD76c2s9jDfcvrTGI77yNeQ+njBPvX0j2b0RThQ9WjF7vlvxvb2qU2M+l8n0O/a4JD97AEy+6IVDPy12375dYjO+BSnEPSJZTz68z5e8FRJUv3amKL6VSa28JW3vvaljCz7/pXU9kltrvlprtL0HmGu+AbLavBRlFj5SOV4+9eO/vf6i3b59QqW7TQjavloV0b78+fy+18UEPLwcUT1uyrM+tSZfPI0jw71fmQ6+9JAAvoUR370OjYW+PaqsPLoYCz7F/n8+t71jvEX/u70GSUs+Ut79PFEZsT3WVau9JgOXPd2jrb4tHYw9iMtVPq6bab4QAfI9f1esvhFniLwBHB6/BxhfvsNc4b2+yRK+r4Ikvji9TL9t8Vg+tIzFvrYI3L2ha76+4SOEvaGzxDm0dy8+TZFDvnqvnb2L1qa+g06CPR+nsr5QRrk9","+siIvtPDBT35Vde+8T8CveVEbD1sLeK+1/CzvutlcD2BxKm+uUDZPk+aML1zqca9YZXFvnRU0r7yOgK/lDctvr8j5L2Yp4a+hWkBvzjVbb9SUYK+OUDKvmUOcDzuHA09iN1BO/Yqsr2z0fW8v9wrvnE4dT6EcYE+TrXlvURfg711eD8/nUj0PgV3wjzVwl++5txDPl00Ab7GsEs+u7cOv7/Omr4w1Q69ADIbPuFDhT4v99o9UbdQvp/9u75/RHQ+b9TcPRRiu7w2paM+9lHfuiupFb6pVAY+yvxPvuxGRr/rqjU9TqdIPr7wCj8Ge489mZ3dPXY4iD7apxw/194Gv8LsQ74CXKE+GF0DP1gYW775+6e7iU4ZPisBJb4zarA9lrqKuxIrrT7muqS9F0mCu6CDLT7iHKa+n+fNvUgxvb0wRxA+C1SYvk9uO76/IMG+r8INO0TIwb0dvU0+mu/lPTLcMT4dEkw+NU1ovpnLEr4NJSe/HA6LvTxKPT3fGMY+7hqevtjYqz68oO68EHYYvwTXRj4VvZc8hsaAvrPOgr1HvDs+eYUgvfCztz3axnQ9CtGQvr7REb383mG9TQSyvn1JJr4ToKm+mCykvi7Xkz2xVb6+zyKFvZXzez1MFNq+uNzDvWz9KD8N9u2+kqbUvs+rlj07EoG9RbJuPuO4QTzl4G6/u5NUPLZo7D54KJq+jUmVvgnuFb+OCl4+Ki4Qv3NGSL8C05I+DgCWvkuS+b5oPcG8iXA4vti68D64csa/2Nk7vx5Opzxte1u/4kg0P8W3uz77RUC+3hd3vxmsfb7J4Eq9ub2GO90DPL/pMSo93oU4vnnJQ7819Ay/Z/qOv2ZpJL9NDDK8UXpSvVju+r69MEU+8G3cPEijAj6573Y/yt8Gv2YT4D52Wle/hogBvyth+j5lnVq+OpEav9zmBz/Kj3C+O0VGv2RI9j5M4Hu9sKs+Pv2gJT8aR0q/WhmavurNNbxASo6+5+EOPyUqnbwrWgo/ROYDPzALXz7JZxK/","EVgVPqRd2j2awbM97CSVuxTo8DywAtw+MKscvSUxuz5xmmk+TDY1PDl5dD7NqZ6+/4z1veiv5r0ZdeA8zqxVviozajz5TQs/Du3iPiwogD1U34K96Gctv254Mj1KVwk+Mbyjvfbop7zEy0I+552HPnkpXT4S6ao9O7QVPsJpQr7WCD0+8U8RPjbqgr4XV9C95b2CvUu+nT6cORU+c2QFv83rvz6tRwI9Z+UZvobZOD7vrFu9zW6xPq5/sb7/T1C7B0caPsmiPz5TXgk/JV0uvuV7qL6EO1o+GB7du0lUsz1L3KY9Kc0AvDwOhbx1hpo+VNVBPv1VSb6bwNK+cDTbvNCVMD5UB+2+5yCKPjBrwr3UmQC+b17lvk8TCD9i0oQ+6RE8Pk0K0D6N6JU+UO7aPIpE7D4FncU+vPIxv4x2Sb0ExmY+3LoeP/gdrD1vk74+DkcHP3dnGj/DEma+5HyOvmuQBD67fBI+227JvWbTBL4Bji6+qFFRv1l/vTzVS4q9upYPvxqd+jwNSj6+WLkyvhPO1r3KyaM+nDDpvo5Jgz9e/Ae/8kScvrLkOz4VXRO+9M1tPN1ljD63C6Y+0as5vn8kCj429lQ9GahKPjbV2z3pvWa+2c3wPWl7Sj5uk9i/W6/VPrikF778PPU+4D4/vvMpSz5uF7U9+JYXPUh6uD6d/IC+y2MnPwEwyL7B4qS+PJ08PWDx8j4dQA4/QKmIPIB4JjuCWZM9IN+YPh2GxL1iH1g8/me2vrUcBb+zcWE+4bDGPeem2z5ye4I/5q++PvHTaD9neBA+FPaCvVyWTL7stik9DpODPhTmIzxEoqW+61vVPm6ySr4EJbw8vs/QvYkSmL77Ddy9nDiEvXkcHD8Mxzq+0dScPrw3NT4RiuM+v+bLPrwb1T5YdI27aziYvkKkHb5R3Nw9zKMZvuVrBb/6R5W95BY/PqIXm72XezW9cbinPlKpJzyfV6a++o4wvzkqSb41GwK+7Nb+vl3EmL52tIy+fKDoPXB/2z1lYDg/","NWk0P5GtUL6wcIu+iHdcu0zP5z6G+Si/FutsPc6V7T7NRoA/ON0Zv8dJaj+YZog+mDEbPqXds77SBBe/9sGXPR/VuT5dpuW+Tn4zP1zjtT5EmiG/NMJWvjHMZT8pHZM/4SaYvD+mAj9fSyg/gx/Lvpg6Dj5Fr/k+peSGPwTztD5SDGI+lVp0vkIm4T4xRTC+pEUIPzF8rT7no+w+PmcYv4lxnz4OXL6+SHk9P4xMEj9M/dS8UjA+PrOJpj4KgZ6+2C2cPkCCWj8dlsW+MgWIPd13vT30fgS/tE8YPsg8Pb7sepG8qw8svXKxMz7hlKA+Skp3v8GVgTww0x2+BC3zPv1Nhb3pXs49e4yyPqsWAD4i1HG+39VdPv2rH74QDK0+OhuLPqG/DT4wxCc+4LSQvr3/qL3FIsy9OGddPAP8v707wwc9j5ImP9Ug2D4hHEY9ls11vbm2oL4gYPM9dhMlPumJg77rysY8o6qJPQutwz4No+Q9Yh0vvPuXj7ztlgo9hcf2vUhiwTyH8ii+P7tvPPwH3r3Ai5C+TufGvgbkkj2uLaQ+X0iJPsQwcr4saws+c4ZCPkzuhj5vIww+hGHAPUU4y72orEA9rPWoPose1718EGQ8iyu9PuS60j7c1hM+YDl9vuf2Gz47rM69KdoVPkPGgz1ozb88Cte2PUk1F74S7qQ+EjY6vjDR0T7CNxG+flJUvoMO6j0A8A4/jvL9PXB4iD7AAfE+NJKJvg0k/70OCwQ/f3rbPn9nnr7XAkU+ZqOjPZRvVj6d4Dc+W+vYPj/2xT4K4sI+7M3DvWMSGb5XLRc+ZwVnvX9Tyr1QdlI+ioUGvnueUL9YDpC9D/XWPTTKU7/qv+e9/yOdPE5Eub7tHdc9UAMwPrXKQb72P3U+JOKBvWMCBb7AZLa9SQe0vF7rgD1QZUI+chTOPQmul77x47m8c211vMG/vT7ADjO+c58qO9Tk+L6RTJc+8QeHvsvYpT6JRJO6HS8NP4n7nb1rtzW93/d7PR3Odb2ue2U+","hcPsPRpRxz7eUjS/OPNiPpXXXb7Ems8+m7szPzl/Iz5FU6I9G37OPgN2kr5SpPK9lEyavgZJir0v83k8gQNXPjdgrD2CmHY+r7mtvgQz3T56S9A+AELANjLHrDgkDDC+O6kDPuBjKD7GhwE+nGQRvRuFlj1Mvvy9A9eDO/6rEb68ceC+43h4vlX5lj40msg+FZnMvdmkwD3jnSm/1iOBPpBulj3HW2A+3moEvs4/Mr2k240+o4CIPMUZVb6mEFu+tvEtPHkzgT6z8Zu+C7qrPbO0Lj0LA82+5G2evtShHb0rt4i9f/3yvVkhOL8NQd0+mh4evsU7hrvA19M+CnpHOx02BT+wzAG9DTmlvq3k2z3cuE8+lRjvPXSmur2W1aS9ZAjIPp/JQL9S4ww9IlqVPiT1ez2nBCI/nQHXvq/eSz/YOic/L54gvSqNIz/XA7Q+yobbvosvKD3DHh+/9UbUPrIDfL4pjLm9viXXPhhrnr7+J6s+kxZbvlj1Ez8ty5w/ZeuYPH+pqT36aac+WkYPPyrjnr3qKrm68gJ0vrvb0D742vw+E69Pvvpk0D6+jW8/mYUuvFI6zz4ayR++cbYZv/vTYbs3Jxo/+fANv9/AwD2v8LE+F2Ehv8pdvb3/PbE+tVmJPev2UD+nXtK+/gApP38UF78NSNS+NAASvsLntz5iN1q9I4kxvzwpvr05RWk+lgwjPWTlDT8JAnO+FMYdP2/P6r4FJSe+dy6UvnETdr1gg/4+CKIJv6bYRb873p8+MWxCv3mr2z43iSm/vUVtvUF8574Rpns+uE/ZuzytP75mEjs9MtbWPHK3hr1dlew9yh4VvU6taj3Z6b+7j6XGviX0Ub4AP2i+NoESvotJnTx1txS+YY1bPYUggr+4OoK+1A5ZPndzDb5u634+5OMOvrMtbr5+9Nk9mtQhv6hM5TxReiE+fHYgvwxVkL1cY6m+HdCWvk+x8T6Na0W/FYXwvmTBqb5h59k95jsZPmTa7j7x8Vq/OMyJPRzzB73vIji/","U50tvpLKV74tI6y+nV00vjKAkD7ckfa+iEqPvnaWxz7duQM+hQKkvip0M79O3yU+b0Tevmr6q71+CD2/KPNtPsABgD0ddCa/5I1FvwLveL6RKyE7LQ+jvpA/qTwO7uM+/dVhvUg2sr0F6Oe9JwfPPY9STb7B0/s+uWh7v063X75PTFC/wMFGvXh7KD7eA7q8MRIWPvG8oD6ZjuY9tt3RvXzqeD6NYEW9YQjpPUcqAz8ntcK9xc+LPpTY8TyRFhk8cPpqPasdBrwQ2CS/399qvcWRHj8nyg4/aLwsv7osFb8PtkI+KO+kvagH0r6jp0I+Lo1OPo8yIj+aHIM9uXk9Pi9lP74SYCs/rxWkPWPz1r51LuQ+Osk9PyHrSj7cJhU/MSX1PsWTAr9KYdO+fL6gPXhriT7yhmy/6sW/vhpfmL6qHoo+HPcov0WRar7/YtC9qZUFPtxHWD2jKuS9xwvUPszXUT7qsZc+rT09vtvKXb1ywsO+HTzIvhLpEbsKKMM9XdmFv8m/Lz5ebDA/XpcMvwtLPL6muPw9NLgYP3HXqz6kxJi9GUrHPhYijD4jXeo+oBYGvmStGDtMQkO/6gvSviodcj2cG1++xgY1Pr/CW7w0c62/ztLoPh5Mkj5BIKu+bf2vvj2yVT5yeWY+c26gvnYXk73b4le9Ep6dvg6cQT8rT6G/kYrLve3auj17tjm/j0OQPsOJYz6EY3A9tQAwv08ISb/H936/auZxv9xZNL+cBhG/BU36vnh/vj2Cq/++V7j3vlg3Kj57Nei/AYJcP/l7dj7LrOk9iWxFv4joKb9IRbm/CEkFPqLDOT7zn5s+nvoFP93nhb/L82m/4Jluv9yF6b531f0+ZeSCvmq4XD/ScfC9xJ8OP39AHD6gwEs/42o+P5/KET8apLy+54EJP3OSAj8uFmI/qn2/v1j/ED5OFQS+lHSuvkoUhz5i/Im+SCvFvi/wNj/bn5C/fWSZPcV/Hj8tfL493NtUPy0rmT/FyQ4/arWnPn9d0z4nIPq8","eux2Pqh/2z5Y3aA+bUfGva5rU73i6II9jYSiPYx8NT7aicM91t6lvFmw3jzKtFy+LVCEvq1I1z1PJho/e94cvQ5+cT5fPaa+jdXjPuSEw73euRE+VLyuviJesL1qDIw987e8vXmMm7zQHiI+ZMNDPQDchz3ILie+aWE4vmAziDt2MRi8zWjJOQWK9L3ZZxm+YGcoPHUjCb50xkq9aeCRvjR1zzxOJDU9qRN/PTfGej5MULm8uTHfPjteGD5+8Ku7p2ynPcsPED6uuq48wACCvtQ22rw9Yzc/WicsPneDSj4KMU0+kHdbPZLhJr67Y7G9MmpZPP4cJL3anV88G1lFvmJqQD1oNAk/9YssveOsar0HG/y9jzSnPnLPcT7Lx1w+3SUzPnoiFj9UvBk+amMLPT7Uij5EiCo+CebWPrDv9z3f0FI8QpfBPm4REj5rM6Q+cGE8PtQS9z4DP2a9uuuZvmP3Hb4dkT08904HO1jrlj7fJ2G+9Tt7vjcPQT0dOj29sUarvjQ6nz0Eu7+8iQMYvUMaSL7z5mM+SIP9Pp8enj7o4JA+6/Uevu1lnb3Ou4y9NmCZPTyaNj5FFbs+Cjh4viuwLz0/COC9m4uwPts4MD0CZw484Y0Ku6SUgT4RUAo/ezG4u5udSrxWqVU/pp2EvQVmgT3Hm288oYQgvbFiCD6Vqsw8hqUZv9gADL9nrgM+JslRvpcj6j1BIN8+3J3uPY40wD7JBuk+/uoFPqvPi72cVKi+jbo0Pj/apD3MhqU99ji+PRCMpT5chKQ+sx7kPeE8rj42QIU+qERzPPjqwbk3LU89DV+mvWtGPT5qshC8RHYVP3YDBj6b4+i6BvObPfTmwDxPv4M+YY9ivRdaBD+qwao9Q2G+PYzq9L4JoK8+9qmKvlLAYT1/yUA86xY/vmSIpj4A82A+EbgxvgVmUz7zQiE+5UVsvGIe7740Kiy+2aJPvbAHXD6AGd++7DZlvAn0hzoC4Le+bagFv0tLrzyyGzo+lsN+vT9D6j4g/pM9","drhUP7cIFjwXOsu+74fvve41DTxeXq0+VaYpPf9ZUT6ig6M/SPAdvuLUHT/QO7E+FmI8PjVepD5Hzja+AveWP6+23D5ihwS+PqVGP+CemL1hDxG+9vyOvhRe+z42Umk/OwKKvp+f473liYs++YP5vYTgej6r6FM+k1tPP0ypKj94v2E+yJoZPid0NzyhT0s9CROfPk4Hib3ykiO+przlPSe7M77DuM48tJRGPzpKOT/2JxW+F6/pPVupuT76BzO+aXv/PLhI9D7CF9q+fOPgPQ781D1zmYi9aPmwPiKuzj212lU+VpXHPvK/cr53Sgs+nVozv6EUZ75YYba9Yi2WPprcyTzOTCK/jvxpvt7EKT6+UJq9WTSBvYlZJT6/ffi9zUG3PcMZ1D0s+489gSyTPgoCGD22G6s+i0nlvsASZryZ/0u+0q6JPnGNAL4ccia9/AntvaP7dT7TzzW8r7Reu0inQT5d4wS+fvGfPXaIK7wjOlY9ZgLCvYEKKD3A+wK98+6lvnCw6T6AzEM+iZOfvBKaIj7I2pM+z7cWPYqXlj74Kq09xb7KvT5Bpr2l8+u+/88YPcreIr/BnHo+RTu7vTgZnbwzX2Q8zceKvr5rdT5dqS0+qCOKvuH+WL3snQS+tVx4vvMaBT4iIIO9TZ1qviRwDr6pEcQ9gS4YPRc2lz6sn6g8IoD5vkzYyD6KpTY+dEtavamCqr4kd+C8aS/Rvr73sbxmNfa+DI4rvgLSL76lRYe9R5XjvjH5Ab80EMo9ECojvgcIBr+9d268ULbQvlIEebp7duS+N7knvUbKeTyUd+s9V4JevuyvYz4W5pe9DKH5PjT4vD7csuk9wb5ovXS3Qz+bLNQ+4KaVOLGXg71hhBc+y/KuvfuAwD4Ahiq/P/UkvkPHVT4KLPw8sjlVPgpSMD7Yaem+rYbpvnwkAD7JwKI9UJQrvNNr/L0NOje+fzOovpZyzb66nv++N8EEvpr3SD10VwU+nPSkvv9S6D3Ccki+bQhhPghNFz/kLiy/","K176vbBcSz/CsAc+vCk6vcPzzb0JKom+whtNPrLbMb6ViqC+cfFbPZ/1Gr+gW3c9+OfSPLfMszzYpN6+YasGPs6exr0o5K6+Glo6v6OCmb5ARni+oudUvqz/dj7LeI+9Q1uxPqZRaT4Lera99wwWvNmlBr8sV+M9o4gLPnyT/Lwjab8+MkijPpmCS778gwC/AqqDPj9lM77qHgu/bkguv75CND6lVyu+TwecPaJVrz65obK+9Vw0vhf7az22D0U9PN1qvmUqdL1Tc6c+4R5uPthGGT7HIpy+qSatPt4Akb7CfHU+yan7PlArcD4BYrY+zxGOO5A7bb21E5u6WtfkvqjYZL+OqW0+KuoxPhu/mr0Thqi+tUQ9PoeZWT6yIqG+tQuqv1pxpj4qgya/vOAfv31eDT4KiZg+gC+0PnQRpb8uZ1K/FEx4Pf/jrL7lP8I+fCEYP4+z67xGZ2S/MlVNv2XHED+GG8e+Ulwsv5pfIj3z36+9XpU2v3eFTb+DaES/IT7IPcpcRD6dtg++T3WtPq3x0D1dJq2+hTCCvuKXFj/K6h6+RH4nP+mHyr8iBbC+HxLpvVqoMzx44vC+pgqwPr36c746F0C/f8OyPrCsn76GesY9hnRHPyR0BL+Ijrg+fdzhvc4Ukj1iCsm9rK3Dvr2V9z62xvK91N8ePv6jA7/XpHg9+KXQPuqPoD5CIzk+s045PYAahb0wiRs90smEPswMfD6rfnS9r3Y4Pg+XnL3/w8q+9HZmPRkVkz2PvUG+540MP7MvAz8cqeU+XBy3u6Ohlz2X/6S+QyhTPfwicT4+N9886DIlPVa7mb3OFag+4Y4xPuSTHT7tTbm9QpkPPpAWTb6e+GG9WPnKvbhK9r0uYqk9d8fevSMBtT6HBdy8D+GIPo0XHz7CcSy+eXbmPlkcmj4dEss+x4HWPoPdTr2/IhM+gh5TPuIxKD8pbjA9rBCbPKbE0z5ssfM+a7eaPL5rEj0qWAu8AyA4Po7BCb5DaqM+LIyPPf6C3bwIJJU+","tBmvPge0Ab+yZd0+6J8IPk3A970hC54+dxUMP+VoNj6qoqY+pxg+PoQVib3Kc3S+wwchPvX4eT7mJjK8T101PmI/GT2Qc1s++VUCP5G88j5apRQ+mDMCP9c7DDx3HLu+bmpHPhLCGb0eyoo9xpPqPdxGOr5C5w+/JH9mPj49Ij54nlW/sIU/vhfUSD05S5S9M0EPvRnDVj6ytAy+MxJ1PrKWVz1onAO9H0WlPb+sRL4LRc29CFuBPq1vlj5dn+G+cQHNPfQYVb7hyKQ9uawyvsHssb0UL66++f8rPqnDob2XE9k947PRvaxyvD5niIS78igePsMAe70IcMm+vs8ePlP8mT20jW8+oIdov4BxbD5pSYO+wXPBPYTmGj+NYj29SkqbvVVpCT6ST7C9qeNAvvnQkb7CFNy9aeXrvbYojj6kTwK9+/3GPgG4Tj0Es+4+PIOtPs0bBz0wuwS+MSfbvapKMT6fIOo8YfahPf1eh75AwAY9cJVZPRyTOb0zuEu+SAeUvuIPt75O1j8++HgQP7vUubzMDxc+eLIiP9MgxD3yc8m91F+OvYKHYjzv9YG+d1dHPmRXiD2mSwE8igmFPItj1j0nvw8+8slUvc4vOL7SBPE+wCf6vgi3Sr7zVlC94jiGvTRuU7744Gu9wdrdPhTBGr4sxAe8c4XTvc1LKL5sYSA/q58QvlH7ML4iXIU+VRm1PvoLIr0MC4m+6HoyP0j2bD9XzPy+v4C6Pqa70z5HgO29TzC+viQlmr6ryV4/OvFVPxtkzDvsp4k/NwScvXwhFb/IjbI+Nq3UPiwj+D6r+Vg+ULz+PBxe3z60uiC+v/ldPg+d6z7h2kU/onmgPzIBsr5kUYQ9kJHzPaxFpL4CPpK+algkvQG8L70brHm/irPVPqhzML8aOgA/y4l+O8sR971uU5U8qEzXPeAgD7/9iZo9hKksP4P+FL9Fs4E9yBTaviuxEb9pvvU+rZ5TPjVTDz0M42o9qgYCv1JHYT7Fqiq/2jMJviF2Gb7MiwU/","EZoWvs7OLr/GSIy+nlMFPj2vhb0Z9O8+D4RJPs6Dnr74Rqw93hS5PAlcbD5noIg+PJtNPr6WzrvEXKq+TvsovZwjAr6XOUY+BYScvgDgkD2VTQO+g+cVPuiegr1hDwG+VTq3vXSSPb6qYY69tOeNPRVpFj7b0Oc8pL5DPluzED67CcO+15xxPvYJMz5W+nc++BI9Pn8Sjj7s7gS9hC6sPh2qqj75nnq8YgoePRs+Xr7nxNo8xgMEv7eEh7zUVQU+o8KOvQt8Rr7XMzm8SxWEPq85U70Fuis+7WlIvcbJhb43iY6+abJgPTRYLD4SpA8+rXW+PEukMb4IxZG9sMVePm2f470wiMi+QOkZPwtzLj6ggAI+CxvNvkcMnj5xkK2+wIC4vO032r7tav29YBNzvc1jlT5j4c29DV0xv45wWT2B/8w90rYEv/WIBT1XbPS+OYfEvX8ui76b1468ZwNrPv/dhz032KA9EjMLPo5Hbr6BdaE+Gm/yvBhUEb5MccU9oGVIP8gqzz5INYW+YtOfve4smD5Bdqy9rocEPkXk5r5olMy+49eivRA7F72NPb8+vd+BvXnoA78sVrW+S4oUPgf1qDvLeW49koOZvkD1XD1Emzq9xLCivja0dL/YXCq/ixMrPr0PyD202hq7wE4mvlVSLL74T9C8ZXFZP4ckhb60udS87VW6Pm2tvj4kz7M8n6iAvkbvlz66syy+cmqhvrxppb5f4yI+viS7vghhfT3bFEi9GhglPOqqBL9l/T88YufQPTox2b6qqQK/r2zCvpF9CL/rU4K+jINAPte3ET0+2s299/lAPVvUUb1Q6Bi+UF/5vsYoBT7lARU++U8APiXoPT6Notg9PDF8PH4d6723drg8lp0WPgxYsTzkW8a+UybFPrTkW70MHSO+XOWYPsWOxL4nz/u9bd4FPgOxBL/vx1i+BhVLvdHahT4XCJe9m9/8PeUYRb/ygKY/Cc48vgcrgzxxbHs+OG+uvfjjaj4xACA9Yr3JPVoV+D6Dq8W9","Z2eXvydcUz0rLe0+DCeNPu51O74BVO0+vm+CPr73rb7nb6e/AAGrPhEwKr9IDiO/ZG2rPjhabL5kHpQ+7h+Qv1rkcr+WqFA+cD3UvlR5Qz/QJ7A+f7OoPbN/gb98SBa+TU9VPgPmD76baKm+tanrvc4NAr3i52e/4pVqv7htXL8SuGm+SxvtPpVuhT3feDQ/ogZWPu7agj5ym4G9E+CBPxisCT4ug5Y+B+LEv/cLYL5c8my95oqDPZFrKr9b8Ns+Z9vMPUCtCb8XcgA/A5wYvjN/EL4a0CY/8XPgvnyrmj6vTyk+M2Kavj3VkT68HPi+v0UIP5le1D1ng7k+vi6VvgmpKT4ux3s9UbubPr5nbj6B2Tq+sSKfPneXMr6teuA+a5sjPhyUPj1nFFM9Ukz4vR/LvL3O++Y9kAz6PJahhr07eHU+gsoMP9WjLz6jkHw8i4p2PaFdwr5m9BA+NNGsPTIvvT1/qoi+rtquvAcprj5JSps+79WsvNlgZryr+so89lKwuebEQr5fkoq9dbsrPaw73D1I4WS+0tmUvnqt7r1qQK4+9TyvPa8qwb3/kws+x25fvSGhiD53hok+3hl+PX8qMj6i4lA9ok68Pszh07gBryk+sE5AP2BLgT4kgoQ+aZ/UvfEAMTzqicU9hQY/Pu9P57zWjZ276CzUPaQF2b0ClDw+8ZPRvTwuuT7Rym0+47TYverJ1r2lUJs++NR4PmU4gz5GiNU+uBGIvpkBhj1H6yI/AENiPquDo77NGY499bkFPaeHhz7tLjk+C9JgPkZWeD6wZrc+X65wPXERgD7v+4C+WgIMPsPi2L14m30+mB2pvgAH674DVkK+HWjBvQB5575Ocle+YngHPKVnSj3Gha09YytQPlfXKb1PCOU92RUrvi9KZ76ZjJE9HhmJvWW8/j3fEXQ+pA7DPTBqAr/m+mE9/r+ePVBd2T7eihu+JaqTPYcN5L6g7AW/lmbqvbN2gD2FYwW+DHyWPryu6rtF8XG9BxXrPQk/777wEpI+","uKSEPZYo3z3xS1i/RLOxPqhxur65R/0+N2IBP2bThD4+Ny4+OFZPPrMiNb5zaS69GXmnvluGdb5W5aO9fJRuvYN1+T1KpVA+KadWvJUGDz7J2/A+B/BRPkSeFL6hKbm+yXiKviNHUL1WsCc+XPUFPkBRMj551Xq+CwjhveYRhTyPvYS918bvvujnEz6sVs0+tSO7vXoyoT4r0VI9HWoJPQJ1t7zmW6c9gSbiPSWJtDv0n9E+orGxPYRsMb632AK/76FWvUHgpT38HU68M/gwvTVm1r1irxe/7PEUPyWCB76MeyQ9Nzm7vsy+0juhhMU+rfU6PrALQ76CjMO+250GP7CltD7Qb7u9XDtFvk1zET42mDU+4on1PvWKBL7GeBs+cb2XP1ok8L7A8Ig+r+CpPqEiKb5BeLk9a7ZnvaPCMz9wNEA/H/NevjOrOD+CVwM/6UK8vlnTYz6GJ02+m5VuP6nyUb5iGA88KbAXP0l2sr24VaU+jn3NPikmRj+bwZM/yYSzPRn3Kr6ZxXg+lRBUvimqvL45QyQ9TxVLPnDFJ78WEYE+GczVvrcFVD9DmwQ/+KvMPnwlGT7FTOm+kgVJvs5y9rz991Y/z0JTvSMHkj2qVwS/ckkwv+57Q70S58U+q0svvqLOHz9WBgi+ODUuP45tFb+8mJS+tT6fPdhq3j7I26u+/0TIvmLBs745cry9nyyUPbQx/r0NHqS9accsvpiS9z0VARA+KRBevBcHTj7LpKA9OeXgPRCRR77PqK67alvUvsINmr6HafK8dWMFveWUf76xKSI+rBrjO1gD0DxOuiA+sLQivRcugb1y2m6+BHjJO/nbWj5OohO9T90qvW54aj6clnM8gruzvDDX5b27TXG+hMM+PrJtUj5xKxg8Sh8QvhWVc71BBOU9lFhGvklFw70ViKG+zQddPstUC70CmTe9n7LBPIY6PL5jr1g+yWbrvLCzU79WZ8++J2ZpvGCytDzsePy8cWBGPdI/Wb4w4o4+f/6TPde9qT1mxzw9","HR62vW/F/7zyFAS+/Aj2vSR9Fz0gH+c8p7OOvoSkzL64zoq+Isgav6suqT0kE+E8n2Ebv36GCj38ML09HtkovkN2Yr38746+VSFCPbsZ/b2tyba9A5HvveUDUD6MPJi+QfyNPc10eL7nxpA9JZScvSw2tT4lUOE+fD2lPfYWcD0hf48+F3xcvXJ0XL5zkH++sv8LPjJiNbzMAes9mYxtvmZker7xvzc9VtLyveLggj03Czu9TOZQvo8dib6euwI/CQwyvdBbCb52At+9Ib91PmdKb76ZT7Q+5EQWvExQaL12txS+VwNYvG2+o717wRE+xvYovoI9mL1UDic+i2uGvn7kHbyFvxK+3qmcPlfzQb2IcqE90n/9vqy6Eb8iHpq+LuFpvjy9sr3qumc9DmcKPSLErj6HYTS8i8STPG8QyT0ZrRO+bNXFPuHZij44jPS+8I4IvsnArrzhS1c+dswTvpA1Sj6dUBm9FFYyvc5Chj5bLO6+tQyoPmm4Az4UVRo8dLe9Ptz7kr2ugma+g4LdvvkWuT1l2Nq9GuylvZKmPL5IYIs+nDyNvhxhCj0zRPU96QgEvzAUtb6JZgk8/siYPgGZPb7cFZy+IPS4vJ6eqj3RI3w+XhMAPyY8kb2jonG+5e4XPj6OBT9mGw4+wJ2Qvt5THT4hHLA9KO2bPehvvL4Iib++mdJZu+LPPj5eqEQ8XdCdvRroED8zud0+Im03PXVcU78Rjqc+lRdOvsxhj75IdMU72BhEPSwydT72vHq/XzAmv+SdFbxKS1a9EyEqPnhP9T6GgEm9Y3OyPn1HDL/tKww/tzqkPTHXOb+70aw9ynOHvo448b515dS+C9CYv1LkgL2evjE8Qd4nvtpOK77yUgG+94povhzfF77HTAU//iMCv9RQ1z6NJNK+dtfPvlWXY77Tyni9o+TUPS8Xlj4HMc09d9lbvwEFAz4oRNo7fj0pvpEuLT/cNXO+GO1NvgjA4Dtt7C+/wqpIPsK/i79TY6E+iPvbPcxIHr3jTQy+","OoAHvrUAz77zEf+9V67GPVRejz1/jK4+2jnMPdS3V75EddG+VylBPtxxhT4EIrM+mRqsPs0iD77h0kG+p4VCPlTBK75vHUS+HN7pvg9IiD1Hp4e+KSm2PvtWLr0hXN89wae4PiRKBj04fhi+Hognvm+Ck76x7Ng9ywxSPuEs3r2JsKM91IQGPnx+8T2pFRa8GlwcPhQTBz55jo4+7LnIPey5Ir4io5K+jkJXPghNpb4mp3O+yZUVv42gA751Q1M+/SDQvM2KLb6R9gA+2W1HPszY973kJym/BJDdvuUYWD10bvo9SYltvSe97z22avw96X6Rvmc1Rz4/D4k+8PA6vj9ZOL7n30K/4tn7Pdyftz1E4FY8S23Yvk3zaL4SOyW+QP+BvkM5SD0wNpM8JLPvPfhM+r3LUMa+kBvDvto1AD4IjPy9GyD9vWbsWL6jSCW/YoEKv0VeWL7C70M+A4hqPs8R/L53GFS++0KrPZkXu74EvTM+2ULnPtK6xb1Gh3o+vyGWPzaasL6z3jY9O3OYvRwTJj5ml9Y9aHgFv1O73L5vZ7m+TJPbPQHHeD6oPRg+8bkuvPeKpL1IVbi+qTwdPsmeBT29iuQ9I+Y2vi0q6z2pAA+92DdQPU3RtL5v9yq/r0K1vqVf0byftAS/nMz/Pcwyl73LgXE+Z7OVv+03br6tDw87MjWPPT72pj6ckdM90tOsvft8GD7OjUy/MNSRu/Gxyb1xXNe+xI07PgOYmT5QQZ0+3bZAvgi1eL7Qkii+HOXqPSGQAj9RNPw9G2ntvh69Mr4KsYa+rvFTvTjZnD1+QC6+EPzcvS31kL2qxYq+04rfvll2ULxvrdo9ECSVvgOD5z5idoy/zWU7vslPGb+W1hI+41vTvSttXD/d4fi9HFoFP3Rik74HiS0+Jb2nPucrBL9qGp29c/2AvcYC2r40qDq+alWuvIUg6j7VqbS98J0WvmPGF73/vuE+hMCivUMJBz6pu2w+z71LPzlFNr7BrFM9eoC+vbm75b5VKCM+","QVJSv1ffAL558LI+yhCBPudCmD1AOna+JxJdvpMY2D3NHUu+NvNAPSGegT8v6Wo/jH20Ps/5Rb9tqvQ+IEhNv1cwL74svTo+48rBPcTgSr6Cjh6+rFGSPwQqED7f4O4+70ADvRMMnz4buOw+7JM3PZoYrb61i9s+Y4WLP7kuqr4JqA2+ht8Qvw6feL2Is4e/z7t8vlUOPz/yLQM+um6jv8kvLz4tScq+HLCnPo8ygb8U2+E9o64ovhlfub7NqAE/r/iLPtHh1b7HnRA/NnIyPUC7eL+a656+/HuMvtx0s75MWQK/ercdv7G/HD9m642+GL85P83uLD8FvC4+NcMQPg=="],"recurrent_weights":["h4agPAr/AT9qKno9RIcXPo0fcD2H5cw6Zn13PTQKtj7GCzA7dnonPoT1oTwbDXA9dEsrPLqJGD5CfZw+ipguPpRQBr1BFYc9A8eAvhIOJD5YHNy95SsHPjZ+YbwCRI4+tRLbPU1tMj3AhnE+c8fxPalZtz47NdA85EW/PU6KY7xQwIo8csP3vN8ypTujwJo97heLPfAYDj71Ak8+HEBRPBURpr2jxRM+mgwgPmY+sjsSPV4+4FJuPX5erD7/0469tth1PfWsZT7SFuK8bsK6vF00Fz3To7c+4n5MPTkI3z3vrT8/kEIePmbFir2FqdQ9/MHYPVcXrD4bGHE+qQb4PZ/p7D1QTgg+JVGQPfWeQLxNJZq8aZZdPXD7JbzHxwa8+3QxPjElxr3zqCk9BgqVvTiAqT2KWAQ+ZbAdPp2LBbwpluK9kaxfPoUYAT4y1jE+TkbMPX8EibvcW7M8TfWgvUPD5z221Ao90G0dO1F/jz0vGu89BR3SPahYBL2+V1Y8cbLBPHFcAz0nU286ZMfwPWQ+GL6qmS29USVGPKIhzTw6k2A8UugxPbb8jryYByY9JP/DPV9sOT6g+/G9OSlqPVfApj0Kp5u9YZbsPel1Pb1Qu5i7JdWSPVRELD5Mexo+F6tvvMWgDr7hxfa7RBYSPcxloj0tV9Y9SiPPPGqTWT7M4YI905B3u0mR572I9wQ+SN3EPSsE17xeP/Y99CXJvSsj7j0qY0o+nUu2PfWY7jwulaW9bKpbutx3lT1FqZi9o9V8PBcLhD3DJGc81FKFPS6THT6/TiG+MjyMPZtOvb16OYk9BTYHvS3zOj5f0QE+kYXivPTf57mfnbC9RbHEvE20yr3kk7O8Qc4HPWFrBDy0Ms698GujvH8bfzyXhJE9Xdj+vfKQBT5Q2hU9gXCnOiJd1T1//E89oYEEPbHXIT1p80s9htAGvSvCWT2CWJ883T8EPurohDxnN9A9SlsZvW4bvr1pdDO+zCVcPJz7Zb07ES89KKX1O3xzbT1nIwS9","Iy0FvX6QqD2aI+E9dGI1PkYZOj6Fdqs+GvG+vOs7LT7BrQs+1KDHPUieyT0HVq49qRV3PBzAQT5vFpq8w5nCPFQLhj0boFa92MfGPTUaST1d1n69fclEPjbjdrxqwoS8isIUPhw36LumdUQ+OaoKPVLSRb1P6pc9QOOYPXfhyrwXu1s+qMObPfHfi73aBWQ+6iPvPbn3PD67pjm7rnQUPpmwor2WYCa94pf0vUTGqL1XKY0+Wl7quy5+ubzsX2c8ceuRvNLlmb2a7xU+huZkPRtOKT2PZ7M9wMGGPaEmF7zjKv49EaHiPUphCz4hHdw91PSmvSjjiz1oEo09SSokPs3zGr7+pia9Rl2NvjAaAb6EriO+abRwvr7V2721eiq9a/oLvlYjYb5NZZu9nOAjO5jkFb6XjFS+93EqvZO+Tr5IId+9ITLavMcKjzyFEGy99JhevjVVGr7Pj7C9A9aJvh0EIL5rjN89qBBgvgGTK74X0Gy+lVbNu4D7Yj3yUWS9rQS/vS1B473kada8n/IXvn/CSr2eNpO97IAbPYjjNb70lp29wbaGvYO1yL3jIJu9qlTMvfMW97xSvqu+DzVNvt1dCjxdlsK91cBMvUZG8zoXJT6/5N84vtge973oX5u+NelSvbBxhr7WcMe9CWYHvMHzA74c55W+EKxqvbzYK72rdLa8ZClJvc9wVDzkYqE8CFfJvSPsnb3KMUY8PIjOvKU1D73t9r897dY6vbkTxz2+e5I6VZRYvUSzfLyz4da9kZ0IPc6ttb0C8A4+6UItvqRtEr4GuYo9M2gmPUnXvr0XYEC93daIvSHCc73g8fe8bxt+PQ+0m72h9ea9ODkOvpYPFjxZ1g49Z8oevJo+nr1hQnE9DQ+vvS2Qwj7/b/Y7ft48PZNl6TxJma88uTLVvfbc5DvUi2y9RvoTvek0wTxRO6g9x7IavYq72zy4/PC9TNzHPUtc7L0lxq+9XvkcvveHtDyaReU9tUuNvtPQeTqqWJ29IEwRPM7cXbzlhYy8","OuvZvFu/fT0s8N85ouN0vUwmuTtskSi7t0H/u9UcuLsLnWW8tq8XvVMy/rxwhB29pr4DPTmwDL0ugTu9TSX5POtLRj7LflQ9hSrQvaMM9zwLB3E9Y4uBvPeJUb3VNBE9UqgEvOcDUT5n2xS+R4OAvK/ADryNwga9VKCJPPr4UD5/Agk9nOdkvcpb8jzUNya8SQ91vJB/XrzgMXu+OCn0PXQzRTzsXZS9sTlwvWdCD7wjoai9qRQ9vueKBT25j8O8g3KwvBaIqbzNMFA7vXXcPSnja7zxux48gsY5val+TL6pAJk9Na1qPaGaWb5JmRS+/6vIvWyiBbznK0Q+0wufPTRIv7wyuKo78SylvkbIrzz5Uhe+K8KpvWqrtL5qqFS+N0LSvfCd9b3wULS9lK6uvdQPO7w6lqO+uuobPJL95LzzEUG9c1ZBvIouE762+ka+343zvPZ6ir5R9By+sNsevqX8FLzBlFa9rVIYvvY8P77QnDu9Db2Jvj22Gb5ElwA+qJXaPlJ7b76u+uW9bQmgPvsrNr4aI5G+qseRu8bFDr7+Hz096pEmvjxnyz2Z9BY834KevYBAFb4B0zK9jBZ7veHINL0aXw+9uCtbvs0gJrrqon4+Oku1vWW2Cb2tb++9c7cPvmpqTr73ibK9VhUhvtOBYb36goK9kzHCvYNjib6OMT4+scqSO4uXibwpL1G+pdqhPaZTjz3YmgY9tdhCvTa3rb2fZW6+z549PsP6qjxJPqa9bCN0vi6YZr5EAH28OUzfPhrlI7+7/LC8MvSDvfkRyT3lQuY9o1qjvGoJQ7yuJgy9g4lAvpb5FT1W03+9lESmvku0Wz65tyc+hcnQvE+13j4jZoC+Oc+LPvkGNb6OdFA9hGNEPgpMxD0b814+Za4KvnVq+bwCwWQ+Q0fWPM6zhL3zHki+SQSlvdtoqrwiYde9PKs+vvdjub6nRZ+9mbl/O3BMzL5cyxW/IqpIvoI4j755MBS+U4BIvvl8T76G8IY+CbfePfu6tL1QtI++","va1NvhFoA76sNny9dJwsPSdqrz7bpgu+quABPvZtXL2wWtc9L8oCv5P1CT4FY2Y8aoh8PVaDFz43bQ09c/NevdpCD77D8km9PgPhvhyVp748BXC+TbbGPGblUj2NcTW+46mdO6L1nz1y1am8Wcp1vk5d6z3qUOO97w3bvcEhhTzr0Zs7VAhVPp3j0D3847+9mJvhvKurxj4CmT8+28gEPaUCxb2zOS49D9rxPZbkAT1PIsm8fXuAPFlniD69rou+FVfrPLH3Kj6ecsg8f1kAvhBSUz5P+pO+YaHBPRkE4r4wG64+pZPEvG9jmL6beOW9LGfkvY6cMr6rJvK9LhQVPqiDw724Q0E+lqdpPiLeM70M2Bm9Q2psPfK9dL10eSy+2/ACPpTB3L6JzJs+UCUePpk8Q73tqNG9v4u3PamiHr78Hje+kNuSvuUwMT4+oIW8BC8bvjPHoTzMuMQ96ESHPknjr70Neik8xe6/PXC3j7xq6VC+GhWSPVLSAj5GB+s8V0qQPp+ONz7b8hM+fsoHPpnzOb5XRkI8g+x1PgBxzj3v6II9P6O2vQTMhT57VYA9uNkiurWdqD412yi/TtxYPikvqzslMRU+9goKPmy6+z07nVO7PPV4PTAkKjy1fQm/Keq0Ptp06L0Rk4Y+wJT2PA1kqj33avQ9ie17vtTVRbs0EYw9bQvaPAmzc77Xt4a+4M/bPRWs3j2wjRo7aX6TPvq8ur0Rp668DP+EPqsGlrzzW80+YLShPdN2gr01lGU+vV7XvvGnOz6kGD6+rQERv64B7bv4iNQ9mlu7Pf41Ib56bnI+G9MHPaSfmr7r8ji+0QSXPQGZND5Otr29mBYbPprxmT3ehyu+wbmvPasfAz1iBQg9sneSPhs2Zz6Lrp+9htiJvaeaRz4meWq+557kPs621r2SRMu9OcipPtG/BD+xmfw9Z/wnvo3kG76EbQE9EMEUPq32+r4UXEY+bBQoPv7pfL5FDJe+es2Jvpsuor0TI3S963c2Pr/8nr0Lh1W+","qqzQPV3nWT0ywKI+esmNPKLZFD5LgLS8R3NZPncs/LuGYf0+dFgpvFThij5l5MC9g0JGvuN/Gz6Xh0g+TSOFPCqRKD1UThQ+XMpJPmONPz/M5wA+V65jO2HzVT4KyEE+H9HMPuYxjD1UImA+J4p8PmkSPT7NjDw+1JhwPv7JEz58n4c+1bH8PowrV7vy/Ew+DAtwPqOczT0of6k+4I5SPm9s0T5YrPo8CS80Pt8KSD4kE6k9yMPtPlhwbD0RNf0+fX0TvcQnHT6aNDK8ITHpPC0Uoj7vagU+KIIcPV9L2z3sJPM97xlWPiY9Vz6mxMo9wg8aPxp7sD5wOt49V+gEPkRojD73/pU9yKnIPdyd6T2UioW9I0GWPb70/DxGqwg+s9aYvWXwnL0qB+w8ES+Lvpwsk70SsDA+1siWPFSuPT7VVqE+pdwhvWTuAz4mA308QyGbPeIT473Kz7Q9QuWFPuvl9z27zQe+wfZUu8CKQLzacGK9yW7uvIxGjb0pcEe+alKQPgqjQr4mDv29PwFEPVdjN77nT9i93WCRvQBmnL22Nwg+MZDGPGtu6b3bPzE+KjfUvYU/Lj444UU7Q9wDv9n0qD3YMQU+XP2wvP1Axr2QiJe+UtLxvW3MsL3v+5I9rDXVPrqZJ75+3Ia9QjDCvGWERzzYsye7h9foPTSf971yLZM9UD32PJanjz0Q+2C9nXnkPbEjE75duX+9vCL0vfVxgbvyCwo8jp3lvdjfIzy+NF09f2FPveIIVj1KmP68JOprPr6Xnzx8O7U8xX/GPfi0cb2SahK9ZRa7PMPxLr+gtL+9zWg+PH5kkz39W0G761gfvSjDRb2Dy4Y9yTEVvZwtBb7ZFJI9yCATvuI4D70/0b89PhBdPTwQyjsV0Ti9/j4KvchewT1+G3Y987VTPVt0QT2Vvjc+2LFWO2PCubv6dqw9w9bDPY2fUDsF+Ig9uDRDPSqzXL3kKlo9yYk2vTyUGz6Q2Jg9U+wPPDzAyj0QEzm9PoonPkLwTDt6Kkq8","mesZPFxipD30zRM9dUW7PSNOxD1svEs+/+JcPqr0Lj72cDE9vkKjPi5xAT3BrDI+xBvWOl6jwj10SPO8G6pBvuj8MDwr4C49WWpHvYQgp7yvFeE9UFuNPY+1Zj07mVE+V+cnPAUY5zxRhtg9D4oCPlIkpz6KjAk+k+x8PZSIoz1fPYw+NpZ/PCn6jz4mMZq82ZnCPRoCej1qPFo+B0MtvaSfHz5eFNc8cgyUPUnnJ723Ksi9FhszPmlSuz3yQfK9FRi2vJwbXryEth0+aagKPmsycz57dZS9MaexPO8SQT50V7G9IlcTPrZYgj4oo6Q+oPNyvXurqz2LH3k+8mskPvZr8r1DbBU+nVh5PgMw4r0mGIu97bt2vKytij4qGxy/w+HlPn6/171Ogyk9g+1lPqw/ITyAt9m8FZ/JPSyylT37JhC8+V0zPRpZUT6PmyM/mw6RPnvy1jzknos+RMZYPoARgrxbx1U9hB04Pmm+rj4bnMk9rNQHPsHhvT6gsK894KkbP9HJhD4Jdsa9elU+Pn7bqT71VgM+mO2bPpLXiT74VhY/z7uJPsqBXz7T4LI+/4XMvYPvDT4q0Z07NzJ7PpBlHL0U44a+wO8Hvfg5rD3ye44+ybSGvHhf3j1b1Wo+K7EEP5aeiT4YXBC9D4gOPoJqKD/vG8M+UgP9Pfte8D2SRrg+Nac0PQ/v8r1KziY/YRIevjlZqT7q2jI9H/gMP64xb76h0P28kMoAPlo+Z74gmmY+bfBzPr4y3zxXdTM+sCEFPwe9/Tx2n6E8tjK0PdVzYD1l2ZU8+eKNPuVf2D6wGuc9Xy4yPacADj4kIaS9lfGBPWqO3b3IGsa9810EvvWWuD4OEhW9pV06vNzZuD1mWdq9D2tSvSc5A7w2gz4+bcJivH0PND3p1Wy8JjBKvDtR5jzP/6S+EkRYvSfUAL/xnsI8yoZAP1V7B739ntE9Hr2EvnX5GD5C03g9hAYcvS8SHT9HQZs9gtN4PQYc/bw2t809gkmFvfcvDT0gdo++","gLRuPaeuerw3nXA9BZTNPkEBSzz9KLo8nrMIvdlJhb3Fnek94CqhvWvCW7uzVtg+GyMhvuNnzr1ZN2Y9+1jzPVnOCz53ZTA9e1h3vUQIqb2SpUc9nr+HvQcfUT5jAp6+PJQAPUzOZr372ZE+piMHvZC7eT2gVRm+78uJvCPWar2pMgi+yNN2O4b+1rlaGoO9KOGevEyxiT5PPXg9YV4tPpiXXj3aBrO9EB9DPe/wL75gKFO+Rd83vrWwB72Yz5i93pfivbwq2j5fmyk+6leqvd2edr0LM2Y96r+PPbNr/b24yus+e18JPstUOz0D/UG9LObAvg3cXz74IHa8Cs8avmDRFr56LbS8vn0LuywWO71sKrW8GrB1PVC8gT6LZSE+JTvGuqy5ij7V+ce8AL6dPTj7tTlXC989/9C+vXLfhbuTloc9QUuBPhrR3LwoCKU+pqYePRV7zj7S1RU+uoULvSDYST2dMks9320RPqccnT5wTrw+iBHqPXYuET7afpw9hpIoPvgNS7pr92A+ALVdPjHWIj4RO1U9U31oPSB0HD5GOlc9CtcTPu+VFb42KUc+YD5fPtkeAb4MTw++yhxTvv3zLz1vXsi8qyyPO2pQ9j0Hdjo+xgs3u+moLT3up3A8TueIvSTu3z6pNUE+PEcavs/bKr4lLj0+vBOUPhlutz1JF4u+Mot+Octlab0FDwm+79CAvkxGvT213KE941s5vo17gz0Hsek93aMTPh9447xD4TO9k7YEPOMkfz0gXTo9GtOKPeH+XT5QRNC7vWo9PJ3umDwO+pq9wbekvvO3g76MxoQ6zpc0vj02lr1eI628Y9GDvlXqHL0SVWU9soWdPHWSRT1t1Cc++eT0vRgQlb7j9na+4AhPvECnQz69+Q286gApPn4RHT5f/ku9yZAHvtJsOL4nNrq9LgSCvuCx/72p89g88VcUPtID/zxYvw8+Y9j3vQsw3z0bKBg+jY21vQmuZ70tqIq+bEmQPmzvPjxRuAq+TPoMvvt2jD7wYLc9","sTMZPFkVKz5jJ34+IcNXvc9Wir3hBKw8tOq1PbDk+r2P8U89TJeLPYRTNr1I2f+82veJvfO+ZD2ylK68R5oKPUGmOT7xA+U9o6E3PV92Wz40lY+8RdLRPQf3ir2fRAM+zk0vvN8Qtb1ewyy+RrekPNK8Rz551qW9qZLdPc7ENL50A2U7wXNBPhKstr3e2gS9pq2fPfYyeL192YO9pa6TPTaqpz1r8jq9/JcuvdFPwT0/qxo+/nrBPbX5oD35uC08HGOVPQzBTT10WNq8uTQIPvOg3j0xA0Q7K2lDPW6tDD7zrBY8cIGaPepFML1ehgm9yLbuvO5wwr1HRe68T0XiPWW9mr01/Ik9K0ahPa9CRD5uPi08Hx06PX4mR71pFLu9WQAaParFqr3kqZu+4AkMvhA2nj5iUWm9fAs7vvn1ar5h9gk8C8YKPSSu8Lz+df09IlaJPQlACj1Ivwy+fhI7vccuR76JwCo++kQCvj6vD74xE/07EVGwvSiW0rz2w0o+fNWDvVJWWD02nLM98aAYPgYpIT67RxI9B3uxPaQ6NL1eTh09T//CPT0OFj2vJ2M7FktaPRKFP7yCC7+966msvcqxiT19X00+Sm5hPWmcBL3dhUq9EsdUvvDm1D38RiC9ed/XvGxr0r2Ci+I7R1vMvfnjs7x2eLi96mifvSHA572bJuY9VJIMPgMc4r4c6wk70Dpkvhi4M77bEwK9CpQ/PSTwDb2daum8KgBdvmC3d7525AQ8WCGAvvlH1T2AeoS+hyRqPRFplj6Gy7q+vSJzPOlccL1lvHO+GigOvR0wSztA/TU+xVkoPvEwPr3YTIU94RTWObBVir6GsYi+9N2JPQAGtL58IrM9i7s7PWK4g77yZNa9TioFvmH4cb1dIci+hgKqu80Uc701mcQ8ji6JO06PmL12GnS9c2KYvmm4iz4rYPI7jbtNPrgquD22Ahw8eJpavhRPtj2sj4A+dUskPhrVRr6u/5O9qBeEPTEQMz4zoiK+yrwPPvvvcT5wo929","LKrKvRnt2T2EfZc9+sVdPey5wT1CsME9gJrAvAS/8D02U0w+yunvPeW0yz0ERBQ+aLUIPqup4j21GpC7c6H3Pcg97j06TQc+lKgBPeTwqT3xnxG+KSEAPkflhz7t9pU+w1y1vTXV6T3w/ho+rfOHvWF6oT4RvjA+zVKevIjCTD3On9W94FXJPe4mgb2+eKM+HGkOPv5HNb1p7UC+mNWJPnZoBb4e+wM+wj96PeQUiTxhQwe8G83hPZ+jlz7h/0c+wugvPa9JSD5p4WE+m8AjPhORpz7qqSQ+GH94vQdN3j4WVB492q2YPiPdBb128uI8RHhwPXhZlD4qqJ89wZ3UPVOsfrpEZiu+X+r8vYnKgz1/sxe8JJoHPuHHA77xbxm9TmqcvW+6lDtVuSu90oD6vcVRub1rQfS9599yOnzfUbxvna+7PIyuPK4uFT28cjW7uTgXPRbRf70u3769j9itvLuVLz7ybMe9388Fvgfkij3E+3w7ipmdPZHFzj0b2Zw7KuKHvoEbT76SBwy8X8PHvfhgGr2zgBE94bOZPJAk7bwKJM49eNZLveOUaTynyns+XUIZPT0sdjz1cLE8Lon0vPjUabwPC1I9iNumPG7YRz2RTA++K+UbPTG+ML4Zh5S8vJ2wvUk4q711cQq+eL/zPLq0DT4eYsY9V5vgPXEipz2ZPQe+g1OQPop3Pb72Ory9y8IfvUdw0T0SmlY+OiNcPVPgHr779rw9ujb9PCgdDr0189W9rdgWvU1P3L063lu9pp29O1xRLb3W2GW9VcUOvn7ahz5tpz091YUVO8c6G7zCzPE+Wf/ovUKfuD2lHSW87QT+vdp6nL0gJa89EZ2NvYm22L2UFfG8aDoAvrjvtjxFE3q9BuCdPJgTwr1MDsq9lOnTvitWSL0ydMU9crEVvSrfq71IaoQ7kOYKO1cSX77W5+S8+oe5vNLwwr3FdsG9GzmLPDy+GT4GoJK9i27jPatVjT3pf3g9v/hAPPSy7L0M7Iq9E+lKvaPcNT4Q+S88","iOiuPaTPDb7Fb08+o4IevSFwmD1WmGg+QqQHPfyqpj2BziY+iEuVveL/Uz6wJ3c+sN4kvLAC5z45x4q9o/zhuy1Qhj5B2kC+abnOPnD30D1KS8c9DIIjPo6W2z3NNls9bHFDvvXq0L1ozqs92+SwvdSfNb0xjgQ/p1C9Pi/gAr47oq0+k9PsPWsRAj70714+tanlPTFREzyhZga+vtO6PqbV5T3T1y09IJqDPeoUAj4lQtc98pQavIC52T06zoQ9TzveOgq2Kb4Iu42+kDg0O6/V2T40GPk9A9igvaSa672XrSs+59pHPmPLY75hLhg+mMlePYxGjj14/HY9Fk4GvqN4Kr+gVly+Z3aNvrNCib7KCrK+Q3PUPHR3qz1ZelM8iv/3vphBeD4f2hO/skgTPlXkoz6uHAq/aK2hPfvoez5HbSo+Akj4PWPx977tihO/eH8ovAau1jodZk6+ov97vmp+6b4G/7O9Bn8OvUBS8L3E40W+xU5FvktZoj150Bc+kQ6HPmFEtTtWBwi7c5BjvlVvBb5nscG+xWYqO/+r6LwycUq+NRBCvAPULr5kJfy+FddPvT6JiT2I2vS8OPO8PXO1Mr6/ibW9bgeNPdhbgT4x8km+w0VvvXf70r2ldyi+GMTqPu9oU74QZjQ9TdhBvlBTCr4jSo++a3nQPBR+uL3E0g2+IB0/PTtmqz69DUc90KEsvQMw1b09dxG+oMFgvqAmjL41Pao9XvbAPfCH4T40BR++OyQHPORHuL0LRZa9yuSwvaj7crxsC+y8w6hPPjvawjzekdE94FD9PL9RYL1sbU2+5Uc3PUXChzzP8iu+Ic4CPdNQIL5lb3M/xxEXvaJXd7/sn8S8aOqAPpG+e71YuOo96rdnvpsI+L1Ri7W7ePXWvOyOLj3SDAc9LLp+vsSzCL4mzGS+mfIJvupLA763now9dXtYvjsNzTsjwEs+lUlevCkalb2OfeW9H+kLPpZv676w+AA/3eKnvFtCu7xab3I9CTtOvnKco71rAcm9","MtYovnycP72qwri9QDvdvHfvKr5qFqG8mEicPZ+5wj5qRu48KG/FPKTxkb2UIRm9BoxaPVObtr34WpM8DYqBvoaDgb6JRbA9ElSlvfVzjLyy97k9g76zPS/fYz06uiy9/gvjvTXVEL7MIi095Q75PNw5nj3PyI4967dVPOK3Vr1jM6A+svF8vZxSej3FBs08jkDSvcCsDL5XaT0+SQlAPBjcf72aeFS+XI5dvtuw/Ty4MBc+c7tSPhDaY73xUxG+f7qqPC5iSr501PC9k9EpvjCiE715dv498wGiu/2AqL1jCoS+CrLsuuS5YL3eQ4e90zUvPopwoT3Wka69YiwNvE4j7b3mq4C9opl4Ps2/nb3SZiC+oIoBPdaOZrxuj0U9zgVCvkSItTz6vhK+hFtPvJYjPz6kOKa98gxIvggUnr2ZpaO9747PvRuwNL4exwk/6gUBPo4SoL2GhDe8K2p6vu38XL5e/ze+HCjcvU29Eb6Gw8e+A8QUPS48B77U5UO91dRfvpgt1j5x8Ka8CN7OvVr/7711EK29QazGvtkriD2CtZS+g8DtPVo7fLxq4fq+SwJZvpP5cDuKKL49jzoIvyv5ar0fEgk+2ptxPX2nbL5CA4C+zd3MvOiUt76uSO87WU4YPj9OHjupMVu+eDNAPgb1rj6WLpy+EA5avi44wb3k4Cu+VV2KvjdB2b6QD8K+9477vfrHmL0MToq9CxijvnDTWD6qzZY+fkssPijtED4yjlC/WGwePsFuPj7IvW6/WrpMPfsIgr2RVtw+5AYNvrrq3L3bjWc9ZZsQvZvVr772+Qm+JesnveUPhr6YO5G+iGisPM9eAL7ntkg9AjgQPsSyCjxFKIA8bmHSPucFDL2tV4G+3mNyvSrNl74YbjU+AYSWOYkbZr49snO8hw+lvdc1Fb/pkaC7CEmnvlZ+HL9oWf4+gOY6voU9j74cuig+XwQPPgngQL4eT0S+eSFgvdw93b5My0y9HJZHPlNcWL4wfgy9WVpbvjP9Tr3GhIg+","C8HgvWX3GrtClSq+iVQVPrcKRb71eua9+2X2vHu93b2VbO6+KZQ8v6WhHLxrzoq9h+bPPY/9hb5pYnE9Qe7xPnj0C74dU+q9Tdo8Ppm6Jb0GYs+9aHejPFPQXz21h6O9DFyAvnNCv75szwQ9++PhvVvhtjywzKG+hqeSvhyrzL4bKMY97YgFPtRn+76ibfO9dBYaPsRevbx3VWe9z1M3vSWdE77yimM9AZAaPtwhtz0o6Ei989sZPiq3vL7qjQ6/9YyOPanXhb2nh5+9J5GAPpUsWr4ELW08n+cbvpR1trwtfvM8ApxevngQU75CZQA9zcmQPQhOSj20J7S94+vtvOmu0D1fKWo9I2cPPj2C270YxFC8mQS0PR3FjryK8/I9HxrjPsv3fb2UPwC+UjMOPpeZ4zzb/bA9nm9OOxHP1b6ADni+jsFIPVmwHT5YYZI9NYIJvLjKarwcmnE+GgBcPZDmMr3fFnq8HVXAvWJKMzwqA1O9pfhRPvItjj3wjpo7fSFEPk5hNj7iJDe98nAHvi03pz00f6O9DybEPX6Y+72YNbY9ktmvPeOxTT3eKd69AXKjO0doZDxlKis93yO7PoAoYb2Zhze+RyqcO6+f/Lz+ENS8xB60PWJXJT5/5Sm8LOoOvQvC0T0Qby8+3vSvPbPIB76gBc48DA/yuogM9j2q+8c9xgY1vcEz+b7d2DS+eX+IvnQBx704blC8O7gcPt4xLrxtHp6/rEk7vrkZBr9dJ1K8xW27vb/YO73jnni+t9oXvoErsLxPusM+RlGWPhwvOL5kqm6+E+ODPQT3A74LJdm7IQGZPVAL8L5KjzY+O9HWvnLV0T1iPJa+KylKvuC7QL0N6uC+e6/CvUolib5031y+zLaKvlDRNL7KUMG9vIMMPgPaST79DxS9v2WVvqgnDr+4N7i96GeNvkx+zr7q7yi9oiZTPYxzG73GPUS9lJ9Mvsg2HT1EfmM+NfMlPoik37uUKPU8mBKLvgmSSb+gKpm+w0ZCvin8o702VnU8","XNDnvfFJmz225Ne9zFXaPcHiurwoLs8+LqytPnOMkb7JqDg+CYQlP3Ll0T5Uj7y9dKHOPhi4Lr4rnYk+haD2Pk81rT6k8o89gz6aPtOkiD6pUY0+u6+/vBbIXD7w4yi+z01XPR+Cpj06ozs9Aa3BvSK2bT5rKvI93qL3PKzYHb7E+ZM9LrCyPtnvMD6DaJa9NKSjPnZRYT7Rxw29o5IIvsMTpz4FfN+8sGryvf8YQL7Mg9U+5b6Uu3LxIL7eGws+oMS6vUgsIz70jU4+lYNCP7eivT65PZs9p3YCvVbwCD4CVoi+FoQ8PgVhDD2wOVK+7nbHPWDKW72gliA+Ht0wPvzt1DxUBY0+mh2cPipV677ejJE+W4dePZnQ2j0IrAu/zUQ+vxZp2TtcWcQ9LgBNP0RpC74TOqk9+tCsPesDzz5VRes9ja4vPioheL73n3M5bf67PLRp9T3RdIo+OjGnvo6HCT6pIbE+DhcTPl5mAz63OQe+rYlVvxtFDD959eW8KUscv8/jUD0Dl6e+DHVFPg1iYLyICLS+zl8MPi86r70NR2Y+rAjGPYlfRT6wpsI9o9cMvXJ2gr4v5/E7wkKBP7Y9jj58qmw+vByRPMWFbj+ce9G9gB67O7dOQT31Gcc8I/P/PkRyDj3Nb2g+zTeCvcGpW7vrnIu97ka5PQgUnr6eIg++zW3lvTUOED6/kD6+MyyUvPNI3j0xwtS9ZJW5PDVZab4+BkA+jzEzvl7y/r1tarw9GF0DvYDkij0L5pu+OylCvCYrzL1T5SE+qT0LvWMSeT3i9Qw+aHIiPgAoK75rFs88YXpwvYAnTTytNrK9/5kpPZGKMr7+E9y8sIctvfGT1j7PHhk9H1SnulkcAr1I38895jIAvOAnNr6jun69s3cUvvGup7062Cq+P41XPoIsIr1Kwjc+jrAKvV60277m6Am+5CCYPeWqcj1mwZq+N175PGaj4byHDN09Xhi9vXv43z0uow8+81HfPBS1ej00P1s9aVjNPSjuKb0g5f27","ZTCKvmeJoz4yAk8/MAnbPlkGPb6qX3w+zyOkPqRyZD1Q+oq+4x0uvcIin72UnEo+Vrq1Puw+CDwWoEs+79fXvaOHEr1GuTU9EsHcvdgwCD62VsI+Qu6wPrjjCj68eYe+4AJ/PkYBIT7cniK+ZN9YPcAT9T0sh3g+LmA1vhK1LL5KitA8CGqfPm924D5ze4M8HEnEPWWFybwfADa+FhgePLQdkD7ZDAE/2/6RvDB8z73T+Sc+NMjrPZx5rryvPMi+k9O0PYAlprU7WEm9VGvzPryg1Tx+e54+agKmvo3tqz3Fy6o9FubVPqIKZT4NvYI+7XJOPrSj070y/q8+7GeiPELIIL7pvwc+uGGFPpf16D3NzUk+5wMbPqa8ID5QCxU+HpcjPuGSnzyButu8tiC1vSwXyTxlrh0+RN2ePf/JAD7ONdA8Ma+Eva+1cD3FDME+mDAjPId7tbzljwY+Pz6PPuTRWDxhTMY7ZKtGPkHsqz60B00+Qt6cPl9Swz1yju0963O6PmTkpz5BXA89EFRkPqsKWT6tXbg9jRtxPmvnaT4BWeE+zNhJPfmAmjyNosC9vvXzPTEhsT2YdlY+c9MAPgA+ET1APRU+jeltPdAZ/jx/Lq4+rtgrPUAjkL3ZtCo+YnE6PtX/0D0vYXy8Y/GyPNDEbz5MxdQ+b1AGPkpaZz6Lo+g+7UP1PTU/mr67xjw8r4z1PD2yCr0UQGk8OKPPPdRkpj3uRWm9mO6gPVsrN72eBj6+n0GAOpgJMD6RFhU+Pu9cPuavWj3WwoS9sgGbvOKutLtzupC8a/PRPf8yWT1FWE0+cT4aPJEaUz4NkYq9hWqTPd7AkbznnLi8TkUDPhzvJb6wz3G9D3GhPSlc/D0QBPk9jN+LPNKT/D0d8E09CSrEPXYZdz17SBy9GN3KvAb5iL25LwU9NTG4PYUkLT54i+U9tMapPANXCz7pEqQ9pmi/vf4k6jzFqPi7TgwiPIrIBz9LGvG9QukcPt3kxD1XP849eRLLvKhdHj1uJQS9","ZuyXPR9OJLwds689+HrFPUQ+ILyoXkE9GVIAPNue1r1PboO8DRg8PcloRj4xt4w9WP8WPVMpqD1Cu1u8Rd0UPidRnz2H6Qa+3R1ZO2C5Cb0hOTe+M1g+vqvrCj7POay8n/yGvce6bz1zAo0+WacnvSbNwb3O4BG+EUkjvdpe+72AHcs8zs6VPQYF2r3gXxK9HI8DvOKLs712yua7e0OZvf5BTryo+dw9e8QhPUy3aL07pnm9laVYPf5JVj45cVA9iIHbPbhykb36cjc8t3bfvZasKbyZHdi8p455PVltIr4z16w9SfBqPOBCPT6Fjzg+cNeSvWNHhT1P6TI8Pqs9vcmt5T2aeOU8xQiKPpONfD2f0R8+7Kl1PmGmgj4YTUQ+fIIIvrUMyz31b4096aapPZlvsD0Ytl4+C2CJvep/nTxOu54+546cvY6+P7zcsBw9dEdVPvzCmD7OhIK+ouGau97ZlTzFBBM7d6xfPaZoRD7YNqA+xvgwPgZVoT2Mi909QzyaPhzM7D3HMEU9/kLPPtHdjz4dKOg93pAVPsEO9T22c9M9sUuEPQd/jD09/t09sdokPrVvgrzVwKw9PrrrPShF0T0xh7Y81sIMPd84+ruvaI4+gEE7PbvkC72EgPs9gt4LPqZEAj6Bmji7qneAPsDnxj2PfRs+RIMhPrWoHz6Z0Re+zzRQPjWvjz5U91U+29LFPgK+Pj547Js+AwIWviseF70G4ck+pweSPtQMbj6DPko+JTMoPjNPbL7ruT49p0aavop+JT1pyGo+LR+ZvUJo27xjIDS9t74oPZMfoz6iD4a96zRhPHiBOL0cHBg+SOH2PVdVtD4i55K9f7XFPtT4o7wf8B0+xLMwvu1px7xvnCC/Rv4MPr4MHT4+so4+KAz5PerU6b2dgDo+hCaiPdRmoj7UrUM+N6SHPqfujz7zoQ682Fs9vYF1AT6AjTu+yKsgPjrRP7y09i6+/yIZPqevpr73U9g6XSsEPvpVRD5ZM1i+R8mUPS7oCT7qC5e9","W4hQPlcAjD4Y7JQ+nRoMPes2Dr3Jc36+lNe7vRgWgryhl9k+GoYnvnBqkr2nM+w9ynGgvXJPWb7GGdA90FWEPmiXw7xU2R0+8NlavrEONj4P6hS+Xg5SPuCFOD7OlqC9JqAVPEivqT5Xq8898JM7Puq/ib3hyq69/JnJPtNbwz3VD2E+1vxIvll1zD4IwAA+OzHuPodUij50RrI9MbWJPphuub1m8gA8JG17vucxJDwfuNw+NCcVPNm8nb6YeXw+3ShnPjgQwL2xz8Y87gfsPptrpj3RwN88TOcSPu5FCT6CfBa/zFzCPYFoprwgKyC9P3RGvXmT5T7ZnRO9xjY1PcCKRT37c5I8/tT1PNtKtLu046W+algNPSKupbzhdqu+3XzbvjdXBb1tL4O8w6rBPeVHTj4ZxoA+JK6OPUVHLL7Yqye+nfLfuzpLBj40b4k8gHsZvmRggb0YzQg9WcpGPkaJm73g9dk8FrCqPRdzGr1PapE9X/oEPn8NtTx4o7+9kEZ1vU1mgjz2cxo9umAbPW7CIj6lVnA+09WcvZavTb7TyNw8D29DPOLh0z2bRLC9TiKDvR140b0S/90+cPr8vSwiBD7N03U9c114vf3N/72lAoO+MoZjvGo3pjw4Etk9XudHv78Z0z0M0Ia9PGmbveM6Wj6EDye/yS4Avd20Fb6jexC+VszgPrVHhrzpEpk8c53GPrhX7T3KnrE9SGKWvVqssL4tsfA+SsX/Pd1vFT72tKe9i0WDvH+GfD7IMkc+PrpFvnQY9LyvD/O9aBPUPU+eiT0NdR4+00cGvo1jqj3akCC9D7/CPWM+Qz6bwy8+Uba+PucuPj0lE+C+6GC9vaB26DrB8oQ9ZSatO5GXyz3UoSw9qR4PPi9BYD5puQc+/eAAPoFOGD+/oS69UpOnPutSBT56G9S9qzaXPvmNRD2s5g4+yBgNvPplmTuQsT0+3QHrPR4RXD3xA0M+lmpTPntdwj7maSi8c2DKPWovlT6Avec+Gz8Xva0xNT5JdFo+","5HpJP7rwA78/qN89BYO4vtLo7bypdCc+6IW0vanSfz6SP9g+xTU/PHyIHL7Y9zi+TMT0voS9jb6OXXS9pIhnvhK72r1U7169Vj9ZPuvLjrwhi2w8p04HvqCc0b1F4Wi+gCTUPn1EU712pQa+hv0WvQoAHb9fqMc9ExHBPr6J8D2VFqu9ZxPMPqKFsz4hFbm+fTvgPUtyP7ytsOO7BSLoveO/ZL5nLiq+EVANvR7p6z7D2Zw9OiFsPfGFuL5iunY9bPssPXUZ1L1+XuA9UNMPvsTmab71W6G++vWQPiwL1b51KXa+7z4Mvx2XnD6NVce932bmPj72hb7Fhrk94TQFPkK/hL5Xi9G8Hbo1vsMxHD7KuhQ+yhGWPovDuj7vM6o9z2WVvsnYIz5hl48+yj+Ovr5jzz5gk2M+wGY4vkKyUj5kW4s+XI8MPq0djb7swaY9kC+XvYIZcb3hmoc9atBkPpmsBz9eFGe+6NBaPlwnpL1UXHE9IDLrPH3u/D3v0U8+3pKxPhCNgz4JsJi++pu+vXBE5j1ptqa+8zBsPfsbw7xPbgQ++DBlvUtYWz5vqB49ZaBQPfwq1j6qppu85xqevibkoL0YEMU9YwFWPmtR271aXC29j/YlvdVdZj7UU8a+gsjHPdK6ND5W5zC+ftzrPWoCjj3C47q9muasvQ/bjr2QD60+bAUTPi/RCT05w/I8fQRlPmj5o7rU+hS+ilenvbKLcj3g5aS9pqJfPcVJXz3JXO29KhWmvQY08L2PwSC+wvvRPXvCQL6rzIQ++I92PWFwtLyiJY09DaalPT2D7b0PGtW91jKMPTGJWz55S1a+kl2yPKQMXz3wj0c9BTDJPVrsmryeHBq9csSaPWMoB7y9CgE+ree/PY/FBr0OQiC95l+uPOx+JT6asiw8o3V0vnWmCL7EQyc94AHEvcQrQj6Ydxa+E/+fvQjza72Z8Ta9X1gFPB3lqrtm00y+8DFCvomzYb15wQy+MYcHvaxuob3nUkG+juHJvDOBKL3rm129","gflAPlPxOT5+LiO9k8QFPlGHKD4982u+A3arPRgInz2WtxA/LmU6vq15Lb4k6oG+TqIgPuPRYL4ZU3o+OR4yPkEfVLs/Vng+qG6SvRXwfL73Jec9JnLzve+91zynsR+9pPcGP8Q/OT5X72K9TOkIPomE4j3R1Ly93zXpPlFVID4pPkC+80oLvoItYz7NZj6+kj0NvihFx730Y7M+/uv7vicBDz4muaQ9T6GSPitdor0tJYq+2ugePkYtgD6qHmg+g9NdvInXLj7zy+k+gfnDPts/dr4g68+9Fp20PTPmjj5AxWC+JnqZvWmapD7tI6w+to7vvJK0Pz7rhCE+APRnPIsA/zzG7YE+V84bvvW6cz4g9qw92Z7BPAsF4T4ajxc+T3GBvEPOZz3qN84+zMiBPOyGIz0sTyY+edFnPv/FgL6VegY89YkOvul00z0silc+RLxVPr+mm7141rm7q9jVvSDWCz4+5uS9pRzuvY6IEb6ULJC9wsMGPQmD4Dy3TQM+BQz4vQ8PNj76Mwu9jzE1vRE1h70X4g09dMW+PYjo3T3lDcg9fig6O5WmIz0/A6E8BsAFvRJ1BL1Lbm2+QXlkPhyYqb0wKAu+FNkDPei9Jb73aQM+iSsDPstHST4qW3o9mngPv757Kr4h2XO7lCCEPv1/1j2aMHK+wts1PkLNer5ttYo9lcAFvn1lXT5J5Ce+NBRqPcx27L08gSM+H4YEPrlTzD1XjgS+tazkPcqEVj0Ai0A9b+mXPW3RQL22PDI9ahCBPf3ZjLxGRoM+rOJ9Pt5mAr7L8Bo9/O2sPd6uvb7dQDs+JMTOPRhcNj6XniO9cbIZPSTnKr14IKc9NuiEvReVsbt4T48+y4PDveKLwT2ibz89g28FPXf2AL2hrFc9l98RvuBHEr78nBs95JzYPTPuhDzgUt09VBEcPkRo+z2lkwM9dn9iPQuPpbrdyww+tgfBPVkaAT6d8yU+gjnZvcbQj73v40o9GkU4O8gBmb20il++7AYoPQMb+Tw0X9U8","xB+dvIDFgT3Z9Om8a34IPio4Ez4h3x0+V4BCvYPUa75hV3S9bPh3vHJkMr7EvFq8AxcBPiVk5jvZ7SI7moAxPj/JDD7tCpC9tUI4Pr73eTwyRqq+ei4IPnByOj1H6KC91kHdvUPa8Dvb66U9zkbZvYzqvT10Lyy+B8xYPT2LOT3/ipC9Ca9DvuLqgbsvN/s90PkzvOQQIr7B4yw9JYBAvjOBrD27VWA+s3DtPOHZzL11Uuc9J7eSvMt8ML3+qmY9Fdquvg4HOD3LImw8z0cWPlMmdTw8Ywu9uK8BPt/Kor5cpSM8Ra1Zviicvz1eqYE9Yfy/vfgvub1YeuC8sdwFPE3fXT7qvGI9pDApPbIs1DyyN3W844zfvbL8wDz2bw09mXgDvpGqNr3n7KG9Kefau2TKe72WFnK9D/1tPttnnj7HVZG+bQpXPkx8oD6rquS98PwsPi5SOD2owyi9hSV2PCReMT66GJM+Dl0NPtZWzT0hGFA8BsD1vaX8Wb5q56A9j6OPvueRFb2rDR4+2BtQvvL2/b30YxW9YWeAPmUOAr6sEWQ+TcxWvRSrvz1rh20+3MAVPlKHNb64hoG9fKkCPsAovD2e8R0+GAEZPS1IbT5p0SG7jxZyPTMMmT5rI6o+odzBu0QJ4L3TKkQ+JVawPS1LRD7pPwI9cf1JPjIVS77PYqK9XemsvLsVkD4t4Rs+padOPpUuYj557e0950ssPcbNEj3Z3w0+S0FUPqbABD6Ax1Y+7WJCPp0uxj227lI9WVFdPjZXUb49wvo9wF1PvV1mjT442YE++ReHPSdMcz52+MK8wa6HPdKsQT5RJ0M+cdUSPfVOdzyY9iQ+alKzO5cgCb651mS9TCHDO6LYAD43FlU+K2PpPbF15L2BNHY+oJATOt4t2T2dT689BeInPTx9OT6Jl6C8vxVcPts1Ej68BRK9ZsHxO9iArb2r6Pi8EMrJPr4YRj1hrpc9URYNPhmF4LxzKzA+AQ9RvI3qJj6Q/gc+msR8PuRYCL7tMYk9","CjuhPafMkD6KL688nWw8PGUcqj0/hYG9DzAtPQQ3oz0mV1M+8QxjPvgWfr3mC2Y9m+YwPG3KUT3VCWy8AKhlPGiYqL2AA8c9NsRYvUW+Ory8gq68pKdTPZnejD0YU4u8oqkhPfu7jjyMpmg9qiYPPom2JD00giq97RiXPuwXO70uh5q9D9z2PUd+XD5Qnw4+Z3sqPYRwMz5SF7Y+cdmIPahatT1i8uW9L1rTvOzpCj5kXuA9FKtKPqRNV773k1a7y+CSvHEUjz09vmi9eMRBPO99AD2TAUI9JU8RPdG5tj0LGS296Ik9PvmX9DxmI5U9+HkhPUW6H70AHKA+P1fUPQFJ5Lz3SUe+P62rvL1lSLszx8Q8A/fZPQkIh7zaWFe96vOyvSbfxD1QUqK98OvQvMrmnr1OnNG9UqvxPVwfTrvivv29d/1FPeu+ozw1UO29CN+TupSsSb538da8ToLFvfOmLT7J2AO8z4uHPG6JXbzrLxu9O/GFvBi0B72gogY9CN/wPX9k573qBxg9KM/EvTwkfr0Me3Y96IKtvqjLvb06so67dYtmPKFNizxj5oa7DAItvT4Roz13s+y8Ix4EvRYzgT0XhRq9ZI0ivujVkb2Mz5u8BFhPvQGve7383eo9Zly7vX7SU73Oa+y83JQPu97QOz0gana88iKQPb4kcbwmORG9DAeYvMr25T6IpM07+IkkPQ7Ui7uTcGE+d8ioPtnmgD1wVUY+xErwPJXXgT4UP7Q+90zlPnc3yj3wW7m9v4QLvJ636b32gQI9oqASPhb9gD4bzK4+D4CuPF1p/j2Rfai9/A3JPYU9H76xeiU+fyoKPtSAzT704d27Z144vXCLiT66htY9JmpivTXPjz6uw/k9pr4VPmKVqD2CjbA+xjASvQAkZD4Q8L89SeApPkQ7Lz4OQBs+dg2NPgIYnD6x3pU9jzXvvZNvrTwZHwc+ckQQPyHexj6k/Q8+UhGyPWuTuz0OrHo+iw+Svarqmz2R/h4+kRwhPTA3obvaiF0+","m+sXvgA8KL9Hj1e99zGfvl0WVb3Nctm9g14EvsDjKzwGAAO+bbYLvrMPor2su529byGYPKfaR71h66S+lDpWvtspKr4SrzK+0JWPPLAUDb4bSUE8PrUTvXfikbv79Gu+fLY+vjG6gDwjv4++f4EDvsYUwL11uDw8EMGLPIctob16mkC+CpOavdAa2b0UCEW+0iC0O/vjcb4/voC+ed+vvENbpj3Zm0K+dLQTvnZcgryp5XS+Vl4avmPwlb4C3gy+8q1UPeZ6aL5ijPi6J8BcvGQFL74rP1e+GS9Dvoad6L0uF0+/ZtRDvquW7DyOlxC+p3HuvFup4b5/7pq+B61IvA3jiryc6i+9YdZdPf61DD1lqgi+qt5jvU33BT2lLhO9399AOwgS2j317Fu8M2TlvTRoV76fsse99HeavcxEPj1Cf8w4smpGvfN1y729zOc9Im3TvQadyb2zoRs9BeO9vGOHX73+ZwC+zyyEPbWyGL5zuha8oRQuvkMQAL133C+9bYmzvUWwgb2/pTu9oNMVPdIggT0iYhy98DdzvhV/gj3XKci9pgX7PUOWmzy6g5e9rF38vEpDbL5R6zu+fDEHva5iZb1dLS+8ruxJvjnfGj61q809+TLjvU8bHrwkezW+fF8RvdqAYD1GQxG+1mWePbUes729ydE8vyo4vUXDZ73QPRk9vXmkPWguur3hPBW+mX6aPX6Svr2iBSY9gB7IPI1ZVL2eCS49f8cGPVQRmD33SBs+kxEQPc9grj14Juc9n4Q9PV8Yrb0+soS9Az3Xu19BTb0pzhY938OZvdBs1TylTME8m/66vKEcOL7oYcG9BTO+vORUh73oLYC9/nlhvZt7bz3A4jK9kxqtPYhR270Mu508PMc+PJ3IDz7xg8u9TAyauwEk47x8MA++FMBEvTqKK70SiVi9jE4gvWgQzjzxvuY8amsTvpVrIj2c2p2840y+u+xRDD2X9jq7tLcivk2kJrxBLlc73cI2vRLLh7z2GOU8LfoMPtG4A761IKI8","dc5CPE+QG760W4u9PqwBvlvYBr5P/sK+wsH4vb46Q75Fssy86dadvjZ3Hb4pJzW+8pohvcTI3r1HVMC7ZBlSvNR3Gb6fDjW+/+8zvpLHejy75s89zpJ7vUwKUTtfub299Hw2vDKHMD5oHLy+sIwoPQ5QEb4Ihh29Ftobvn6KHL6Hz3K+yyVkvljai7wKSY6+R1MMvnatmb3eXsO9dOZaviDgZDxA6I29to+lvVMtED37igu9kWTSPcvnl75bUDO9HRe7vO4ZyDyui3m+LXCNPQUWhr4uV+q9AoOBvecrET1zCOI7xMB4vp2cIb7Ioye+1KrcvQUGhb0DcDG9oVfyvSE4wj5Vo6S+afFyvpbFbr5jUwO+zzpBPERQjL5ZDlC+tPk4vkAtVb5gCjS+5xeDPumnjryxUpO9YNl/vLrAVbvV2xC+w43IvfxFRz4mEvK+/tD5PD6RBLuJr2C+m7FCvtHldrxdumy+EKlJPkF8Fj7x52m+jpuaviagFz6AZU2+GiF4vmnZEb7RPq8+cZugvmXBhD181U88kwfcPdkJ2r6tQ6Y+kFqCPCNSjDykjUK/0CPnveviyb6UC+C+i6OrPvzHoz72KXW+sjtevfjgjDxpi7G+XKwMvrK8K74WAKS9/l+jOl8CB78A8zE9FLCjvqmSej14kV86h6eSvhNu/7g8fbe+xagsvPnIKr5DsSE9RHu9vQP7GTw2bRK9Otx0Pge94r6LmKG9nQ/lPaB1Nb580B29i0ZovdLFCr7Uouu9Xlv+vepQ/b38CAs/rP3XPXaDZj2hUGs+bNiJuzKwoTw4XMS9l/hTvg1X6726BXU9+Si0Pd8ylb6qyoy/hlluvSo1aD7gdW49eh1gvnd2Pz6LATa+RlsKvfFc57sOrcg9QuORvqImCL1Hiqu9pv/ePa4yMD6DV+W92O9WvbcfDr5ypQS/PnftvPEZHr40ahW/CjYBPoTY971K2Nm9bQ4OvlAYNz6wjDw+LkYYvjizZr2Uk32+5ZlaPtGWQb4Bcl0+","GnLSu7v0Tbzprqw9RlElvB6QFDu7rpy8K74bPffsTr1yYI8+bSnOvR5yzz1gAak9QDhFPfQ3HT0flQU8LJ7OvDvelj4k8aA8bSu0vZvHNj2i6Mm9bwM1PtruuDyLvZM93riOvZ7XRjxK7/i7fx+4PfoJYb2d8Wa91bZPPchNXz59cgI+wSL3PXagq73Ozhy+qoF4Pb+SQ74uLe08YtkpvRO7HD7jSGE9iZohuqmKCL2Zpsu9FJ8cPu71GD7dweA9KC+qvh4BML0tjT09X83bPlR5pb3yQBW9aBm9veHJk77EHzo+sIwNPizKgr3bYgI+EmBLvDZwVD5Bx6e9F7/8PbBsWrzDwmg9YiFdvtmfNTwLydG+vNcbvrlHOr3uMmY9a2zaPGxZD7/MfvW9hH0kPDeMQr5WvxC+G1jLvbXfT71oCGs9D3PyPK7Ouj1kRh6/7lVMPH6TuL1ypmG9/27LvKfhij0xK4U8jf+Kvp4uBD19aIu+AkKsvsyRdL7Vk3W8Jt9yvsoGDL1B6fU8hoIEvssaZr1pmhW+K5RYPuxhG77VAb+9z6aCPrjFor14Y+M+0h9NPgV19btrlp2+TkBJPceVGD22W6u9DDwqvYURdzt/rm2+3XqfPNdyCz7SspO4F3BAvinrcDzDYpe9pqfQvtCrTD6WIW88lCRsPZXORL3r2B4+MyydveXwHD6gRVM9K8LcPXoIST7vQyM+gl/oPe4OJj7RncW9glBePnG7VD5BXlg+HMzhPdCsaD4aPCI+zdyQPa+DSD3sVQM+5XS2PVuSIz526VE+zQ2ivcOsvj7Aa189thbSPkNHNz4OR6o8axbfPS99Vz3BPp08Zw1qO6jKpDvcSba8WnG3PXWZwD2tlZU9OwDqPQZ7Fr2A0k09RayYvaKAUT1XmzQ+upvbPVON1jzyiTU9uJcnPsk0TT0EANM9amoVPrBadz1b3mi9MErkPv0hkz2i22E8rjsQPhkBlTwGEwA+MeWfvX2Rxz1QXNQ9e96QPoFDLr37R/09","ML/kvE1Anj7sooU9cK8bPtW96j0K1zo9TG0vPu6PD72BawY+a2WsPtMq8j2ihS+9u6naPZBs6T2i5Le9NovauxoRlzygRog9/hM/vpYmtj1rsSQ92tp4PNGtKz11uNQ8CpfrPL59nbzvAbG8JAszPfua67y71vq8QFQMPsNyC74Q4Ws9ocAIvsE85jwDAmc8mL4nPdmuTz6hQme9RBtHvdzzgrw/tMQ9qxbEPHeJT70/XjM90aeQPqiX8b2yZ5Y6f0paPVPgwDzDK5Q9nYL4PIw0jTzcgTe8V4yJPV7F6j6l6xY9SPRMvW/Pnj6CxZE+3yGsPYfftr0qCtM71PCAPZHKqr0yjja+LxBivgWeCL2U/JO8DX8RPEnAg73OCii87Km2vEI6rj6nQNa9hRZAPq97Xb2Cb/q8k1tzvceHg7uc/iw8C447PmEXyD2xtUg+vVmNve7SL71b47e8Ne7ZvdAhLz7f99U8lcZAPFV64ToAAU+97z8XPHNtnT1Kd3e9azmQPbhYWz2mDtm8ReFavS0kjj30usC83KinvngZmj1DR5g9zZmpOtDfjj1M0ym9C6Svvax3Hz5xeAm9k2tfPA2xzbzDOh89LJXPvobcWrzOA2074oG4PZ5Okr20tfU94pNgO99Nmbwemza93b62PS/hlr2/BFa9PlynPWQfD75R6WM8CBKrva+GwjxJUCE+2MXIPTIXor4OFio+Wm2aPtPXmT6btk09FN+2PpOLJT47FQ8+L38RPguyIz2FnOq8AqmKPotkdTwGq10+QtW8vdxxnz3UcoY+YXdxPRzWmT4Tw+q99VhjvTli5T0Df8E9fuwGPfkLHT7Gcog+zmEGvv2+Pz7Hy0y9ydDxu6WCPL4O3xI+HLO2Pk/6lr1z7oe8Xa46u52kqT2akZk923AaPXyZQD28Sz0+FjRvPtgeDT5ejWm8fh6puvuwl73tYc49mwb2PlSchj6hhmo7QmwWvXQTmD2TDv89DpIfPRpeIz36Iv07PmR1PAo+ET5BjZA+","+YWKPUQEtT4Gyys+r7rvPmQu6rqsXss8SQqlPMjWJ7/+MAk+TbEUPvplRj0oqxI+ynUUPtaryD7Vfow+Fh8APsU+8z31JuG80Z07v1lxeD4u3gA+5l1GPk5PCD4ivyQ+t3HwPFx3xj1jfbQ9LxbiPbGJQT7POJ8+zUgNPghh/j22zf89QL6KPiCnCzy/WkI9MDEHPbrsMz5G2IM+e2kUPf5HH74m1Po883QSPkP+0DwknLq+r+5zPtIWJj0sycS+wPI7vUWBPD7x4vW979aqPtgWlT3nJYQ+mI8wPh65VT6BdI4+hth7PutfWb2R0hK9kdFgO9uOij68bM093HqQPvrP772KNl09S7zqOp7Ndr45hY49D8EHvuyVHT4Qeom+KKYdvYwslL7STKm9m11APv6jc76iwJm9HFaYPQToUD6uvdw72F3fvJ/exb3AQW6+94wyvRSV6Dy3N3C+MmgKPa05PD5P4xG9NcsXPhZUC72EmGI+/JbBvK1qEr+O+YO9EpsavtSiH707XGm+DxPpvHgGNL71udG9lq/Fu9rXJL6e4608fdrHvZZFzrx2eVg8b9sHPcn9oL5PuwG+Ph0wPgnxJjy6iAK+9cmNPQ0w8j18DwY+iIsvPNbE1DwyrSO93gajvXcZaD1ZQN89hxK2PYYcgrz5F8o9hIjjPYgPuTvXcsC9eXa+PbQrmr5PoYi9iejfO1gP4Du4et29FH88vPBiRz40CYU9AFVbPlZzTTxNslA9dKukO5ladL095j2+WBYPPse+YjweBrI+h1ScvVEpYTy07v49Xj0nvrwh0j3QlN09PqfKPJmtaDw3JHu83EUBPpQ+Xzyatdk9xIRUPbYHpb1x/9g9+dHhvFofd70JRYC+eaRAPcMJYjxVmuY9ipcZvhka1z0dR789TeBKvB9lK7xXUIc+Li42Pmi5lbuZYAG9GGRsvoIfOz0sKd69LK/3PHjY1Dtkf7A9kguEPI6YD76YMOc9nnIIPZoM6zy4fio8afmsvn0qDT0+Ju68","FDyxvcM1Zj0f6Hm+qt91Pse6uj39Vw8/TBCMPMtQvj1bE6G9A8KDvuExd72TYnw96FWcvdTBpj7FmU09V2QmvjYBmD72Q0w9JGeevRb/0T4W+u07e7Y8PjoqJb7HkvO9eaIgPkq5cL720AC9ifERvHIDVT5iXEy9lzeYvTpenb3Zd+89UZAbvmSWvDzdSss+2zc9vIb7Xb5wlGa9oX0mPsIKlr0S/lY+AATPvYIcUD2H9NQ+DMUwvvHUlj5dmts95T2pvICM7r0jz6W9EX4rvky8WD6WLBm9vWEEvnU4Ej617ms+0pjAPU9gJ74VUqM+sbsAPQQRNr0ZICg9xZSuPp3TUL70L8U9tFmMPjb00b0Lg+y9S3ogPg/0X75W31M/C+LfPT4hYrw7/t693cWEvX1bQD4vwhs9xGS+PghFv73XXxo97AupPj7Mzb0240Y+QmTLvidWDj7Ztue9ITLUPOxXoL4xKwI+Vl0iPtAul72axny9hoMqvjI60r1pDfC87fuqPVD2rj4JyIu8c+TWPdZAkj0Tk4m+i3jyus6Dk76eIQ8+4zUkPrcecr3dliG9FX7IvHeZ0L3pQsY9TfjSvgZ9CD2Hneo7XKidPp1UW70Y3we96YvnvR7djT2nqcS9jQUoP4bHPT4REIk+sDSPvUEfwT3I13c+v5lzPaBbgrx47xQ+iduuvXBKzz39G5o90ZxEvU/Qij5doaS9jbRIvnabxr5TXag90Pm0vVEVJL4Dd+m+R6k+vWCTWz5n10I9VkeJvRz5Iz2faw4+90evPT2y8T2fqlu+OKquvRddir6iW/m962zOPSx/Rzx6mrs9j7cVPFX9Tz1wr2k/VcvFvYxJW72XGU68jZ/vPPFkJL2AcRq+GZO+PcFdV72pR9G9HMyzPXcfhryWwws+lnYQvZHCDT08rXm9w3Lbvd9Lsr0nqzi+IUGTvRSWDr0wgQy/eeIavq1tBr4bfVE9zET5PfJjkb6ZXmk+DE8PPRzUbT5aap490ZFfuuqUTb19SXu+","T1oQPgFg5b2HSoC8Mdb4PSe6Kb5Neog7LbCTvYm3lzx0PN68b6JSPctzxDwqmYC7qEccva00Mb4Cx3c91egkPRARjz4ZFN+7dyrQvOgpgD735gy9QdKbvlslFr4Eyrm9hm8WPYKoY77rNuA9FKMuPYX157zfBO09SBS3PR6swDzlV/s90uZIvJE1trz9sxO+0hrvvRMBRL0yCBM9GCI6PoU0/7yc95Y9bC53vegWhb2wVyg+zal0vT1LID6wQaa9G54qPVbAbD16oKi8fcmEPT0urb3xV+W9B/efvf2DbD5h8fW9A0upPRS+Lj18waE9k+FSvqFcaD7szTW9PfcLvb6Jnb2P/2Y+hNl/Ph4HLj3i1D0+SE99viCk6T0Kgsc+GSRgPh1tUj4+g149zJqfPdgEX75kwLo84yq+PAzNwL3BX6s+413vPZLe672blf0+dvurPKjTHj5GZCQ+IQ5tvFWjpD1txw8+1SBJPsovhjwUu+K9y90IPlkxeT64TBY+x5n/PlVgEL6priC+QrlZPe4SqT298kw+MlZCO6FWBj5l5ks+3R9jvico1j3W/H897CACPqFZkrzScM890znTvlBgob1W4Y88TjmGPh4u3r1yN24+LjMWPvjcaL1cCeS9I7U/PR5elT4JBhE+8PQFvcFAGL1QIEU+F65PvUwpoz4A2bo9zXR5vvP3rTs7d549z0cRPjsQGj5QYIG9JJ4ePs+VET4kSCg9ZNofO2JAiD3TbxM+LMo6PosNULxWek08rcGAu9E+8j6epDU+ahYxPaFxzT1I7zA9gy1cPg4Erz6/t0S9/dI/PK6kTT4JN0W9rn6MPigm2z0a7iy57aL3PUP3Yj21Lz89sDQuvhiKjD4i8MM8kykqvQ0mbr6sRBE+AzFZvE00uzwMEsQ9hDNNvaABizvCnNA9T6DwPkbNDT4IlRi8LtEWPvLYkT25uQE+aXKLPjub/7yk2nA8nB1/PnxVkj2CXLk+zipjPJSlIjz6/n09IlCfPhJnVb18epA9","MjelPIgIIz06GBO+k5qvvSa10rsmxZS7Oay3vfiCVju8hek9o7rTPYSj+z2W6NU8VJl7PBsdoj2bl+a9PCmoPEaJlbwvHwm9PghPvXXfnb1CI0I9JfmkPV4NF7xLzo09rzqqvtUD0j2pK/k8xvMJPS6WZj2ev/g8anBUPtNBlrzJGjm+tFfLvXr3w73798y83wbqPFIE/zx/5IY9BLKvPYq1vLvDLKQ8qKWHvRER6DxGw+09LqeCvdMhtr1S1v48pzu6PR6Mvz2gpsw+RYWNPatPOz3YlcQ9GOFmPR6c0j0j4K49/E6cvd35gT7t+hU+aLoSvZGCsj06J2A+BUznPLVqjb3WgXW++WwRvIBQZb6MGb67u30EPBJHNT6v44W9vcLgvTNFAz73Mtm9ysk9O79iOj3oB7K8rlJePhDYRz3V1E++yNiRPv+YgLrCVO29AvphPR3olrw470+6SYahvG6kiT4Hy2y+p7sgvWqJ6j384m883fU7PRiHhDxA/hu+anzKvQaVjjySJao9i97GO7sDJr1HhqK9TYhNve8YBr6v1XC+E5W8vYXswD3qava9d9RKvvjc3TzTB2k9rtHVvV1ISbyoo469ydg9vlduHD7anFM8e4epPr6Mnb3RqIE+Lu6dPa7gQr342xQ91RbSvepPJzyXRI48AiEoPt4qor0eLEG8hqX3vC7uUT6Z+lQ9wRcrPuwIeT5WNSA9bHobPgtrrj45G6U9UumGPvIzFj6+m8k94tnXPuHPsrwKA7S9baSjPjI3Rr74x9w9276rPqsGFz6JCYY+mkhdPkF/Sj2PD6G9BuyAvTTe7Dwet1w93cXtPJ312z41I1I+dfEWvgxBZz4hCEs9FpqsvTaY7LxhV0E+rA9fPdsjJ73KqOY+dMUfPHbuHz29EaI+9Sv4PSKMVj3SWOm9+u7aPSp2DT5VFtQ85i+bvFIcVr6hhCG9KwXGPov/jz7vZXK9npxWvaMVQz4hzzE+boTbvai08j3zjEE9WgpbPQNi1zx7wI4+","2vVBPHm8kDuwYvy8gH5SPifWWrzbcCm6+AGrviyEi717bbM9te5BvZQIxb5cJhy8G1+0vKVwrz17cRc+TBvjPYap2jxdrUo9otiSviKmqb32ZiC+LX+XvbMmcj3pKlI+i9SFvqEy3rviPgM+/UA2veZjRj5H0ww9kp7NvU1aQj2be6M9bx2bPXvkOT6F+hY+aAOgPdRtKjxkjzc9Gk9LPn2r9r0wzI+9oU1GPQBROr5IGji9gsUGvuAJqD4PPre9wxX1PbO2Dz5sjSc8tL1QPu1ur7xUWls+lXe0vVFRnjy5whM+tXPaPTFlFD05WZ08eu7tvSQZiz7p7UQ+wKv+vY+Z0L3qvPm7kHQLvQ+W/L0UFli9OrVqvRXDUTwlyqQ8eZAxvmr6VjyNUjS+n0uqvUCtaL4zceK9XuJMPsRqTL42YqQ80/HYvcIKeb6rFL69CMHoPW9+Lb0gCxi+k55FPeS9Tb1EzKG+bwJZPismr7wbPl48iM+KPgBNAD0X4Oa9ijtIPNAD0r0CWUm9GeqovT/U9b2a1wm+2HEwvBnE77lb2Co97FBsvIN6Q7yopg8+XA4GPUTw3z07vr48XiEAvQc7hrxTue29kdFFPdMXR77/u4S6MnKkvVivAr1WfQM+X7VMPWl4Bb6Deuu83pu6PYn8+r0nHg08SpR0PJZEgb0Fmza9sLZIvsVZLb4gMI29d3wVvtqH7b1mARc+nEbsPJT72b0vbo49BX1UvgGWfj0pX029KNu7vRPPoz3homW9vpq2vZquNz7GiRa+AHrGPariED4ZkI+90dCkvSx+wj0WJSI+OdpSvaqUfj6OGIw9CgCrvX1CBTkQPAU8UVbqvcZCBr3Uy3a91P+PPdBil76yLjA8kR/Zvek1kr2DIDk+xhsPvvZsVb0T65c9XDITPtMpjrypl168mCXJuel5DLz3i4w7+TFAvVoSEL65fu49z4CWPfoqXzz9Jma9PKsiPnbfujtzX0k+3nBUvWtnAT1hveE9pAufPJlzxD1KmRA+","b5qnvTaeEr7GaoK9J/M7vsgeIj6vUoI9+aV5vUjyzjvJMaY+dmUWvmyrcrzW9SQ8GROVvWgLNT4coh2+dwOHvh148T6PFG2+8owwvDiDRz7yQGi+3aAoPcoGQT1VZC285HUOvsAPur5Hhh8+SWWpvW3Iu7rNmDA+CxKxPuP0F77HmC4+LuVSPtuvZr7gYYU+JxPhPVRyvL4KVbq+TcJgPgWX77xWAjm9cJUOPsth/zx+aTa9AL4cPnsgiLyw4B2+cuiLvcPgp71LXDO+3AX/vciu7z3uT8k8EGwovvTnZb5CFDw+ve+7PhTsAb5WAnM97kpHvFzbAD0ztj++fypvPhHXEr6IYSc+4HRIPu0FtDyypB0+A1FMPoVY/T2PqhI+eSjAvJ2rBj6suY69eOJVPRisKT7PzKS8X6jEPZXxyz1NdAw+Bf4ePm5O9rz6GHs+/sqTPeVhYz05OZO8P1aKPsJsoLwbsMs9ytAZPRtZGT4wNKc+mZ7CPZ4XIj5vFy29HcrQPY4hST7apXo9ky6PPo/3AD5nZY49arblPRMJhD6/kxc7gWzPO8/XZz6N30M9Y9eIPZ0rAT7dhiQ+SB2CPcITyD37tX8+ubQuPOrcGz2cmw0+fMt9Pmb5UD2RiF4+0NkMP6Pg1z436mm8a1ZsPFtg4z1PuL4+u0KbPrzwHz703XU9pPdBveAhHL2wG8m8lM3VvflUfb3DCzc9Hgf7PCRQlj3O0Z69tc1KveqROj3CYeo9ef0bva7f0b2bBC89tbhzvHAp6j3pVK46mgWKvYzbxD3F1TU+lAX3vVmI+DuoQoG8teMMvTBKHj42zBw97j93vOUvkT1rGio+UJI+Pc6cw70egsg8gFRnvIjoqT1nyp28rL6oPTQ8nz3DcCU8znSePfuVED3Mx149UlMoPmjxO71MLr89WOAxvQrwwD17KKY811gsPZj6Hz0wNt89UJUpO145V70OZyA9ThzoPaLFO731oz89ko0aPkp+5j2Ro4I+F4dMvXySZj1BvsE6","BvrCvEuSdL1AIMm9hy4dPZbfZz18Av09pwPiPIi0971NOwi+vLAYPpWJKzx0avs9FKZRviA5Nj6B3c69/3kavZ1zdr1RyZq8ywK/Pa8xKDtdNbe7NK/1vduNLr32HGW9c5JJPGUrarumM2I+KRdnvW3MHj5ZEUa9/6isvZ1CM77e7qy9ZbjYvZR+obnQOxG9q7acvAut8z3Sxou9hMJmvMxtj71Kj2S9VaDxPCh8i73lYtI8ZuYsPuTmdz1Jchy9z/MWvYB+vT1xAhG9PcyRvFF0vTzXMkI+Kp1JvdHYQz7vUng9DuIsPsjTAb7bPBm9AKNbvWEihb0qdZc9/AwKviCIML0tigg+T7sUPhzW3T3P5MI+Eh6EPpJnmT3zWYU+NA5avA5w+LyftGQ9uqqNvIvxHT5P1yI+sUnvvDETnb2V7oU+ERe7PDNMvD3z6SU91x0zPTvVhT4OGKw9lO/RPKUgAzp1opG9+s+3PUedED5ZMb89Rq91PfnPDz6pmSi7H/jRPk6tFLyR5pW7Lz86PmDW1T3EqyU+Zn4QPs2G+z26EIW9nQ6CPQ53j73eGBg+4KfDPW9tDj15sKU9SyXgvUbmqT2Qq247Uxi3OWKyAz68kXo+beRFPSASsD1hGmK9FhhaPTcUtz0KpaK95tKwPvSIDT3mLm+8HypFPVtDnz3qti69qjTevpVjKr65LSa+qnSavbysHr4OFli+rIqgvFlZXLwMzWu+vSmOvF7VZr0M5Ou8p9CJvcPie77Vtqy+gdhbvqJAHb7EttE9nQw7viATdr2o41U9afumvRFkOr4BmLe8BGwbPaNbgr6ATJm+jd2dvibvyL2ua/S8lmiWvIXiH75ipNe9NZVpvROVhr5PqK691ugqvhoXx70nLa86V0oYPYILu739VxC9RWfOva++kr3lPw+9rF81vigWVjxv0p69N0CEva2jKr2mU6q9W/xrvmarIr5BEFW+NWlUvo+g6776jYW+zqDYvSUaG77q0iW+hx2ivpP4Wr1ZYL6+","BtkXPKixrr1ZLZa9yjemvVnK8T2gKii+vrGVvXhoXbx7/vs9BEBFPSc10b1whC4+cD/Yvb9W+Tu0UlW98zMiPi6Ptj2EH7G99ER7uXejRD0iwuO9jdr4vGHdlT2Qw6+9A4pVvmf4iL2Ck5S8uS2lvUcldL1ijpq9PEIUPQO8r7xyl269Sv61vM3dIjy6IM66bk8qPb+H4DyDavu9GDo8vMbIhb0K3tw8Tg63vPGjNbvVTsS9SE6tvubs4r0inru8Fg6ePYa7cL0tcwy+Y+wevGEV8ryDuTq9WQ42va/PV762W4U97qh5vT7cgb3oCl+9qc0Hviwx6jvkMXC9zeEvPlNUJ725Ewc9zfYoOxuC+D0gIAM90XSlvbKuBb4Nbt89s1DaPMoUJb5JlqU87GOGubChNTwYmMy8sobLOhNEYb0yBpI9OxJkPTZGsTwtsos9gi72vOhbGL5Zvra7p6zdPl5yfzzcTQS+CVd0veUw2rx1e5u9NB4XPQxoCj2xCJY95IjbPafoF708nQi9BWtNvDQUOT3NCDq+x9O2vCvSzb3SnkK8Kt20ux8z2j2HNo29S1HBvdyQMb4eJq67+uU+PJY4W70q0Au+V4oDPntK7jz0m0e+fiWVvXKaLL2eMcy937XMvd0D5TztVbi9em1Euscx+T3HCE69EAZzvEkOkb3KaYm8fKtAPRXRN76c55m9S6MpvtYGlb4lwYS+fswqvthvPL7COwG+uO1tvXnpwb2G3/69SBT2vY1PVr0Xxau9+TuKvtZKp71CPS+9JWzXvcgrlr2sBYe+BAOePXTuYz1Ji9w9dgX0vYl9Zz1EqKK9XLczvifYKb40HVW9ow1MvKk6T75ybyO+mSv8PAq+i75o7HW+3f70vVpIL760HNi9/Eu/vd+xLLzAVRE8eKbEPTEyF74NdFk9+oYCPlQe2b0vddq9Md+WPQ+Wo70IWnS9PJ1avodQQ77qsCO9uQwjvctDhbyAdpS9+nqsPVUe6r4OBuO8LuhnuoKH2rxauq++","wdZWPve6Cz6zFV4+YrsYPqqeAz7cRPc9KORQPiaAgj3SRag9q5UxPZIQbj5curc9UzIAPo22WT77olw+EXjpO007Xj50rRk9gR36vVydET5NRBI+l6FjPoo4ZD2WOZY+uw2AvZJoVT3ZSYw+UqBmPrV25T30GZk9+SaavGBhTz5z/U69/Wy1u+tzjL3gJWs+RIUUPoJIMj5zPCc+OPGUPc7+w73iCni89PdhPrAPjj5oGug9UPFmPKCVRz3kSPo9YNAtPUZA7z2Cw02+Kq4YvUh7mj4scw48nA8sPgsKmT5H8LO9bM5jPe4LID7jJww+PfsOPuvwlD6+QQG+RnjtOsc1tj2O/Ok8h7vvPTFr9zz1S1I+gM2qPWysPz0+4lo+v1TAPSOkFD4zluY9UZT8vKzywbyWO+k9f3aUvYJN7D2636u94u84PfhRbL5a8kk9We9dPO01Fbt8BQ89k4Sdvdb7fjxGouq84dH7vPCeOT3ljyk9MT9KvdjfcrpM9u89NoKsvaGtwT0qi2U9HjArPkBGZL35jfk9H3B6PpeBoz1XPPe89OuUPaZ77rsMN7s9GTIQPuoJnj41qwy9WLOiPMf28j3ztKq9WL08Pe/WgD0LKAI+o/OBPBcLKT4ME8a83JkOPdIF1j0LGcM9HwKePWPG1T1KTR29p5ROPMdAIDtB5MO9euqqu+Gv5LtgocI93zUrPYxjsT0RYeA90l37PPVGzL3+oU4+SHxWPjje3D1vRlq9jG9IvbZjQjwC4a89MtisvQ9pNz1vkyA+CRuJvWzaa71dFzW+pjlhPWGgp70TZTC8BDs7PJIsrD3Pz4M9baRHvAhznT3myTQ9zwzXPCKns72U16G8e/i6PRXR7Lv/Oz+9grFGu5X7nb2NoFs8ZjKvvbfLXT031QY+acanPLKosT26/Ku89MXzu/Z/4zs/DwQ9sAvZvb0Gcb3E5TK9nHcDPqa2NL048zG9txybvRjsXr2v2m08UEgNvlzElD1E3Ay9kDTaPOiqDD7wO3S9","mqw2PUEmJT4f86E96h0tPEWHxT0rLeQ92BCuPaJKSz4Rn7U9sS4uPSghEz43knC84CntPUgLYj5dM+g9jXHMvQPHO76t6dI9n/MsPVGkVj0zfrc+avPFPeY60r0lPBY81uuDPXVELT4c5IC97DdpPpJWzD0u/L88e/h6PmNjFb7p8Jc+IY++PSe5I77ZhKk9/T6JPkjBkD08pOE9deoPPrcMVz3Pyrg+fYPrO2WWnD41Ei07TG8zPlrjGD460ms9iMHmPNw+tLuVkj89t7KvPCAWsz6Gjic+DADFPZSmkLo5rHo9oKN7Pb/orbwMrg0+OJnKPeVJiz3DNgU9wBJbvm9CFL6b3oW++v8jvkpx2rwgKxK+gYOKvnB9n71U7j++18YCvnke072I0u69PCjqvK7ORr2UKZC9jkXhvbQWob5j1AC+yAI4vPDqI76Q4Xu9g0amvSJpmb3S44895eDWvtE7Kb7hcw8+JVBAvr54z70FF4a9Fs2hPa5sKT2gvS+++tIWvmNx272nySU96V0IPvqncL72yBa+Oee0vg+gl7ypGUK9m3SSvl2/yr3iZ8K95fKGvumgNb4yuhW+OV0PvthOwbvmrRi+cuvuO5Jnsj1IjqG+ucXLvL5rubzRcue+0QmbvngoLr00nU++AJHNPMGRWb6KCJ2+FKgFvu30N7wA55A9yR0ZPX3wrT1AVou93r1fvcsRwbi/Q7O8B2jQPR/oCTyDJBs9k66zvY3Qn73WSNa9iD8TPYhVr71uRkU90IJnvd80Xb4QJYM9g4FWvdkn8LwcaTS9YYElvk4amr2yFwu+L9Q4vBPSwL2NqS6+/VpLvRdlD71wzSa+g8GZOpUOK706PEu9rbThPEitlL1SDEm7ss3kvXXcY720kXO9tfGMvd9HPL2B0Z69Fus3vGZ41b1C26G+5xekvJfxab0/7rc8mT3fvRFhvb0aymE9lS1APc5Fjr041Te+clUIPWzqtz1gR7a8qkchvl1lur3Pbum9EERUPtYuVL0ACaA7","WK+nvUHuBT2G4AI9izEdPXahr71f4xu+kN5AvMid/jwbh0S8IHkVvkODxbyeqpa9b+ODvd2VPD33G3W8cBmoPcWOHz7eX8K8l3CBvUxPujwRbSE96JqbPfPzOr66nIs9qHFZu5QsVL1DvY29S5JXvdeV2LztWtE7yzJIvoYyqr3yeWK8yYidPbPCJLva36i9GlwQPhR3xLwgMQQ9ztSpPMO2aj1uAqQ7qPcPPSkLdT1Kvlk7gTwTvjwsP73uRwC+ReKfPFdOeb3GM4s8fjuDvdu0Nb2JLBE9Rmw1PZorHTxNfR+9rVNAPrZQMj7nEU4+6DIiPYvkG73tRVi9IoyTPCYyfDyTbRW+NGsTPZ+m57xNMW29KQ9mvpmumr5uYTO+bG/1vR6U772HYdS9YKztvdqktL6fV6A66Z3aPNHsFD4j6ci96n9HvZYtnz3suhM+hc7NvQLHwz0BJDy9ZL2MvaI4aT2FBRU9iNiFvF7Z770VTGG8hn+rvesotb3RM609Dpc2PYXci73FRSs9oBARvUJEs76aMr2+wfgVvq28+jya7YK8CysFPuPZMT09xBm9L7UKvXxy57ypPqu9FgvcvMc5yr1NO8i8LqNovksVB77ndzu+7r0jPHgVVL2sOJU9oVqbvdnbO72f0Ze9sgyAvu/fXDu+Qw++ONJevUoxQL3hOOS9NmVbPjSYoj5pH0M+XoXYPK8s1zyARIg+FRcsPaKKBT5UTdg9EkJ/vTg7kj6VVyg+V1tnPgJiWD5v1AI+xgvdPbzNBD1dedk8CANIPkj97T2s6GY+WOuqPWXnmz7PX4W9bNLkPYoAGj5fpYk+TaePPr8Vaj75H4A9aLdmPt+BQD22sYM+rgUSvQ12JD7DfKQ8b357PRhuHj4eglA+GQVVPRSLBj5N6dQ984WbPdxmgD7CBCo+awt0PvyyOD4P9fA9Z3M6Pm3Zwj3yLHk+3MJKPo5vZT58/Ek+jnAZPsBylj6uJsI+rqc5Pmr1TT64nnI9eTCaPuYDqD5tyX0+","YDbdPOT5qT0xH8w9LwfmPRmDibwDtLa9wryVvO7TlbzDG1m8FstQPAAXrbxK1qY9mjvGOlLHWL1GDhw+FmvTPds2EbwNOvM9L4hjPo1GmL0b9Q0+voIYPStY5bxajo09FXaSPbF3wz394j8+73MavYcu8b1ypik+i+eEPtmSz7wuwWS93vFBPd3h6zzUAP89QP0Cvni9Gjy2Kjg8k1iSvfT/6z25sa481LhePS2JDT57i7o9krZbPYavRT0vZ1I5Oy0JPmr3+rwlpAs+3ho4PYW73TwzmJE8wR7bPX2baj0/+va9n8g4PaAfsD2+nd69nm0nPvAMmD6r+TA9q6NBvTQKg73Y0tQ8xsAXvVNOrzv8gzO9W2n7PLxztz125+e9da4dvmU1EzyazBU+bUwUPbHYsry7Ebi8kQs1PcWRqDw+DMq9DytevcL2uD3EkFi9Fkb6vKLndL7JmCk9aePBPPC8jz3YkiK9xD7KvWP6vzzheRq9KgM2vnwZFr2QmCy9ySAvOowhKD6QLos8I+6qPbdNOr3/VQE+yt/qvTKz2Dw09hy9vetMPamvlbyZxbC9BNQlPbVuDD79wA4+S7cAvpLJqT0pCW69+NjTvT6NA76MlF+9BhmePZXzF748dcA74RdiPehE8bxw+5q8JGYvvbXwkL36mRu+uuTMvRhTUD0zpgm9t+ZdPjoZEz4Vb649/yLmPp6lFT5WBTU+95JTPqB4mD08soQ+oo89PtM57z2GFk0+Fu7dPUgG4z1x0Ki9TxTJPSCIST6ZYtQ7RO4/PoRHGj6yxYs+cOGwPe47qj2tis49NW4zvYp8hz26Sl0+ncqJPSW7Bj7472I9hG71vNu1gD5lPqo+U1n5vIgnYz2eBmM+KKWsPUthMLvyX0w9WnZtPaPVWD2LQZa81t/IPW7Zkj6K1pI8G1LiPRfi3D2NmxU+xTWyvfU/Cr2iUv096gSBPll7xj23J7I98RmFPdBuBj73hRg+MtwMPhOBZT7fAxW8TjYPvExVOj5wrag+","QYucPNVjM70Zijg+IqeJPSCjij1e80c+t0AdPsNFDz2/s5c+zAiCvA6QOj7D4os8oGQHPlA9gzwH/Ty9sKEGPrnZJT56EQe9H8NfPsKtBT5UTLQ+QeeyPSGRFj4c0IM+XuwNvvJMtD4U0GM+VzG8Pdk8kT7jDXc+Dwq4u+XxDj4tB4a8omyjPXelor0nhyI+JM0MPkm/CT611Be+dNGLPlSCRbuxcms9TKqZPQffmz0uFXY9VDRJO3bDST5v2YY+BASBPW1OHT4svL6961jfPW/3mz5KiUs+bkkMvOAExj0W2b09Pa1aPr7ox72QIBw9YQsAPTljdD5kt5y9mEAkPotrjzzf8iE+ycpavA7Cqr25bpA8B3UuPaISojwYLJ69Xnj9PQbMuj1+HHe9C5nUPXxAgz3AQlk8JsSAPaq23T1K0MG8q3MAPk2qB70QSzo9CCpzO+ZXFj3KUhC7stiRPcnfLbw/CE27U/nWPQtB0Tx6CqM9BOGhvA7Q1D7ahIg9+xe3vSqOzj36ca89iEMrPsWXpr3YPQG9fZRlPMA2mr0l9kg9kNe/PeEiuz0G6+U73SKMvZV/4zxpDDC+oPt9Pdc3Bz5hKYE+ra+nvf61Dj4mpq678kaPPcJSuT15nWM+9yW3PDCnqj1Hh+y8W5JUPaIN9b07I0w+TkfePSeOFT2Mqru9owcFvmtOdDy+2gK+rxnRPQMbIT6MwTS6yTd2vFtQZr5yKpM9InX/u5LPFz68Vnk9SfrSOb6TFD4s6+m8HbsJvlKqLD2CXgO9XY8Xvsjjnjx3ACa+f9eOPQ1qr73b6Hs90ECgvWyBt7wfZhg8TdL1OzGRtr1jyAW71Za3vgMUTL6xYFW8vguwvSsZwb3Dg/o8Pg4nvTo1FTygsga9ZdZzPdUCc72SBlY9oZG7u9myILzBAhE+2U+PPWOoFL3H2BU9AYZ8vTQ/Cz4k1Ps8cQkxvTY2gT29kLy9+l8hPkT8a72HmAe+1Ca9vF81aL2G27S9lKQRPMOuIj109rG8","wo8ouvoGcryD1q0+GxMyPdtfJjzRbiE/4+WGPpQFNj6EW1A+d5I5vuAz3jqtNQU+yC9sPhVBDz8XtYg5JxSpPRcbNT3wKBI9WOvGPcroxb1nAi0+g9OLPoNeMj7KN6C9KzcJvv5CPLsPWYI9wNo3PY14hL0GfI4+W9xIPC7EITxWl78+Gt9XPnsdcD0XzNg+sut+PoJi/T3gbBe8rKUEP7hl+DxWAFg+e/FuPbRstT02ZsY9lG8VPptCJD6r/zs+kQcwPeTEXbzFXOG9hWvrO8EasT5SkZo+jIUxvX3S8DuUNCc+LH9vPlxlSL17FY+9L/sOPpjL0rtumdE79peIPpkuNb6JAW89UtwcvE+6lrzM0Q29+aOjvePjyj3Qbys+hdq5vWDlVr2uAkm+uklJO3Vb+L384ro8GOWkPbnSRbyTrTu+DNgFPoQ2Mb30vGE+0E5SvNF+Gj6Zoz6+qYKivraMgb2QNtW9cOglvTh3Gr3MlPM85VKVveBcBbzILTK9+IedvuXE6D0KXSk9J4A7vSP9fb673TG9D4UePs+22b3lkaq9vgHcvAhmLr7JS5893Vc2PtMsDL2pI9C9M8LcvVNdbrw/voY9NFKjPuwHPDwrfHa+4udDvX75mz1PWl2+CEyYvAgZZ70dvQ4+zOW5PdqBnz2ONY++9c0SPvoGKL3hfIG9DhYzPWgBcDxAFpw9kzbfvXbDIDxWPTq9zLiKve2QAD2T2Jw7/C56PY5+CrylTvy9eUJVvRKnEb7OQHi93nMUvQ7KGj5RyL+9fhdnPmTpFL7KUyC9OqqkuxTJML5utwm+tp+Evbi16Tx8esk98dCoPRQqqTxDzzu+VjVYPRYxiT2R9iE97WStPSj4lLwE1KQ7NNQauy8BOb4VQim+Pv2xvaHNILwCyBO+M+6gvfW0Nr2Z+Sa9rjuXvN2pBD7KH7G9vcfIvE5v5jxuTA+9mTEfPg6+oL0iurE9obqDPk0pbj2A+UG8CiRmPDHGmDzrwSo+1X+DvPOqdL6UgiE+","gZO2vPz2D77WewY+Cz2xPRrYzz19SPs8bGWBvi7DEr5H+TE904BcvoVQuz63UUQ99KYcPoobIj26IQC8vPw3vejiDT5/a4c7shrNO4zjej7TVde9a3I6PV4dj7v6wDg96evevTFHy71zmSK+vuWKvgeLZT5YHnG9ftrjPWGbBD6w1Zw7kYdDvgbvBT6p/Ag8ac0MvZndXD15yQs9YxMBvUF5yT3cgzA+MZ0CvWaXKL3uqYe764UJvrOGuj1f0eu9co2RvZEJKD7xF8E9UfaNPHONSb1hhEm90Z4NPpE+HzyXGkY9E2RzvS6YPD7sDbg+jYb0vJW5NT4Ky6+9vpANvhw75z1rQ8A9HCoevoQ7bT1mpVQ96Iwpvns3yb71AIO90G61vmyBCD4Dm6Q55SaCvgXWp71X2T2+g3jsPfEoGT4wdA6+/SaiPgZVz77uD+g9hAI1vgR0qLxKDck9u153vsgMFz509WY+Ad/PvMaQeDsVo14826LWviOUr7yMW30+TmPavioVRD1GAfe9CVVyvo6cZr5eNoW+TqPWPKbMiL6ox+W9R2AcveQgir7K2Ym+aWP1PY6USr4bIh2+hqwUv6krND0xx9Q9RBioPjAGObyaOtq+y6a9vUyPFT4mq1c9k0MzvuYWJryQen4+z5zXPciXOj01coI9mQlUPoNzr735fNk9ZBoMPWSXqL4sLqu+SSu8O1Msor0PBJy9gFsfvbwEwD3/45I+ffNgPgQpbLvqiuG9gtMOO2+zsb5AkGi+BMvbPul3jr2jdC29eCgQvm88gL3bf2O9gSjevBAZWT0Hak6+oZAJPWc40LySOpK9X6b6PVsa0zwa6eC94XgMvlTpPrz+iz2+TGAiPa6+XD3QVY0+ejm7PAVoEL+7phe+RY5IPjzsBL69pbm9qWoqvHKWm74ZEQa/pPkgPsNCmb5WaBg+YvqHvdcY3L7Vnoc8FaMLPjXjd75ssmO+Z2DgPKRq9LrmdwA+bYGgvpiwdL50FnW+oVeHPUqT872DWim+","M+5Mvadk9b24AVm+b5BdvUL5D766SPC9qhHOPeu4Lb5bkGS+1OK+vekA1D1J5Mg9PL9Avn15ob6bkLC+EGzJvU+Trb35Wyi9rmHzPQs9g70Qir07mX4UPdF+PT3BHre9pZMRPcr1ar75pCM93HIYPfS0AL5D0ba+OTjhvjQPpLyxxtq+kw0ZPsDQqb0XA3y+rpf5vcKGSb4+i7G9vtxFvm+1j76e8uA9JDkWPpu+TD3wc6++xPjWvtCjCb51lYK+DV01PcrsQr70yBK+lv0ZvE4CXL5VZI89l/NtvrAzcb5OVJG+FXyIvlcEkjxh8MY9/rh5vSYq1719v549vYl8vtdQ7r06NFc8MpdGu0MxSbwghJC+dZr5vQ6LHj3NLTY++DDEPYl44z0Rfry9i0qJOzmPxrxSNhY+LkmovShZ3T3PBZu9qd25PeeZD75bo6O8NbuFPoAYRj6Kvqi9aJKfPngYDD2dgIE92O0jPGcO5j2wCNy9Nhn5PpqBB74WjZG9Ayx1PobVEj1bT5a9EUlPPpzjk72AOSC+x/hhPZMVyjyMHtS9hrkJvpjpYr3ux8g6BKFiPfhOUj5yIKq9lEUdPhI3/r2py1y+mVD0vW59Xb2hu08+AEIIPiV6bz10aeA9d8YBvzaiC75pA+G9vKFrvAc1Oz4e1Ga9C4trvYfpWD4jOTE9TkVTvrLn2zyD1oK+hdxvPckMwD0qU4c9TgrOPvyuNT7rSgW/VJgqPsS9Sj5XVWa9U+jtvOcwVL5cH969ltzmu5bUCb7FVnc+4Wq9viyrND3MqzK+tfGTPiGon73IOJ29CuIWvvT82r1P0tC9quU7vp4Pjjq7+8s+WGHGPdGuqDqYfMa9/tIfPc5VmzzGEcI9GadbvrzVm70eE9Q9xIxavZTDlj2P4D692M0cvozInb3heZG9oduOvpPMDj6EnCM+NaZBPjkZpr5FmDw+RwABPb9AiL6VVJO+hQXNvThKHT4uxHi95woVvgWFvL6yTZS+p8Yfv1Jpkb56X2G+","Z7Y9Pk/1Jz48n+s9KRKBPpXdOT5WyVo9v0iZPnVELz4OjFW+FM3IPdSsET4QZjQ+fPRPPnwsqj32IkU844jPvNTMID5VLhE+lxVQPv36rj6uKpE+I6sCvmn6pjyQDsI+nkTtPbRa6L1yVG4+PTMYPgbhOT6cEGI+HOwiP38YH71uDlC+L5fdPrpHbb1zl2k+KPPKPcYQeTzb1Rs+iuNMvJx9DryxZri9u3RFPonLlb2NJn48vic3vmWDpz23AQ0/eUeTPWBj8z139ai8ofXuuwdeJT6sQSy9QmiOvFcjOz4GFf09eVM0PnEl1bxvnjm9MkLePgDvJj47rA89HqedPsxaPD6UHVS86AFiPthIxz0KM3u+Lvl4Pj5E/zzZHBk94fhQPguimT6vfsk96JWgvUhGAj7dgNQ9JBACveT6bT6IkJe+uJycvQfDiT7Mp+S9Mji1va1IPT76N949JWmIPIiCMj5ukSk9vCgRvs4p3D0tS5U9hvA9vtkOMr06s1I9G5mhPXpLgz7fMdU9sithPqEcjj4fSAy+aDI8PMkLybum8h6+4JCePQWWQD3xvoI+jP5nPl5oPT501589MtShvcROMz1Kg/I8yu06PilN7L0+ZqA9jqD/O2i1C70lnyu8diZFPzdASz4mRLM9j+LgvHRiCj7Th3O8zgWVvL+K1r3TD68+QKqmPaylG73TQ7U8BFhbvvlRMb4Qgh48jrM8vUohvDzP9vo9rB5lvaSjbzz0ssK9eR1pPPmgvb3FFIe+V6k5PUByOT3oDxK+x9wNvjMtnD0GuY094zJjvCc9XT4TWAy9D4rZPDhog70N9ok9ogBNvRZ26j1/5RY/61qlPH7MHDyQiuo8wA5ZPY58aD0Pnbg+8rw5vp5uhr0aD5c9uhuTPdbPQz3HJfo9zTfzvCWphr279Zi90Melvd9Rkj48gcY9Z9zYPEiaNj1/PIS90267vZlrFb2ysXW8103tvVcFaz5zczY+t2cQPdVoNr1g5Qq+BnP9POrEbDyvSR27","FVGIvVD0bzuh5oG9qeY0Przq/j3KF5s+dUZDPsYorz4K2Vs+VbrRPt0cBz3HXyA+1Y7mPZ6FXD5BDYg+uugNvUMJWT5Zyxg96OObPZtoPr6T9qk9Y4PyPUNA8L0wrpk9BvJpPi8EAz5lkSa+4GElPluibj5ObD4+4JlXvfLNTL1Cn3s+mNDKvbrSsj3PiXY+x1WWPqR4mTwcxWy9aur6PUbFQ71PDjI/lOsJvmvVIr4oPqc/pcx4PjBQXz5tqwI8RzFTPERfmL1g7vK9F1MTPqTakz1OJTG9GH1mPqOfMr5kw4E+zW/lvXIKtLwqaJM+jzm9Peoq6T0Ktr092qKKPpUJBr62laK+wtVOvgaqxb3JGhO+ioH8vIIdCb6gWFC+B8I4PSMoC743vDe8n5wmvThk2r3pTSu8ph2Kvogjbb4SfGM9+jmjvQ2v9z11HSC95mK0vR/pazyUtQ2+K/B9vve9iD6KHwO+acq8vvPhVLybu4O+YnCfvjnNHb5ValO+ho0DvEoR2700bY8+ru+jvt8xzT06h/69gi2MvisShr7ckqC8QXCNvJCSDr4v+428O4Jiviad4D1K0yS+rdENvoy4I76TVZC+0BggvcCVFr41wmO9A5LmvoBqu7xDDlY9u3IAv8uaV76R2Xa9TXxYvvQL+DjZeie+zJUivhzl4jyV/Z09iAUpPe6GSbwTt5c8hlQTvX9Qrb2R4527SXpHPVDrAr4gmxc+0F1XvrW8Er4ifxS+7aIpvThHjb2frlQ7rBb0PRGFm73sz3S+s3pdvUBwvL3y/IG7BrHIO2Dpj7zS8ay86wMUvds88jzcibK9Rr/evGc0OLyuRbk91mDNvf+4TT3h76M97okNvgXSl7yL2Nk7FAx3Pbx4Fb5SS4i8BZdsvbB+jj0TVDg+urYevn52ojzllAu9SSntvUQpeT2BbBG+K7QhPTRhcL5qFXE9P9z3PRNTZDzhPAa9TJkcvS3EIT3nld+8so14vBGg0b2nwL27D0clvYkSBr4sGt48","zpuyvRnLwT0IzS4+y8kXvgrVqz1/m6G9BYANPl7fJz0xScW66bFDvIpqCr7KtNM8+xv+PQ16wjx5RQs9H2/fvE1VIT6dbSq9LEJxPRsjKL3UsXm9qc4DPkIJJj2q8TM9NwSgPOBwOb3muvG9HaCLvHyY6rvdmoU9W2bLvCuUOT6SY5c6fRQ0OxYz8byrbwq+PfcdvbX8h705Iri8f5PivWk4NT5tDmS9o5NQPSbjlju7dNW8MWIivkEIYr4ukHS85VAlvkLMLDsXyS89h+/XvWEFg7114pA9p9StPZCL4LwibYO9bPEOPvyngz3h9gy+x5QIPgBjsz2FqKU9q3pQvA8rZj0P8wa+v3civnB1Lb23rIC+CSKfvncUWT230tC9rQHSve9poL3Xlgm+3LWVvaOiCL6eRDy+LKENPdo63runXC2+0SznuwSLKr6D10++Fg65vQoLgb6kZxk8S0HaPW3AQj0Bmly8rImivh/kBDsZ4Y29EM5zvQo+Er6KKG69AK8Cvg3MJL4E/Q292IoFvzFitjyLy5q8LC83vh4Fl76qNr67lLeKvUL/2z1mRFK+OUdEvntRYT31oTY+jfggvtcI/71OjDs8g6GIvgxroryRV6a+9d4YPtPDKb0s++C8EbOlvc8VM71ZKqG9GYoAv1F+GD2V5TW7HpntvOLbQ77Joy49PjWevjpQ8b79NWa94oRQviEZuD1CVeM8+/Ywvt0wqD5UnKs+xf79vXQblz1e9p6+WuEXvQLsi71P5v+8R5C3Pv2pVL9rmP+9ACH4vsOKf72xDly8yccUPazdd75wa/k8QXsAPklnQj3V62++IpAfvhUywj3IFkc8dZByvuhHuz4gocs9ukcDPmeagr5dZ5k+PfIQPcaRer1Uxjm+VzIwvAktOr4/4qc9F9dGPoKe1b7/32G9JZ2RvvJ0BL7kDMK75YBqvm8kvL0G6vk6zzGePcybfb6RG+e+BfhdvryEkb7jGcI9QCoXvUdtJL94h44+KmICvZGWBL1XJA0+","FpH3vZBlJL4wCuE97YprvdQiNT7jNX+9zewuPjR7pz7B162/BmZwvkzrSL3ffw8+j8Z+PjksdL7qusm9kxnXvFtvCD/YAZ+9VjeNvUd7Yj5MSki972+EvDBFxL0RB4e+jCFJPkggqr7PVsE9OCFlvcWxujyV0V2/TpJWPmfZOT7kex8+J+fIPeH4gb7W2Cy+QU3NvpQiCT46GhW8N9PNO8uNGL5lTuS8OFBZPpxycT3JBiG+uQ64vpcnm76JbDY9xR2FvH3LoLvxAS+9svKCvFyilb4EhIG9ZXroPQXdnL4pG3u+5G5+vs+RUb6S81M+FSZ1vuP3bb4L4EG93k0TPcLzQb4QTdi8oSCPvXN1rL16qQg8WLkVu71jJb32DNu8+JapPt3c6T1a77u91ZkdOlZ9kj0FZQc9csBlPrlOZ70v4n8+YIpnvcke8D3oovQ7kZgWPDgamD2+XVe9mW0bvfgD0TyGPDE+QoUQPrIKjj1qAbG9oPsbPPchT72fqXq9hWs1Ppl0Hz6r6Va+Pig/PPh64z0K4Bc+M/1HPdiHAz1OwWc+pNqBviyEKD6VQui8UuDYPUaPPD3hO5W92fSNPZ17L75ILWu++6oDvkwv2z2C9mi8dsEUvUwG5zyPw4a+6q8gvmdFG70cq4I9lywIPd/aRT6/dAw+V1hEvGIvgrthgVE+UfqlPUDgST0Q6ks9rQP+vQW6Fz72iEm9soWlvJIoMD5pj9C+O1+KPXSDlT1bmeY+XoE/PDS2hL6HDtI9D6ATPpQDML21/wc+3PnMvkb7Hb34rTc8JTiIPbrjjD1fkx89wW2bPUQTLD6neIa+6bGNvlDUej1AhTA+J212PkQJBj5b0pi+pRoUP4/5yL2AGe49D/OAPUdMAD4ndOM6IDF2PgO0hj4IyV0+6bngvtJnE79sghm7eCCBPWTKkT2QzQy+0+7SvLsaXL2CLLA+LuL1vQGBQL5Gooy9ufHCPjFULL6U6ey9qKdQPtGOrb2xrfQ8gH3SN8L2Ejw/Gg++","pQFivq459D2CMBw94lvlvbmVHz3MKdg9m6HcPJjJED6nS1E+FWa7PvCC1T734Yk9gPOlPgTBLD2R7Xq+HROKPfYNVT4BfGi9ZX2qvJQe7T1oMaQ9v9W7PPP2sD0o6hg+2+WBvprGDj7U4qg95Bu6PB8TxT0bNbM+F/RwPjZ9kD3Sabc+JDAxPuKZdb3ycdA9Ta2mPhb76T0cbXi8GsKHPi5sRL7d5bQ9u5j3Pc/nbT2abH88+aluPTYtYj0tUJk+c4SBPG7/Fj58WKs79rzbvHlwpD4JPaO9Z2fDPcEf/jxI5H8+LW04PWRYBT0Y5xQ+Gjj5PdcppD5lR+G9OceAPp33773enR0+e0nAvfmzPDzXA5Q+5MzFPZqgc73Ua1u92hsEvowzbbvIIME9+ZYTPYiewz0G9QE9fXurPbTkiz1km5s+nedXPWELir3hur89h2+wPRGR4zxTQ4A+rQ0Eve0uOT5Cyoc9uZvGPdo1JT7xCaQ9iLfnvOzngz1vbaO9zegrvU499z1WFR09XVAMPO8ej734q5E8/ax1PuIcAz7Ef6w8825nPOaVlzwLI3S9MBDGPDs5Xj4FDUy7QTHRPV+oQT2CZ406peo4PRlIbr0CiGy+KtesvUPDe71gXim9tgdjOvfUlTofwI08FUpdPQFhPz3V/lW9m1AsPcKVsT2GShS++TCeva7HTD4uELu8ODD4PFPfG7xpyOE9EcIkPfVgAb6wg2M9pldePhlLPD7txsu8LY2BPRSFkj2SVjC+0DkQvcq1eL0l6fS7ePr9vF9llD0KAM+8WxsTPhUCIz1VLb69hbqIPTcQjj3+VZU8jl4xvc+RVD2q85g8MESyPWBWyb1wmDQ+i54dPm9MIr2UvqW9mVETvjPhHL6SwKc91Sn9vDWjlD1SNqg8U4novEVOBj6fYzw9XYJcvPVdYr3o0CA9GhpPvpkpzLzfBte8NPQXvehbTzwpuoE9xEKsPdDtnr2sOvm9BIWVvc3Kwj3eKCQ+MXCNPUZCsz3TfrA7","rFgVvShFOz6MAV4+IrwqvTXJDz5d2WI+amsQPsat2T4u/pQ8BEylPYHmGj7UWz4+9DYBPsfheT4iblc+VoXIvbguCT5AdeI9L6pcPngnHD68988+4UhVPqeQUb5xRW0+xjIiPcJFmj3ZDG8+NSccPjEbZz4Ny5g+aF9pPt3Xpb2OnOo+6LZiO7XbLz3kZ0s+66IwPvRHoD6eETu9yszIPvMxuzzjXIs8YzcEPqC3srtp3zi8xpApPZvEtz4Yzm8+DxN3PRHTFT2DfYQ9w/xFPrBEQT9+oLE9tb0dvmzujT0AD0i8RUlfPhtNNb0DUrI9fvxSPrajBT0JqpM9MNBQPfZBwj0GM6899Gc5Pva+ADstl428UW2FPuFt0T1TfdQ9mvPJPishLLyu8H0+2Qa5PRrEkb5FKC89PPwSve2eQD1/PHQ9mlVPPpEKDz6YpQg/Xs8JPi/HgDwjJQ493ni9Pr+xAz9PTsi9gguYPasdDj5zQZk+PM5cPuVBIT5v0xI+WNwuPlTwCz91PSs90ierPnI51T3xQVe8AZ8OPsCSbT2IJy8+9UXPPQuzGD0Doxg+9UDVPYDBCz9+QBE+WpchP5YKxD29HRE+4pRGvu/b075Hm3k+XXQePooOLT4XpgY+OSCnPWF4Pz5lYmC+XGcUvQnDdD7ZMoM+VWYFPm6c2T1vt2Q+AtwZPHFNp72fBVW9WXiLPUejujyeq2o+dFigPQvGOr3c1d879HvXPZ/FUL4qpfI9U4K7vP382L2v3mI+qSiaPuzfjL3N0dU73LG8vPJPuj1JqB89VfEgPqSQkT2aD4w+7IL/PSUKVb3I7VA+NzeavSxZBj10HsK9MhA8vpAA+z1SsQm9t0ZIvT+wZr3XwqS+MD3PPe8ImjzS2mc9YIaGPQbaADz2gxG99mZzPe3Toj3AKmw+TW6XPOvUE79yVgO9yKXkPDkOjTuxb2q+5cGTvlZ37TtR6OQ9/KvBPRfwJT/Fnh++aDLSPFIf9ryaaqI+VS4svhjsnL0VEou8","of5pPi66pD3pQYm8UJuGvMdd1T33cC6++p+yPaWDDr6fzPM9sEbuPMQplT1wB8e9HA1uu68kpTwo6aG97SeKvcxYiT5nG3A8cPN2PdG2WztYSy89FqMJvjHJVj3fK16/fvGCveGVqL182kA+vUqHPaT55zyXjCA94nwKPsZ5771SDmO+GFLtPTjJvT3k8To+sF1gPkiFFT5wDyM9L+YLvjw7Ez4PLqc9wvZwPeRBhr2WoAq9OQ5NPZN/8jxhnT0+glZjvYQoCD46AOY9a9fwu6aoUj7DzoO82ystvUtS3z0gWk0+qO23vSazsD2twoE9HVsov76nYz6Olvu9K9qbvcc3AD5NfSA+ZXN6Pm7Cpz7Z9xs+AZswPslNgj17SMw9J5ywPVJZlj57b7i8azq6Pb3BLL4xRc678bWSvXriLr0+QNo9fTI9PkWeh71QzdI99RM9PS4qVT4EwPo9lz6EPYNAWb2YNRI+/w0NPp2+LT6BFEM+ZZJMu+ge2z2eVCk9Zu9BPhI6Bb5cRp0+sl7QPQeARb1T6Y09cjFvPrwzIT48qYM8oBmoPf+dbDpTY4m+OOT6vStaobzruMY9vo6WvFOS5TsdKnW9ppfkPexlxT1pD0w+QGAQvjXWBbu6jqU9N28tvWvlST41Pso9fGZMPqRSf7xMdE09GmghPtiPaj2DX/i9p/YjPqFtmj02QZi7B2lRPorqYb5WkVg8zONNvkU8RL0Thja++8eivpgvMj30gAO9KiKQvrqn4j14ExC+k8T9vjbQlD6j46G9X0A0Pnji3jzQHta8xsPVvRs2u773eg8+nwRsvkM/LL6Pg4285AqGvaj9Jb4KYVs8/5MAPe8Fvb73eEc+eLkuvTzT8DuHl5e+XXs2vbLiJD7HLLq97YHcPf/xaDzIKbS9/AgzvXJgLT2WiAw+QWvtvbdLID7c6ii+ZZ91PVZVlz7qjC0+jy3MvlVERT3+gTE++mGZvVkqwDxkWYu96Xy4PjuyRj6ftQ++kOZUvnAajz1dO6g9","MoqRvZO8AT3p/AU+vzkvvkk3hj1HuCU8QuUoPZqOAj6WSz8+8VgXu7dJsrzfzEw9dN5NvpsmgT2Q69g9ztPzPcpr2j1fbok+JPz2Pcs/fT6B9T2+k32KvWJg6j2+Rz++8v9YO9wK7j3dZik+9hfrPHE/JrwIyyA+fZhHvNS6grzaLUs+9Fe/vuq+p729sdM7zbv/PUybGj3IVCW9EgsfvqTDEj7WNkS91wgpvs1e5T385Hk+O/QCvGO0QD5vKsE86b4WvsfQD76Sfo4+i/qPu4Ba7D2wdb+921czPiojY7y20D0+G+sbvZoQhL7UizE+XpSBPT0KGj49uUK9lZeOPaZebD01g2g9+qAVvbNDij3fXQG96gDJPdZHDz0MJi++OkDkvWivjr4WV6k9rCwkvvokIj5wY6E8k/g4vmNyPT6ejVg+uKCQPRsOPT0a0HA8/kcyvggzCz7jdwy+e8IrvqXz0bzRf5E9aLfAvVbxP7551+A9vNfFvWwpar2v79E8S2ViviS6+r1Rj5095r2nPBga1zxwxUm87lmWO59VrLz+2+29hsiZPjUv1T3AEei9sxoBvt96Cj0bSpW9bFdivTSMJTwUEaS8dZRbvFz/Db4qUFU821bXvWU3/j29ibY9QXJOPm6gAL26LA+64QvXPWV/O72IpOw8UslyPmzkHD6kSB++6kB6PvqIPjy4JH4+1he+vfJrPr7o4V++x6fgviZl9r2LKOM9x5tKvp0iuLiApc29TuwVvkEoSz5ysUG+PJurvbaLOD5josy+EvuNPXf+rr6kXcG9yZ1zPtY5ob4X30M+lFxNPYVE0z0kD0a8WHd+PlqUaL4R1K+9Wc6QvVqEgL5f0ws+be3IvXBkTb5G8gK+Bk6RvcHjGr0dG2e+Uv52vJuCLb1Z+om9aAk9Pu81Yj7TZWg6R2KAvnl0v76bcaE74i5LPqKZkT70+gG8U1SIvnDIqT1vpEg+5i0Vvtm6Eb1GRpI95XcAPgMmpT2GpgM+z7I6PknJCz6gApa9","C48dPu2Ouj3ANPQ9LZIsPkCgUz6MkXg+plozPn3hCT5UedA9IUISPl3Aiz6uIMq6gc90Pi3Ehj4bgks+ROiAPLHy9bs9NI09ti5PPWiYJT7GMFk+Y8kaPSiBhj2VrSI+SCGOPpv8GD24bm8+u9mAPlg8Oz6h+H8+sfpEPZFnPr6tnfI+KHcEPs+2pz12BMc9haTIvabIKT7LUHY+PwuMPvoDAT7HxUY+QS1sPcyz9D3rRPs94vAqPQnlsD33SRk/Wq0kvd65Jj5zazu8dPHoO40Zyj6Seas98yRdPhgUiT7WU8C83YEuPoKw+D2ZwC88U6hsPoju4T70qJo9Lb1CPfhEXT1mxxU+0+uPvHezAL3iLxy9eOO9PfkaeLyfjY29fMyaPr9YhDzbeYC9bmdZPOHxoD12Uzu+6ZofPvceSrxsIki+x5QIPpWjQz4Q1A09p+w/PYm8cTuiGNO8AGEIPUib3j2tNzA9Q3C9vZAXrD4WWWy74UaAPI+3ejz+lNM9ClwevhULHz6bxzM+6NAcPvSeoT0roo+8DFKgPDtDsD2cwbk9eMnsO5i2ST2KoyA9PwBhPbJYED40kXa+cjDPvs3jyTwos5y70mCDPTK4yrwkEJk9reyQPcTnnT26eJ49Du3cPctvGT0Iido8lNbrPZKjcT1KPlM92v6RPVl2oD2Phe48bqSgPYLKwrkY1pm79ccevHWSBbp1drA9/eIdvpSddj1Bb809eJDDvLgRyD1whKm9X8kLPpy4nz0f0To8I5DNvsz4PL2DIv+7WIHcvG7Br70E/IA8fznwPMFfe7yJuLU7QwKevORHwrqYHUI8mXMlvZN/ebxEYu09a16zPZVzTz7JuRg99jRRPUrAZ7tiRJ8+kjI5vW8C+L3K2wO8MR14vXQ2K70uteK8u/XWPC/ncD01mgQ+5bSBvS3kmr13BAM+57/DPaba37yy7bY58vziPJu2ST1Tk4G9brkRPlRPZz1YKQy6AyRPvZXSu7z7Lse9Tijgu/1YAzyBKaK8","Cu8nvidSKD3NX6A9kECaPcMZUz7bEYs9XbmePQNJQj7SCx8+fSQRvcSvRz6L5sA9BAUoPh8AOz7jSyg+eaXTvS7NmD1GMAS6M8dMPli2yj16Afg9+Od8Pjt58r0AKpk9h5YAvXOJJjxbt768VUzmPSF+aj61zkA+THy+PWrdur1Vl6A+DWYbPn0OVz2bEzi9PPAXPk17PT6w49499NeDPPfgHD6DkrU8bS0sPfSBHD0Jp7c94jE3Pdx0gz53C009qcVrPgIpjj0FB3s9s96qPSyeoD5IKVm7P+u4PHqiST70tqM9TzyyPRdOjDuzXrs+gCkhPfrScDzgE/w9sxIBPqsbp72JHso9dr5AP9+mjz6e09g9I5mvPgNUl72H9IY+wJkDP/dg8T3Um9S+oUaBPkzFDT07L6O9R4aivg6GAT6CJ1s868EyvNWCSj6KWdq9wptNPlHrQD5+1xY++QHYPYBpyD2UbB6+4y9BPqt5bj0OE2E9foGgPkysWj56nlI+XAOuPn7k1r3RIAi+L5b2PTcfwj0vbI4+hjgyPn8zvD5Xpvu9uAwZvgjdOz1qglc9wE4JP6NXuTtS4ck+bNKcvQBcKD9OynY+TsjXPTLHq7yJH10+uqPtPDTjdb61skM+2NqHPodzeD6i6yQ9r227PNS3jTvNrn8+Kx+ZPpNtbz5vhjc+Cn3CPbJJH768YUw8mxnBvETwaTxh+zu9LniEPhpUkj2Uw0y6LWJeu4XU1z5eMr09ILfhPbcrAD13T7S8VtyiPrsSuD11lCA95LqHPlkTZTxeOQW9EkFBvn3JKrwaOy49ddKJvT/nCD7BJxg98eDUvfwhwT4PKJQ99WHCPRcgoj4y9jm70KAXPgmaNj5o5uq8AdPhvRIAWD1EUeC7griwPgIxMj2wc3s8dyv7veC+X750udi9DsEgvl1APD5wgli+05A5Ph2kAT1qAxw+IkSTPeY5e70w59K8niBTPVYdhj4rouc7q563PSH8xz3kjUs+1+sPvioVEj5vIxu9","ZTSKPjMsvj0B7Hm9spUqPRPrJb21crI9VDuevf1Fhj5owoI98eGlPaJsXD1lWXY+riLCPc/Ngrs2lTK+qrMIvZWoqj2a4pQ87E6ku7DLDzvM1Rc9MvzHPT0IQ77olYo+Mv+HPY5Pvb3xr+o9WmvLPRreyDwd1B09+MdjPG7BZLw2Hzu+4XsAPWjjMj4BDIm9UPF4vrAVvL1XbGu92NaBvdowjjptwaw9wLCZPoMEgT0oDq47s9Y1vAompT6FdPy9hv6vvU6GYrtPGp69IXi5PAD0Tb3VLoi94FZOvXG1Qb35NNo94rW2vUzafT0wC6y9epx6vFu5lz2SIhM+P2U3PfuKIL78VDq8bQy3PgUiNz62E/A+v8oAPpPSED4W1Sc++4XyPAyGWzvChpI9DdxLPsOZpT5SgJU7JFg5vhDIx73Aa1A8h6akvTy/+z2pY2I9t7i9PeAHaT7tvFO9i7+sPfWGyr29mgy+5Q8AvMYxYrwB7Y+9UYm7PdGrlbsuDqC9lC8YPkn65z7+302+nfGGvOQb0jxjmYQ98cLQPSc5Fj5NUA6+SIcBvh2GmLz0Cdc+d1FSvgw05L0ddjk+UZrZPtpqcT6mX0O9z9g7PKYpjL0e9ZA+63SWvbilcLwUJ8u9lJkDvUD5mD16K3y9m2OSPlWWM74E7mk+9QhTvQ9Ykj6F6Yq+Pi/HvOjaAL+yOEe9/5K6vV7AYr3KnQy+9tG+vZUTqL7apSy+E0yLvTziKr2ivU+9GPzuvS6DCr6XqMa99s/kOw/Erj2INXC+gYJHvtFrzL7JYgS+gT87vmdul748dg6+dKCNPd4Cmr4Mr4a+eklkvUvF2bxB6aO8ncCfve9UkDxHxni+gohSPd2W/b1I5jC+Z9hyvWgMR74jNU++FcisvuVVIr2Jjbm9F4B6vvlWRL7DFOi99RogvZx1fL3Uid28DCFmvr/Dn7x9uMu9jiQrvhShNb1NxJu9M20EvsLwkbwC7Ru9Lo3IvQrGvLwkkt6+0wbTvlBpIL0uhVa+","uBPnvWDaTL723J0+5N0PvQ9lFb4Quoq9J9NNPRkDnT2uJD0+rXt4PZEHjL0qcHK6Z4GZupNDOr3zeWK8JZcRvcqfyL29VAO+ovHyPGUKm71h9Qw84HdVPZX5NryxBxi+RIk0vsVA8DuR29O8D9snObdXSj0px7i99nZ1vDXETL3dKMa79RnrvKUXp73/UvK963YrvqJD6bygtY+9EGnSvat3M70aiei8ria9vexmC75T8Zo90A07vnYHGr5yRco8ayELPAocBr4LhRO8zs8KPtf9lTzDABm+P3Kdvdi4Xb1wex6/oJ9mPTBn3TxuxZ88Gc/QvA0haT3CnLq9cs2vPZSuNb7mizq85D+CPMiiPb0IFZG94DUWvU+Gu7yriAg7mDTJvXWXh72A9Z08FIGavSajOD118JK9d8OJPZAZBj5Iwwy9SENoPMoCzL0Utwu9ge0FPAZcKz56FAo+cuSJPTvBLb7O7Zs99xqdvLMNEDx+mpw9/5WKPEte0j2NdxA+YcRCPltEjzx4taK85HUSvsQZabyhE7Q9obayPXr2Ez5myRy8gcv6vVu4pruAmtA9wkOFPam5tbtcDky+m448vSnSNjzGata8SAtAPU0Qojz/uxG+hjUuPXUd+zy7ukw8BztZvobSID0OxQ89knu3PdudMD6Coxk9qtYDPQ8DljtORau8dKgSvsh4CL5hXgi++x29vQ/T671R5SW+GmwjvhprIL0ezCg65ID6vAp5AbxguJi9/OEdvijpH72O01A8q3qyvIsHK74oYzG9GgcePb59g76ZlyW++bzRvYHSuL3FJ+u8X2s9vLmrEL2PyOi9MdmIvux/4r06tbm9QCm5vHtiw74Zfzu8Q0yCvcN6i70C5Eu+qpPIvUeAIL4WvB+9Tk3uvfvdgr7xPii8ugMvvs93FL6dOcG8Oe9wvhOWlbyJgt+9d0knvXBKab7zdU+9LqXbvkRyHr0Fgxi9lxamvdVc0D0cZhC+Ez5uvWQEhL1LQO29VWNtvZmEBb5y5aK+","XRHpvXhzvr36K7W85E7rPZkzhL1PJOg9IDotvkjKwr1FjC+7DYnJvH2M1b6A9i0+nW30PRtwiL1Uq969Les6PiLu0TxKzko+jJfCvp9tE7yvzTi+a2fdvSMvtDwbdc48qpoqvmVUez3yl2M9J/FCvZ+TGz60Ec85fQdEvXezEb6Q8hG9giqtPCuCcD09/I89xEUMvmkcSbzEH3u8jsaavdv8wr1WubO9w4tvvpiUlzwJoGy+t3UHv+xhcj4vpxO/Qk1tvUBUiz6QaEE+muthPs9lU74MQuo+0ZIVvjidMr7bewA/o08VPkmF8T2fgka9e2IyvURLjjzdg8c90Ab1PuLUGDyeEM680ro0PL79hbw60fy9K6czPdI8b74JY229+ldPvq+fej2gYHG+9Y5ovk81z75uRjE8NHyjPUZqTL7IKIu+OHYoPn+W1T0Mp1u9EQCmvVzFGb45icm9PnUmvZQWj77OQQ++EEFIPaU9DD7gWlW+mfE2uyowdz0p/0m97nAEvwu2rr3CAhs8jvEovteuO75RAUq+jrl/vWZ/D75d+po91aLYvdicyLuExAc+JuMSvmbUDL7iuAi+RwFovSH2xL1ulIy9S9WzPEgZ0T3zrnW+V19ZvkG2Sb5W8uk9/9HjvEOmxDxzk0U+rWRQvVilLz39SHw94XwcvQIpIL4j5ZG9UTQFvsnzVb7Y6jE9LbFovSBzmj0//4+9j96IvcOUJz6cw5w916s8vHY7sbtTGQU9QzGvvNqCGj6Xcw89oiSEvqbDQT7i2Vi+FFdcPtn58r0AIh4+cIQOvS8wBD8R3KE9AUoSPBHOKT4EeAA+pSTovVReED5ug+W78t+nvaPHYj5ZebC92BsrPoAE7r0mcxq+qPH2PRBR7T1nSEw+5xGFvYYfvLx4eRG9WnBSPmTrIz6mMIW+xtkRvt/bzT0NfFc+mNKPvfgZ0j34toS9nWYYvjFA271g0da9jK/9PGRqvbyj2X8+oEjxvYZVDb5Qbo68nGK1PJ29Bb1YTB4+","2XdMvlhNi7zCCI++Sr2WO8tbJD7i0S0+SSeIvk8//b0Txig+Gh1yvLBXbL04DNO8r/jLvWF2YD1cMLO9zpOTvuOqkT5FR9K9VsUhvilOFz50ASa+pQ8bvIVgF76LfZ2+S026vu0Oib7cwYC+oECRvj+yCr057S+9KnwwPmXAGj0OFwc+rDGgPVF2qb53eV8+Z/eOvTZIm76dYd6+l1s1PqdNGb6SdZ29J9cPPhAqWL4a4L09GlOJPYkOab74zpi9IminvSo1bb3EGkW+VpujPcnUgr5FQ0K8triSvpQlYr7uiWo+hGgDPjGYk75Qyj49FX3mvZ7l9LxaDmO+OoNDPjVax72rDSK+Mxdgvkr3T75LJFi+a8wAvj/Aor1gqBG9f1BivXk39r17PQa+c3ljPGwUBz6A6xS+r3qTvqHDGb7N3dy7l7dCvsgmZb70sva+e41CvqWfhr0QDwK+z6VMvqCj2r3FEqA6y7lXvncZR74BpiC+ltzPvTv/F71jatu8lKwhvlPjx77ZbSc+2/ahveLb9b14S6I9JXeKvSRqr73BGMm+0v6LPZ0Ehrwp/Na9MSGqvT7DUL5WiZ++KJmLvo2yVr6wRfe97vWEvu0n3D0fnpm+pUBCvWM+/r03V6++/aX0vbcaAr7gXsG+IBAnvvhXbb7vAMm+hAS4vl+yZL4n0K++mygnPb6K2T2+n3m7kFepPeLKur3YCtM8vqRFPOBxlr0ZrLm89i5HveZYkz6WjOQ80tEtvWncTL0437S+L0S1vOTcLT1xBaW9ZwCbPfmvHryNDIk8b2Oovacm4L35d0y+jyu8vlmkBz2HTB++lTOBvFTzpTuYp5w+mPcIvTyeO77PWFg9QpTfPbgiLL7PNxe+sUDavGFloL3Yl/K8jcguvMoqlb2wXtg5NFbCPMFZoD0uChC+HUeqPHj7mj7fdXE8HguCvFuATr7j2n0+5jVHPqChz72UV2e9hR+7PHZX8r5lgg0+XjkYvcUxTL1ihnS9rhivvIagqbxUdFs9","01ATvXfGwTyf/j88mrcNPGb/B745g4s9B8QOvhNQrj2lMAI9rmztvADgKD1ioQG+vcDkO0V9Br6datK9FCZTPl4P7b0Wze47F1afvNv1pD0F8ni8dZ//PGOyq72dqaY+cJ6avY8qAz/cRn69OL3JPQHiYLsCP9k9zcvaO6XqPDyDpL49DOvpPXUbCj5NywE7HlwUvqgrS72F6Ns8UOEaPhcV+T2asRG+oP33PJCrmT0CMAW+UZSgvS79RTwbfek9N+kzvmIl2LzOLio9hcWmvKG0MT7YqwQ9R60wu0lIb70bayG+FrOSPA4r+T0oOow9ogwcPgwIe718dSm+gSpYPXd56TwE4zK+PCQxvanz6rxRfHm+Rff4vXzoH75vegG+NIW4vcOJNL4et528GrAgvl9Xtz1QKna8La1Tvl8vHD0Wv6s8SuxbvhHcA76pYl28rNcrPc1FVDx7JRq96CICvDZAPL0y0A6+j7ogvlcBTb5u34m+14Blvrrih70YO6i9goekvucmFj2AASG+A4VIPM/jSr7Q7Yq7+ELWveskA768M6S9e8N9PcC5hb7/bzS9LpLCvrd2sb1ZSXi9177YvaCyPL7vtu49DfiUvYhcqr1fJ4u+CGomvsduDr6yNo48ImZAvgofpb2zZoW+IxoEvoIG2r0pPD69RYStvrVFe715B0a85VBcPfpS+byjAbi9SQFMvp3YaTxIoAA8kjrHPSPo9L0OYry9vLwFPc65+b1GVIG98lrhvfT+nT1bqMq9/CVKvs3Qcj7QCv+9cz0hPpIAuL17IsS9YkxQvmS9yL5xUpA9RXIovh9MM76Oro49NfZXvi8B0r3tw3C97V2TveUsw7wfRdQ90pNhvZJ3qr0R/BW++IEVvkTfRz6j+hC+uek+PrVuyb1YHnS+yGn5vcowgLxMics8Nx4+vop4Cr6pmsA9+3B5vfoF/z3jkI28ak+Gvm/VLr5f0QU+O8cqvomgCz5RA0u+oA61PU2+iLqGj+K9lZ/Dvv/i9zsnhbS9","WTfoPXBH8TysKLo9VzhxPUqhlz3zG5E8zErovU6LLz0CIG+8sR7evA4osj0Y70888SmKOwaKVr3i6US+ea66ve8xKrzrVTA+QgPPPcFXLz7TYfA9YWYgPQa2Az5l+DK92vLKvU0Y/bzprpI9qDYBveoeED0PSYQ9SNDNvaG2P75tE+E9uXLsPW79qzxt0gS9/fMAPBiQ5jusMym+UAQBPF6ql72HOaS9nVLtPFJtubvR8pk9m1L9vCW0Or26mgm98z+zvWtdJTxY4OC8ivGCveizCT5dtO84+OUpPtExg71f9VO9tKO5PCYsKb6l2mO9+/PCvKW0jT2zs8O91uI+vEV1ez3K2QS72N16Pj/i1D0Hvkk9QnpvvXa/ATvGQUy84jODPXBEBb59ank+idakvUAdwD2i1Tq8l3JhvguUZz0wbw4+ui5Rvnb5yTy5L+k9rwCIvskO4Dy91pA89Jc9Pbef4DxeN427yoY9viiXkL4o75Q92d0+Pfirir1bKhQ+FM8Pvo/guL3FFnQ8n0k+vY3pFr5K1A8++CPoPSLRJz7hVa+9SF8BPNzUorxGg6S9TJCGvVMjaLzW0Ee9/PZsvV/n1r2LT+s8kqR/PeN0Y71KLbE5rygDvs3eQj5UJp09/ZOPvRGMGT3RN9s9pnx7vBWr0rsvPca9iZYsvuF2sb3hiTw9DAGQPCoza76GgxI+IsVbvuztsL7X0vy9l8pwvosKxLzTmdA8pPVEvrjWOL4sGA67Z1OzvraAkro1d+s9K8BzvlOMRz4efoa+JxiQPTnPlD1xgj++mTmRPFs8HL7T5+E9OY7EPRKXIr6mn608jBFTPYtLe758v0s9ttlJvjKt1r5nrf87x/YJvbfhxL6ULYW+lpsFvpJMDD3ZbJO+jsKqveH93b3FgpA80BJZvTbrOT0BZO69WrBRvp43EL5KNb098bkkPRPhYz5j1Ig9n4+nvtyBHD6VHkI+QIL6vR2yX74Ck329TNRiPhrjET7QRDi8xrFfPH3RRz2avQO+","muhIPppprT69pYA+2UxtPlJXSD0oMtA9F0NlPRQpSrx5NoU+yNVYPsnf0D3BFku9JLyovihHpj37lTU+cM8WPmQbBb4ul6Y9XJxvPtydbT6RuYK9XvaRPclq5D1nqaQ+a2qMPgL+/DyBbpE+bj2rPpOYhj6Fk7W8t/rUPRpWrDwhTE0+lPHyPuZf/r21Sjk+4Ue/PatDHT1/R18+G/G/PVT/2D3Iduw9l3kyPe6S2j3wapw9TssjPj0VSz6boGs+F2YnPkRZPz6Om/U8cN2gvFcrKj7P5lU+o/qpPUc+FD2759A+9K2DPubBkDw9CAW+K3mtPt2haj6WDkk+yR/yPhEPnj6SzCU+mEJGPZob5z0ekgm9HHClPdYsFT44bsM8vRx+PQLq/jt1Wyy9Vc8bvm/1hLw2V5E9ZVyIPT6ADj45bay9Ol1UPp3I8z0/Lou9VczwPfKhaD207oe8g7Y5vW3hGT4wzYk9n4PzvHQLhz2GDmM9oCmCPviILL6BZpS8m0uDvurMOD5Po5Y5fBepPFLUYz4aXrG8N6eOvYb1S72pcco9ywWzO3mScz48OOW7KJeVPACHtD0vaam9KbWCvp134j30Vqa93zKyPbQ4gL3dF5S+V0gDPhuwnz1rvj89YGn8Pt6PgLrURTE+1CItvWlfQD4Sscs8iFxwPiGswbvE4ic+RguhvcslfD2/53+9oe/iPG0zqr0s8ze9AzdnvbCGqz2ycM892zpkvPO5Uj0TBSo9bdPWPU05zDtz9I680V2APNzMNb0bTZM9PwhgvEIDiDziWai9CZ7pPAsmAj5sYD+92T7iPI8rwD3EXjQ9J7saPcUIYb2UtzI9bI2dvYGToT2b1xm+Wt8IvmU+RD124as9V/6Vu6WA7Ly0wgm+Bp5NvkRI07z+vCM9qqkBPQSih73fEoe9f9wkPpxnnD1ZIfk8jweDvT/BqbyiWkO95Vc/vKN5QD3M3+e9SIqvPcu0sz0XT7o9R4qLvelk6b1WolS+BDWZvVc8Oj6xuw68","iXO/PL0A2z1jgAw+Fq4HPgnvwD4IlCw+Hc0MPptKmT7DgS+7UIaCPhMos70rH/U9hSo3Ps/tWD4qDty9YfKJPRFrgT7NI5U+1802vZJekz0t/6w8iM9APqG4frx9t8S9XoOvPRV8iT2kgII+leF1vV7JrT7q9JQ9KuJCPpAbYz1D+cs+ti7hPRobmT2qx9w+n587PnMa8T00lCQ9s5wlPh0Rrz1e44U+64gVPZcrQD2OluY+1aeYvWUzpj3HJd88umc8PlhyLL0wk24+r9mAPXPcrT6WE0M9QubgPeOqCL2u1FU+2EUvPrBCsT1Dj3E+jMc4PZibkbwJpD4+ZmRmPng15z4HYLe+K/ebPdicNz6KlfA9Y50jPnwbLj6qWCU+4JArPmdbUD4kIzC+i8OBvsAbvb2DjUi8302bPlifHj400Wa+5d7yvUyA6D7LyX++onlzPvS4p755TF++ICgdPvfYAL0ePE08gAQ5vtTiCb/2FeM8bleRPm0Fyr4f5Fa+aeE3vt5Aib0+i5k+9dCyvZy6gTwvo8U+WjKNPYngHr1MceO9IVT/PYm9BD3Lw1y+BAV4PKERGL4e9jU9yfQjP44pTb+6/RY8mGfTvOuVdz5vOYG+oEtxvdtGp7uxgOO8ANqivp74m7xLs/a9yXVqvj5Bq74CEly9XB1WPMRc9L1+p7Q9T9Vyvvvdeb5oTL++kuOAPpzOB7zpZVA7NvF1vibk5r5nRHU9cUhevp0sKr7GxrC92c/UvAr9vjxI328+Nfr4vLdPjrxNohe7jSu9va1poL0Zjps+GOq1Pn9ZbbxXM/C9EhB4vlfJzr2eto6+3f6TveDdJL1Aa5A+KQaJvrL2z738uUO+e3p+vsOaNb5ftUE8OPjIvl77n7z0/5Q9edD7vcMxGL6/tAe79hknvrN9ED48qKM8ogKCvo/iRj6eApY+SDbpvf6jhT1j1eq+6XUDPtqA4zxwxiO+M7+VvkDI3r2Cdky+A1X9vBE/rjttY5u9oHkzPqH0i722DG0+","M+82vvI6ZLxmTKM9qBKjvRoAO74WT3u9MUufPQ/WGT7lhrI6iUKFPZ+ygL1ueRa9qPuYvUJsTz14dxs98V+Gvrbmcj6mWBC+An/FPRFkYTt6Mf09vADXPROPhT0f4gQ+M3wUvbqZJj5DHVk6ImCzve3TCLrucqa9GwLPvVBkjD1iSZG+lzdTvr5blbvgAoc8oiyKPTwujL4hmZE91xUEvi4DRj5hh729Hnz4PN/STT4jgNi+LU9uvQHcqj58xkK9iXoBPr+sLbxqfsg9HI5wPrmebb1PzWa9ibxOvHRvH703Sti9DTmcvs1dOz2wI+Q8HfYgPpw0dr7Qsau700BqPrQHbT2lTRg9vmMHvhWqgjysREO+s/4FPKt62D3Wtig+TJ8TvlygMj4WOo09SwH5vYEAmb4vdN+9hVdfvmofij4NqEi+Gt8pPS20hTyZwL09H4pfPCgoi73NXei9OtypPh39Pb6JCv49Mb42PYG03L2A1C2+fsWTPYnXA764xZ49Ej7wvZH3LT4cekO+kr9RPtICnjs6S3E+onDAPo919T0hSs693TJKPbu097yrnI+9oo+PvRGm7z1Yapo+/mOrvonaZDvVwYo95wmBPbTMhL4up708qUh+vZrqoL5jK8W9dYo0vSnHuj6YOt28A1LCvlTPWz7lAKa+qmo+Pq9lSD38+YO+CtyHPTcAn7yr8wO9dX4cvopMCb0CAim+ZGcrPaujBr8kWZ8+3xsCvmLQaj5KSC4+sP6Rvu3DdD5Gvjc9ia6zva8AVj59Ihq9h9CVvobzXb49i/o9P+iyvTKvR70/khm+2I+dPmLk5L0AAiq+KngQvi15MD7+57S+ANeGvRt5fb6r2Ek987FVvYvar740oFq9iVhEvjT+5T01+fm5PIc3vr24lb1UIYi+QrgovjGh0j4CHbS+HRfMPMTZLL95KWI+FakwPmF8lD4dZ4a8CXZSvZHuGT6GA64+1771vXgCvj66wW08rE/9PlqU6T0+1jw8fKejvbC3Lz5Xghw9","dvlmui8ivz2zQKY9xN76u6gj/T08wwk8NwA9vYY3CL96Koe+PUgWvCG1uz1RAAY+PBiHvJNulj65nxU+oEGBvawFrr5Nbhq9PyzevXLSpT4nbIw9a8DtvZSghL1qkze+fKHJvGxMATzMvEW9EGaqPuhBuD0zvhO+PJTRu8HL6T6uG9a+6OfaPjJWKLxYTwA+7wHIPfXhcj6xjCK+X/NjvvrRJj7uHAo+9N9FvVhhA70How8+d9F2Pny/ur34Oz4/9dqwPCbncj47Fq88b6CxPEnrlz7wbAg+e+ojPhPBvj25g+i+BUgCPkstkr3G5ia+yX6XPX50cj5w0Oi9BMmBvWEbNz38s/69+pv5PcvChj3Fy0g8wxU8PTU9Nz0cAHs+n7uKvlkDGL4NKr8990l6Pvr8MT4ueAe+VX87vZGYFL7c/rK+eBuJOiBwDj7/9Vc9nhN0vfQxXb42vdG84GYPPy3uxb1opEw9mnYivqtyjT67sQE+9BcDvVc0w76sIqO9//OKPTOAfj6gEK07qFpEvswwa77XmIC8UvWdPDkLK75vsym+bepyPbuLX75kNm0+yLZQPil2nT1xn+M+uX9wvs8Rwz3GjGQ+03wFPoKhpb2gzCI9IeTLvHNa3D1DgBE+tmcOvRycoL3j+fq9efypPKtuFr4g5EI7cngovm2dkb1+t9O8TwLTPeGnmD6mlpc870I7PufITL7hl8++T9wyvu9kDr7N3+C9OVtFvteO8L36dII+s8xWvh7c8T2epks9yvaqvRWqnz2qoRO7GDycPuqerz24YXa91ZEPPS09hb7iX749olNbvtUmUz7AOLa9HrJlvbxIEL0MDP0+OjydvZsuTjy5eb0+zuN+PEkYz73bxjW9JV+PvfS0TTt90fQ9p5B2vY1gc76Yflg+4kMwvg07fj5CHha+qLskvoQSAL8+vae9FpVavbYX9j70K3G+UdW5PYC8gD7WriQ7hmUcvo/qBr6gCSI+uyudPsu4qT5eR4o90sdCPiLIqrqgsTi9","UNuzPb9onz4z11k+VSPiPWE94D2YmoY8nEe9PYNaNL4Wo989SjGrPU6quL3Z/D6+84pxvqcQSz15NB++SpGPPBgZ6z0IVI881OqQPnEO/D5a2h8+qA/VPWi+kj470p097XsCvvPoyT0qwi8+9G2LPoJK6T0en6o+OAimvZ4TgL476eQ9YLxxPnbt4ruiJgU+aCftvAcc2T3fxfo8VIaWPjwGpj7Gxb89qs4CvqVLkT6GOaS7anvFPmO3XjtpTRU+nNOOPQRD7bqRFtm8kLa3PJfcCj+TdwI+U4RJvdguPD1Wlyi+k1o5PsRe7b3b9EA+myxGPksvnT5G9gi96feyPlO5Jz5n2Nm82dG0PE0cAL5MPu29r4BTPqqSPTv/NDG+7nnIPLI0mTzryuS8So/1vYWQLDuV2Dc9pOUjvEWysT4qeAu+avREPcWiHDxhaN698acXPS79mL1FSbG9pFURvvThhD3pUxG+8ZchPUwiuD3+yHM9oxmmPOHdcb4nOtK9LVKkvjRx2j1JTbU9MsOaPFiScT4az6W9yTVPPVEfHb38npW9LK3tvLxG5j26tDI+oRuRvXX/0T0XfRc+pnlavbMZtD0Zx+O8xmmsvMgQrr2kQ72+9CBXvHXwjTzu8iW9EZcDP5/qqT1mDoa8KuOJPUf4Gz1/o4w+is3ePGtONr6X5X0+9+povXgLIj271hm+rlkRvuVEr71sTwS+noKYvp9IFLyTWw894OhzPaXtzLwSUTc9SZQpPYpEIj3HcHi+6Ne4vabC+DyXf2m89+vwPZW5MT0VRCa8RVYpvnkiiT4L0gM95WQSvDjEQj1oKEa8BFuzvSBPdb7YzHE9WwSFvQA1Dz7BRCu88iBwPQVEjr2UIx8+L+0CPnnxzr0S8Oq9FQ7bvQNTBj5kHaC89b2nvCBPBT0JrHU+6CUjvrKykjvncGg9Y5iAvayHTbyUqwK8nB3WvcKKbD1xuhs9k32IPbJq2rv7pfg9y+mfPYjyzrz/ZOa9v3+XPcuajrzGI9y9","fTA6vuPi9T18l3E+zFCdPbhXDT62Fmc+u+I6PgEzyTwrHSC8I48lPr9/iz3uRAc+lp5rPV5kOD7iYXc+Oh1BvfeEyrw7iI49CEQFPKMDfD3QNxI+Cn/tPW1EOL2cUjO+KnRlPoB6DL6bFSC9ODm+Pet2mT49FAw+CmpCPgu/frrZuGM+s8nePBdInzy2lgG7zEWIPqLLOj5XMOY9PCZXPYjuUj6whXU+RiwJvQL8qT6PJmk+Io1HvXm7DD6pOlM+AEqSvXZrq7vmLXg+V32cPZ2ztj6sAVQ93Nx2PUeBk73CkuM9s2SNPqDTyT3RSuQ+xEcIPdP+Ej7bUmc+lVZmPj5Kir5GfTM+GRzdve+dU76CvAQ+Yb+UPhfbqz5UMxY/atWAPk8qrD0H0sg8PSyxPiMxuj4pI/29bPcCP/eucj1LlLq8iQcFvUg1aT5Vmjw+pxqSPlQsGD6iR8m9ZvJQvpknyj34bWi9XeeOvozTc70xBxo9Yty5PH7xt73azSQ9EGnzvocIerwgnUU+/WolPbTJaj01jNu88O2cPp54Az4ldSe8wpx6PUPZ7L3VWRo+eX4FP1QIlTz9b/i+va6ivgZeb70UlLQ9kWDWPZpyKbw0oh682X4AvpwvhDx6Mc49hcPsvnzUcD0hujs+HreUPkaJRT+bwFO+SzMyPqyWnjr4+Rs96238PbLfnD6dZsE9bC4DvmjIzruPi0G90KAGv44iq761oj28b6EnPapcrr6l+bi+qjm9PscUiz1dAIU91guaPLvPh7x4T6y8MasSPpyDJbzfhSW+XXZWvGsYLb4gtie9/yywPUnikz1CDiK9JwOYPa/JtL3aQAs+/Au5PryYmr1IU0s+AsgHPZ/m9DxoRqs+Xp5GvgEUqbvMUOY7CsEBvmRWeb5RqyK++OJcPCBKoDzL6qa8IMf5PR35CT8SDYU+QTQKPrcTUL2O2FE++oLrPZIu0z2vvMk8h7gQvryWEr6oFyo+P9/XPac67D3EAp++FIbFPjeYBL3iHTi9","zjZKPjRa4z2xC4k9wSsSvU44kb5NssM+sciruqWHRz7/32u+bLwivTynIr4sLgY+T8eCPI0KnL5Xs4a9ini3vfHUUL31C+W92M26PU+ZTr3/opA8e4iRvmeIjD0GE9m96t4WvgGaOz3hE569z6vCvOJWmDwVC1++T+xBvniXET2doX29I3wEvmVklD0sicE9hS5dvrgAjL2HhnG9XoKavkxw473OBcU+69cIvk4N97x9O2w9pYgHPsiStz434oq9rh44PjJ5jj4ttGg8MRCRPRL/Ar2uTii9CYS2PZXH6L2g3xC/Y2OVvZg++rx+9hQ9nznIPJNKwr09gco8GUS4vWPg6T0/VCc+qBqlPkuxGT4UbbG+NkuIPamR9DzRTo0+avg5vNPR6L1NXU89FBOjviG4CT7lC5O9HrxoPu43NT2nbIe+UpSmPoxevb11hHg+gYntPUUpEr5REgg+SChZvmXQ6T0HVKQ+fvSyPtgpazwaWnU7hZ0APLYLpz7g+WY9XWj+ve6VzD404jc+nwUFvhtrdz07CW4+cbYPP4DwLb70+ns+HzDvvDjhSz0pLs29bXEMP71Lu7x4zPO8FsALviCQjb1Fmj8+Psk1PqkwJz4aEGO7gxEoPoFoaD4FQfM+MPTwvZ2nxj6G6n89UaGZPkKaCz5fyCy+DDKoPP/zxr33vEA94avPvQqT8r6i9T45l42pvjUwB7+aoXW+N30iPgWVNT4Vyl2/hOQ0vWPSF7x7UdW+WPcpPmn6cL5OKwe/VX8hvyTUUr+lXxG+scQ4Pvc2AT0lE3y92EICPb1FET4sdQ8/k6tVvMDcib40ZQa+kn7bvY0+8T33De89CZCyPVQ5Ab292CW/kHTLPeBmzTuIPYS+rbp7vcRZr77lDxo+/6OEPbLk7r2ZIiM9HDq3PtPFmL72wp+9AmLQvg/9pj5tzIW8L+mmvSfL3b6OBtU7t4V6vbPdKL6Lh0m+ZYGuPRNaHj6XY/Q93vjovlESpz0kCWA+QvMWPpcNS75ToSO/","oc0ePlpYFb4ha/e9mvSOPYmxyr7L/Wy9inccPq0j6T62kdM+U6++PcJCHT6Cw/u+oXe8PYBcrbxSf0o7rsDZvUWnXz6Q1Te+IcFpvTLFoL1X9tS9whQLPfs/qD3VyCS+CI+PPj9qFL6yuGO9KQB3vnShvTwO0HS74m8Dvy4GGr1MpuM+7NHrPQyYEb5gCr69qKBVPj/wYz5KKxS+/gKpPcZEJb70iMS9QfRDvm0c0jz2G8g8olcwPt75gj4QQag9C+x3vm58371G/9y9AdDCPF8FF7us+e28/lQyvO5gKL6IOow+OwmqveyKfb5DGCO9e/hZPTVkM75nxrm88tnSvAHfHT7k4O29XjLCveafFD4w8gY+l5b8vXkIwT3j7im+LnyEvkPMlr1LhyY+xgm7PbAKW77/giu+gNW9PQR2kT7IUCS+0JJxvLMeujuV0Su67GBvO1AW2T2UMam60um3PoCANz4xUze9S+mRvZc0vr1YLoU94npDvsK8rD2omZQ7bWk+vtSL1z0kvl2+qQCLvbBuST5uxcE9EPIsveMREb59Lyu+FvciPdKDKT4RqBq+MgklvRY98r2FFaO+ME2QPl8JLL7zaou9F+cQPPmfeT5jxku9QgXevfiELL4cSKW8YDVMPmt9hzxr0y29BrenvERMKL6lKRG+P+YFPDuZeD34u6s9RoxKvi2Y6L2CMgm+EzWjvaXnYL4QNh2+ObuvvPVzvT2Wq3y+TjMJPhwqiz0ifLa+D7Uavbq4Ob4hb44+KqGhvWXBXr38zjA+NMNQv1N0673/cxK+6QkRPNiidD4X6tg9TWWIPXh75j0zU6W9X9xOPnj29r1l4uk9KEKIvdcBer4T+8S+3pgRPsrBnj1l7ay9O1/pvAE6hztAEyM9rIUoPZefg7ukB9W9VbwHP6RbYL4CpcG8xOr9vrgJSj59riG+leJGPXMzn7wIY/U9Vvm4vQgPqL64/A8+SelGvUsP1L06zBc95pMYPgFyiL4Quji8YJo/PFEAI74uTAa/","LOSmPNoOnT2Kmmg+0jS5PdVrCT5fAvw9B2d2PZMvV7v38wg+VAGCPoE1Kz4cIQE+pSzNPSWNZz68SJM+F4YdO2D0pL0MWRk9cgG/PQMAaD5NdwU+lgvhvBFuPj66pNI+QIXkPHRYxTxJAFs+DQ0APnNi0z03dpA++XwcPNNqFDvgCcK7lsoNPXHyhT7N/IC84LowPgEtIzwQdSw+GiCTPr9gM76Plsw82nh0PBceCj6tLDE+OJzfPZRBNj6xgS0+ltSFvQCrKz6L3m4+qK44Pkfhxz6Btgs+DqEXPiEYVD6JXE49/bkXPS9fDT7BzDA+r/j6PZJtpT7vu08+lT4vPrVxd7xdSy0+MAQLPh575T2FVOk8xvsdPeBFGT06e3a9nD7ZPeM9u70CM0a9LrkgPiIwrr3AGV89MnthvW1WUb2uhQE+qS+1PE7NlzwBxps8UOpvPaBrMTyAoFY97y15Pf3yjD2KEPA9VHYTPnWpFr0lMww+8UKdPRMnhD7HRtc9v5afvUI7yz1CHBk+RJ90PQO+rL2EkZ09FCQcPpDmqby/5xA9mbHNPYi15zvexL28ciCovck7pz3IhIU9ogYwPllOKT6aHRM+QHJJPsntRz5Gv8Q7k06evYeeiT3hfKa8mJDrPQawB77DIpE9K9ENPh7yvT3bp4Y9apegPIiqNTzD5tO939Oqvfv5PjtS7wU+mrS4Pcvz6z3cWWy9NOt8vVHIFr3yWzA9VuIKPnINUj59JO87NymFPU1bp72BhYK9k5dcvsV+t730Q5k9xObNvXvvIL6of8+9sYGDPKIO4r0ukU6+HeQQvhU4M7t5dNQ8QpMovQJzyb2U/Ye956UrvvEei71XaC298uqaPMbeEr1jxTE+yGKePH1qfr2uRh++TP2svVHbWrx0oKc9QGwzvRJQsjzSIFy8ijOwPYQg+L2nSQA98qzOPMulmrzj7IK8VPl5PEbIZTuWaK89ALjTPJ83Az4qt0K920huPb8Jw7x/cn49PZK/vf595jyM7Gw9","DnwGuxRjJD71wEg+Mp3iPTIyHj6UvzU+kbj5PfNTMz54wvo8OA4RPu0nHz5ERlQ+tcf6PSCvej0gUJo9GPBLPENR5j2Jvak9QbgbPGN2MD6uoV68tYUOPuT16r2p0io+BMf1PIk3lT27MAw9dvAQPnbxNz5nttM9OVrQPYhpP71mduo9cThePi/vBD209rU98nbhPQ45Pz78Mgs+ON4APgNFNzu12429vR+FvbN2Gz5oiWY99PrbPe43BD4mJUg+Fi5tPtyB+jwepP8969rgPcTHrj7hBzA9p21SvP7gxT3wugM+srg2PhmEpT07X2o+CDUjPcUGYD3THOM9u6hXPltLhz0ABOM+VO2FPl6jtD0cNZQ8zFSCPfX+6zoveyg+ySLlPWyvsz0A5H6+D0+iPSBRND7yBEA+hbwvPvIPdj5JHBK+wB+dPomnd750Wfm9duBfvX+OGT799Oq9MzgkPqkNkr0+MrK8y9kpPprI6j0rZxI+ei9aPtIZnb0xbdk8ZH7EPkUgWz5FSzO+DZ4hPrCbvT0w3uw9+lh9PnI1GT52zPC9Z7kHPjVmgL214V89iN+rPhfnZb2l+I8+hI5oPWxClz1YITs+lf6uPWriPT6gY04+a2SZPkA3Dj4TuK495qEAP7D1Sj6qgR09Zu8QPtoMljwWlHU+KZWFPruWKz5hGWA9nb0FPtg6XL0nByA9KY48vJwLgDtUDHu79yQCviEvwjyaHe48vUqKPcQONjy7VOI9DdwEvK4Hp72B4vm9XDeOPWKXWj1RL7M7zkKsPTWXyj0kEcW8mHOdPLiXH71lRsw9w78rPqkcojwVirI8Hzi7vXfOCT39csw8a08QuyZzcj12J1o9YXMIPMc2sD3GS808jdAKPedwIzyLQkK8sQt+Pssr6bv3bIS8xgaZPdXUfz1weo0+I0QXPmwa4j0JBpQ9TxARPo76pD0Q8cw8q/8qPvlaoz3CuGU9Fs0iPrCtRrsKFg49WOn9PX5hAD6fyRY+JrIsvclWszxmgTo9","vN8hvD+3y72+Tgu9EsofvYe3ZT21iRI+f8wbvSR7qL2bEr87f7fjPFO68TyrQTA+gKPcvdg4sz3NJAQ9iI7NPf4wyb0ib6E9rKbMPCzNGz6wLpY8HtInvmy+uT2O7B88qYidvdeT5j1FnSM9fwL/vcmVCj2cRJA9F/q5PTpyp72Fy3296wPAPcWuLz15jo09Ay6LvEB3Hz63fok6I3DfPftH1LxFlqM9jLR+Pea33DuZhp49YPc6PNhxBLxdpQ++bajzPRwAL731oy09pjfDPY8S0jyzTEa9+Lx2PQejGT4lUHS9M52RvYBAkzzDJV291FpXPZ4lLr4+9yS99ojJuwmxbL0L1Rc++T8KPvXcLD4WCZI9HVtrPkxrvz2O6j0+9nNPPR8zLD7uY8K83GMHPnp+9D00j6M9bOsFvRVsFb4ovts9bvbFvQaEZr2nah4++iSKPG6mOz6dT7W9a9zQvTlPij20lwS+c2lfPtGbTL6v2W09Zs9pPsAJQz7mnu49Gh5BPhYVZz5OEwe+vHMMPhxEBLysMF0+48sUPoCgMj7+gxA+TX7IPSXsA76E2n4+MzImPh9giLrJze89K6LVukHcjT2jMVu9Nlj+PVpCIz5cwrc9alJAPeiujT0VexS8wmHXPZnKPD4XC5092YvSPinCAT0zhyQ95xPmPS6jjD4+gYK9IEfIvdPNVb5O5zA9rRlFvvzm0r2J+HS93OMFvbRiVL4XdgW+depYvTMnhb3fTIK+M7qHviI2Tr5A0bO8XntcvlBCSrxIQ+W9BYySvFVbdr6WQ1a+x8pHvalAmL6ttFC+HadkvH7OMb7nJwm+QvcXvgt5rjxXafq9a2A8voIlg75Zg8c9CcfLvU9s6T3iZkK+KqKHvUZHZ70fY4S+HCyIPXNCmb1ca2e9igluvq55vb2iw2K9eehEvpaQTL6RyGy9yWcCvnOo2btom6897eUXv2852L0GIie+Vl/3vuemQrwVVRG+XtA8u7I8Db6BJw2+UEv0vQojCb62G0W+","cA/1PAkpzr3lSSa92+pXvuwKi743DJK83tOJvZwnJr5JLls8zkhvu0l4PL13hgq+8Sx/vTMHjb0i2Hi9OSYWvlEekD2sdjG9rKXzPXx6djyfOze9xQ3sPcOJFTzbptG9W8e+vMzzOb1yTJW9U14tPQxmvrwMJ0+8GjRHvoV6lL3YwHq9Wj9UvVH0yb1NFza+LAiNPTB9/r3GcA89OJt8vcwxKb1hD249e5sfvQnf8b2HNzK9oADBvp6/yD3uf+e9xQATvSaJBb4zCV29Rix+vVe7Eb6JMIy8eSpPPdPeSb4zzMK8MPi1O3pNlL17TSC+6k6Gu2cA571a9/I8pq33PFF8ND23ZEg8RUkNvfaXwLxWGR68+0davTryeL15cA++ACW9PfkMAb7n8ii+4O3GvbFNqr00RWM9/bwGvc+cOT1kGA0+nQNZPfGOmL0xjS298tQuPn8xKD79lO883dOqvXQz0zt43rW8jvT9vW2vhD2tfr28oaexvSnrML0l/aM9UPXrvWwUB71+2yI9cMzRPBIqhjwoUYc9EXFFvSLf7z1um2K8ASetvEpiEL4b64y8Yo5JvV706Lrr9QO9YWCgPJnjTb0CxtE9wnDwPFkIJ71ktDU9vRUfPGMLaz3pfBK+q+umvGJPsb27Mnc9E+eZvc/ZV705IsA9sKYEPh4dXz4FACE+KnHTvSmaBb9TzAy94X/evXacjr4n3s2+75AUvj1cQL6gNRK+D7bovUaAvL1bq1W+0PuAvkpJ+rw+YJm8x/L/PER7uTdlciQ8qs0uvehgeL5aLKu+iom3PbVBDb35UQ4+B0ynvWwrmz2BHU2+nK1OvXskpL5u9r+8j80fvD9+YD2ZlRG+kuOdvIYYZ7zlkhW+VqCNvk3Ma71JMUS9ltqbvQVbp77iWPW803gNvhstJ7wGn/+8OygKvn2AMb4ebdS9svibPZZIk72S8wy+++61vFt8f75eplq9TeQ6vq5TPb3lcB6+fsRnvZddgr3Kzd69MWBEvrzcZb677li+","pEpgurrpTj2jVS2+jMMIvjBDIz1b1zu9LDubPXP6nD26OTi9551NvpClGb4LqEu9V0OTvvjvzL3qzL29IdajvtDbjL6hFio9Xzz6PXq/Xb6TFRM+a/H6vRcJoL0nwa++7CIuPWxYuTribx29Xs92PeECmr5XQZ6+KpRQvrbn6b0SEnu+wSURvuJVqj0P29a7CPLdvctyfr4WyKO9/tpmvmDvgT6hD2Q9ndrGvVFtJr5qifY8OblQPrdlob4O78M47aIIvqkwYL5jAGK+Y53hvnQ1Vr4IzYa++G/ZvWS8Hr6GxGY8jY88vgJNSb4sdM89bNJcvgcaZb4WiKW+ZQSqvX51bz7P5Pe9kn23vZIA9zzLGeO9sjSkvTl0vbyUEeM9nVOXvfZ1wj3YVq48GSlnvp7wjT1Zdvy8fYpfvd+4Dr7wxQg+uwGsvU3lCD0WBRQ9hr+vu8dadLvPK0y8SkICvu+J5zxyM0M84eprvRn5nLxl8zc6S3LpPDH5CT8w5SS/z7j+vfYcMb7yNqY9D93LvOTVxLvW18096AMKvgzdQz1hzyG+B7TOPS+01LqkvzC+R81/vbCwj7ycSIU9LFUAPZrlcb70orW9i8cuPUtlHr26Bmy88shPvaApP70zUQG+sZYhvu0kE702g9k8cFO0vDu7uL4cmpU+h2wAuw3hYD2pkzI9rssFPTsgXT2LQ7o9GXXOPVjeab3cAxg99IETvbto4jyWPq+9KvUbPXHUj7wf5W89k+gYPU0F7rwJjm09oZN3Pre3KjtsVHs74xmIvJxxZ7yyM4e7NHgZvrZOqLxd9ZE9vP7CPYybGT7LqXq9V5Pju3WSlb3Q6LY8dZ+ZPvcMaDxkiM28nZbcvXcG8jxqZXy9ofRyva7T2D36zXg9y+e+PQQZ47z1JTK+V84DPTefAr5oBem9YFjzvVAyvT31p2W+GDwGPm8HH72YvS4+6Hx7PT6QWr2Zzki8Cm4CvTGDtz29Cty77SiXPTdLUL0PxZE+wgsXPujnZ734XIU8","MEKvva+E3b0hTAG+qTxTvg012b4XkwC+Ynp4vcKMLr5T8HK9jrr9Pfrd9rwbpzi+gLNMvubFxT3ZAIy9OMm8PbRjbr0EzKK8TJxUvCga7L27JTY9z06evg2RdTxKDYs9AkT2PCcPRz73yXM8vvMFvoDdR764KRG+slHmvQquPrz2yva9awBTvl6xPjzSeQ8804b8vY83oL0sq0u+JZeyPZ/ukjyVPau9ZfvrvfD9jb1iEBW+X4xVvRJYarycHLq9mdKOvUVSorzf05+9NoMAvM+IDb5H8b48h3ibvAQhED1WpCu+1SEXvbhycb1vBfe9kyvjPYzP7jvz3ou9mMK4vd8bMryC1GE+DZSQPu2v2D3xBpE9O7A4vilzWj6UJOQ8mkCzPdqoxT5fqLI+nw6jPTV3NT7R2y09XAisvdtyTL7ZGzQ+0ES7OxtGRL2T/BM+9d6PvR4aAr2V0wA9r+GiPeZcwz1y6lk9wservI/Lo7wJI+I9AnklPmAlpb1+Edq9H7xUvgbGFD4iOsG9qZTXPZ813z2Nio09+CdPPlYNPD1ctoc+xZ7SvW0CQ77MHXO9ppoQPhcgGj6QtLS+9nUNPCTGdj04dpg+u3CKPjxbgr7GYeK7LVHbPETexDvqKRS9BjFGvmJMe7tXg/Q9eS0GPrRhUj7iD42+Lq8fPe4sbD7kJYo9kOi8vu5IGT6y3Zy9+eVlPohsHz5CnyY+A9zCvUpQMT40M6K9ywyOPlFNgL3pBZQ8DEezPYyQe76Kp6I+SaKNvczc3btlncu9fDb/PdP7bj6af9k7a9koPlcE1L64+Ro+XO73PsytEj4ZLGO9Qd9VPfu8xbp8iLU+zTlxPuivfD23hSk+TrTDPZh9WT16Drk9yOSKPuW9oTyaL9o+aeMKPQu1oz6STJE+UB0mPvp9Zj1jmOo+W1OCPkDt1j4geT4+DrJavpDSmL2q+Bw+22hWPSenwb2z+9+9nn6tvmPvaD3mCJS9Mjetvejjlru4cfU9AJBmPs2/Cb0UkbU9","wlU9vegUuz2qoqM9ikViPFNTeD1mhJW+usN3PipTJb4lcuO9eosbvAR4Gz5+Bh08/71+PQ6cjb1mEjy+WUcgPlowm713ko+9283Ivgo0Rrst1288IdFxvbj87TywejI/1PJBPKhpoD1W/3K8f+WqvbSmuT1AIh++Bj0Hvg55sD7jfDM+t2aJPIdIVb47w4K7vYVWPXIcBT6HqXo9AcPSvVzxxr1B3B++wSP1u7xVcb5CcC0+kKaWvt0zFT3QzS2+feqDPU3vCj251oI9GwuFPROA+T61t4m+n9nvPYpwNb6D+yw+GyUvvrZ77z1ngYY9ktUAvcjFUz6nRGg981ROPfc9ej4T90A+0/oRPyjqpj1pnaw9QOMAvlUXNDxdGte+fLeCPDMp0z6jYpo90eqLvRF3Aj68ax8+lCkFPueHuT3b74u+h+iSPkOR2Tw/Yb47ctYKPghmkT1M902+e6HlPnuI5z4E+qk+5Q+nPhKbH7v6+WE+czD2PbQ/HD8/wVQ+aRSrPUAlbD5jYqs+JMOEvhASkr0WByu9IvcgPpaHRr207LQ+2n2sPV6eQD75aiW+pdE9vqKh9D2HFCk9yKkEPjxuwT1X7Fg9lVroPmhEOj4+u2K9uxWAvtsuers7/d8+Cc46vjAVPz6cYd4+4P0IvrSBQj5j3is+3Hv2uxQ9VL7Zrq+7dKrnuzH1Cz6YMzk9I4XxPZCjXj75XKQ938aLvJtkET2FsMk8LDcpPQ8qL7wGZLI987YhPktU1L0qq1U829AHPrgMwb7EUWQ+t26ePRE7nT4A/5M9LmpFPf1DuD682ZW9cI4SPs2X7z2BiAY+2Bs/PsmZJz61GBm+IFQ0vddFiT2ZTpK9H9VFPRZUIz6bzlM+hZ85PrysNb2l5Xc+FXSLPYRKWj3kYZ0+1p+9PRLnsTyAQAA+gb+iPrWSrz3N2GS5AxbvPd3ayr3N2Lg9zSPRPsgzhL0yuu+9A+4kPioTIL351R8+rvsUvnDkRr1cVNw9SS/aPifdwb2WSgU+","JEA3vL4bZT3Xnri9yBjfPX4nVLtLk0E9vvHDPfuEJL0aPxc9l2GxPR90gTwAPN29hQ4OvpkbBbw2JF8+3VfQPQEUiL166PQ8CMByvQ7DAz1bUQi+9kbXvUQUA73elRy9ykRQPgJs3z1U2+E9I4AHvgvipT216Dq9xFPVPXCo6bvks6m9LQI8Pf2E3LyZPtI9yEqGPSfL1TwLWsc+/y1WPSFrDD3yD4k9HSxBPZUB3T1kbQ2+ASaVva2YtL3Yudo96/xTPD2mPj3g5tw+Zc8/PcRFCD5/3/K8Uv1RPuKlgDqCiCg9pcbJOzPuKD5dmIC96GPruiZA3z3ut4o+j4sHvkH0SL0EJRg9I3+/vRnb7r0Ggbc9LseIN3+S2L3zyim99irBvSZrvzwyVoK9LCSIPD8+jDuOBQ+9VT90PpSdoTwwPlu+9YSIvWenpT1dSZy9SWaovZaB9r3/RL+9dsYOPRS0mz6bsL29rcJePiZGOjwixAC+vpeCvUh2sTwTVPm9uLXOPPfaH75aS2U+cRGovceVl71cVDc9bcAMvnxDqb1HOjM++6QPPR+Tuj1aVz49A+S5vBJC2j3yXl8+Yr+qvaAySz3pPTs9+B7ovjZCRbwAyVK6oCNTPiX9gb7LajY9IsZxvHGAo7xpMBW9e+PwvcHf1Ds+S3e+1PInPrpO8rz24h49gTWKvZVfTj7mGcG7JnMBPiEV5T4r9JU9okyFPpsLLL4NTt098GKRPgQ7CT4tEns+w0PiPpR7yLxkY/I6qPhLPijlV77ypbk9wC5au2/Mhz3Mca89vZDRvSvYoL01QpG+d3WUvcHAoL0nTI08oA0NvPA2Bz92Rqw81VnXPb2QgD6/NM29XVnMvUL6Mz9Hs5g+CcOZPLHqQrxlGyo/qc65PWToAD7F1q28WQdPPjjG8jxoqUs+0/UFPqTK/j3PnRA9tYfHPS+mtr2DogQ8VyagPvwDIj3HPGK8K2A8vYkNGD4fZO09z+T/vbQ/5DtYQ3c98H+FPetCarzbS+I9","IVDfvYw3orztbQa+zwPEvXtVU77vITG+/X6VvNIDO77I9Qq+bBwCPoNm47v5XyS+h9NIPEsNvb1ppoE9MyM5vOEflrv9G7o9cNTZPFN0uTwbEme+WWYjvm1/S76j9IW+vlz/PZutrb6Sui2+TQgovkavMr6Nbt+95JhpvWPxGD5x5fE8JLvuPDUgWr1MsGu+2sRWvKY87L1ugq69acDJvWK/sT1FErm9oQUTvsoSGL7W7GK9y4rcvE3BKr4vlqy9JjaNvWVpT77+A409stqNu2Yw6r5HuGu+SH+WPLf2er6YWby+aBuXvvh0Qj3NcIQ8b2q7vVxbar6SY5I9kfQIvXrz+bzemIK+TwZmPVIRiD1EnYg9ZHzuPRNvlr3IHrC9hG1HvVvTJ75G+Sm94Gl8vf8fhr20w/+9xg6cvX5VlL23R/07PQeDvk40prwphdQ8VAcYvoute73thls901UcvA19yL2kwg2+YjnTPYVglj2cDAG+iuwJvou5Nr0Et7q81cQ+u7FWM7y1YgY9N0VpvQYc8TyjCwa9EUu/vqiPzDyezBA9zqzJPHA2xT0ijae9y9IGvWiyEr5eD6Y858THvXvljD1bSeA9L6KtvTI2hL1pT0q9Pb+ovQpoWb3RcuG8L8C3vMT/ojyjrbG91AGAvZYIMT3oc/i8L0i3vh2jjLwQXIo9dOhDPvH13zz/X8O9vShYPeVyuL2ai5C8XkNgPs8VSj0eY9u9hWGFPVqCyL3IcKM9P/rWvZq0P75LP1I95LuhPa2AHr4/MdS9TdKpPT7ojjxG5bU977uIvS4+PDuNRVS+NhUnPStMDL79mR694Iz4vBT4sbyusru9bL9WPhl6uzwuxsK91M5RO1gri73pmg09jTDXvSbhQj18GJK9V8PIvZcVOj2UFxS+RZKEPSS2VjwUR588uCQ3PfaANDzBh+e9TbyHPX7icz7EjLK8qKyku72yuj0TH6c9NlrCvQUb/b3WzCo9o70TPisjND1Shbk72jEuOSr0ED120Pi8","YgvkPZoPPD1yl52+s9sFvWLJZL44qwK+roSovfk7SL7uTYO+iqlZPZmbjL6ZApS+5lxcvp3W2r78JIc90AYmvati6r0KDOa61uV9vXIOVD3qQKy92lamvp2nv7088Fq+DKzcPb6qrD09+gq+wsaEvcmsLb6wqZ2+WpoDvp//8z25FXO+n51MvQlEurySXbG9wyxLvgpGjb2+fE4+fLxavmudwDrI/HG+hICQvYPEML7lyhE+6CwGvjo2pr7sGe68CdjmvJflCT5fubS8J22bvRrElr5tlbW+yjqAvbE2Xb20vB++11GFvk5vLD7QOv+9p2HAPJxs771ZCpA9cR9ZvhCleD4DC7s9/v0XPiSv/Ltmq3E77j9PPsXG1zzrgcA9+iKJPdoDkryiGSG9GIPZPb7Qoz1Cch4+bQSbPp8GWT7EOIs9nOgPPrpbIL70V4g+o61oPn7gyz2Y0AM93bF6Pjk/tT2GWKo7u0AdPqtOdz1Eqaw9yStFvk95Nz1Q1/S9AJ6wvRmPzD0Tsz09EWKJPnwI+j0LCLA9kZBXPX/MtD2MFQy+D2csPrleGj6iFeo96SYzPreQYrx64D4+yaZNPtZNvj2EP4E9LXC3PtnTuT1KBkM+ZpnfPTKXEj7UO20+FYExPn+Nhj5OpCo+SKGIPhFGpj3SfU8+d3iRPrzOwz0hyco8Yg13Piikq7xDciE9no8tPklW1TzTV3A9l7FMvgUwDD3mUi2+Z1isPa2XiLs08QI+N7LGPeSDEj3LDJm8MYRRvWcYJL21ACm+P1ZzPFTfjrx7C7W9epjSPPhSsb1/lhC+miS7vN5Rwr3L2169paiOPXDFojx8O668JMkXPvoCiL1uDEQ8xGdRPKTQFT5/rIS9IS3zPVxqib7ZVa88msrTOhmMe7whJEa92isMvZdl8Dw+k30+ji/YPuKeAD3ogCe9MJ2BPYesbbyeyLW8Ct9zvACC6DuIY+87qaScPevdfD2A+r28djhYPpNqW76NUJQ9IyKhPSc5u7wRZHi8","oDriPWLkJb72X2M7NETavZq467xPmZc9jLxuPiIDBL4Na4C9JgWjPUPa472Q4FQ+Ig8avktcRzzdbTQ9j10TvlEiHLy+o6+7vqfhPBauh7yXUzw8C2HMPJ6FiT3pIMY8k6ZqPVUxxL1qTI274d0UPBsfkbwk7aG8wOCLvarT/73WJY89zH+KO6KIij13Ltc9TKJfvRbdQz5RBN4+rvtcPKtS771VpPo8KUU0PKvEoD2bEj8+vzLLPk6r0jzJLyM+RbMGPo6/4T1HyQg8blWrvPEyiD27Z3u9dILpPNOX9T6NARC9uE3UvU3DxjyVpts9Cu0hvj9sML6B8bo9ruCXvSGbZz0bc949UVV7Po0lXD1ki04+2mCnPELwNj69Wx89XtTCPTlglj25eo89D4XSPZTJlT77omI+P+64PU6MJ70GREe8w7qUvcrQxz2c7uc9fPXnPUZaWT6YZ4S8xzIdPpciQzxQo668RJ8nvaLhGD43eAI+uae5PSExoj2tJYs8695Evr2krz1CaHy9MO19voO8SrssL3Y+1DzbPRGwUj7DpZy9zVlkPn+SXb6RR0O8rN6SOqQ2hL5bedk93rWZvKDqHT44uLS9+gmFPd9Ewr2B0cm9W9UHvtF9Nj2xjD08611/vK85Uj5Yxjk9pDEQPsftgj2Y5QE+ooTTPIUm0j22KHc90mzYPbO4Lj7NhjQ+7JmdPeAd0z0Q2uw9suw8vkVqgD79AgU+J7VLPibbAT7hrCQ9pQ76PRGwdz4FqEW9l8aoPXxikj1TCDi9JtD3PghH8j1Ohra7YksmPsWJIT55P6e9KcoBPki+hD7osBI+xrO8Pl8ktz6jzsE97r6oPTK5iD5TmDk+ZHP9PTUoQD6yGKQ+cduxPSLVEj4w8Ss+YZw7PmT4qzt6LTE+41+ovYq40j0ruKK+OwEqPsDKADyOUI68oRf6PLkt8T0Sslk+LVOnPlb+HD7Lt9Q9a32nPpSSf77yypQ+RzQuPnxGLj7QC/29ZP22PjBXCz5d9JE+","bJNFvcaweD0S/im8pLexOqfnbz3Jtnu8MkY1vurLXL4Yd1Y+N/+9vHv5Er7gMlY+pUEJPpG1Pjw9QU07Xq1IvbwUBD7MLJY9mAg6vCrZIj2IS6+9IZrYPTAukrzxbFe8ke4lPvXw0j0xXZs+kFrvu4CiJrsEQxC980XMPmkk9j2bT8K9GbSuPY5N6j10ZBM8w2aIvqyO7z222Ye9XSJ5PfyF/z0tnoY9NdDWPUCp5j35zBs+3627vco4rT3lc+09udhYPk6vqT0/cR69P68+PiiL1D2EMps9VHwfPqVa/jzt3m4+CUtMvpiekzyCf789ooeZPfx+Sr1EbW09pK89vljwCD1N2r+8C8mCvLLsUrw4w5885Q0OPV5JpztlAA2+gOiCPfaUmbysFQo+2uwwPdT6sj0XdTw+DxMiPRJ9Hz2En2K+furiPO7QGD1ZP8+9bESNvUYU+j3Vcfs9RPCpPRFxkrwxgcC9+LRSPkO/BrxS7aG9B3afvRwdVj77/a+9svXrPfOHoz2EErs8Zz+8PRIUmb2QfDE91r4TvbDBHLqh/Sa+f18SvTdSrTy3z4A97azXvXfjAz3z0KW9KOt6vccBwz3ghX47XeppvYMUcbxlAei8Usf6uwayprwL5Gc9Vp2FPj+fzb0CMZS9IVSOvE6Wzzy0dia+lcOxPPc2nr2mdmu9/36jPVU6Oz6n6Fk8JpIrPf5VED5sXlg9aMmdPIhHBD6OHsU9/QDpvO7SST6rmBQ+8UAxPi9xNj5sP7c9mkkjPrfno73vWmI+PRT0PXc52j3mmYA+WgHZPTI3x7zbzNu7REBtPcS6mLxawEE+h4SlPgVe0T1xSkI8VDhfuxIGND5JKL898/HXvUk9WT1M/R4+p/KPu6N1Pr3eVWI89liAPa9ehD2ycVG9Ypw+PsHrbT4arYS9gNLlPM9srD5lXXU7Ha4JPRuMSL6Yg3Y+A3zCPmrFRz0YXxA+bwVxva3Gwj3w0nE9RaBQPYvwMD7nOre9iSJkPIpWH70XbJ09","m5spPYT0Nb6K4eq8N2JGvhoanT3BuTm+TU9gPf7Fn73QbwM+fYvbPZcHLD7a1Y4721FrvcUW0j4gQCS+Hs5ZPa6zWz5z0K+9GlQGvnzRJL5F1Wy7sLpFvVZz4rz+Btg9ByjyveqHeT1Yqsc9d5ZOvUUR5jxlcIk+SA2tuvBUNr1uvYM+VoUkvimcMTx9NnE8hczfvVfhHj6VhSq+J3wOPvZsdT1nKBS9FXQMvTnlPb6SR7e+9/hwvfGhHbwO1ue9wYyMvaz5Cb7pbJS+mxeVvIJmfj6QoeS8DmmNvZ2z5j3GaX67dnzCPBf2dL7/IJG+xajePXaJhT5fJSO9VNu3PXdzoz3ftxy+mNSDvhAnDD6rMxe+5VZKvGLF6TxT8aE+NdTSPbzpmb2IEdQ9AI2LvcwxKLm/NgW+box1vdeNKb7g3s671K9SvgCyY77PRiO+vohHvbp8gbz7Lvy94akEPSWOpr2Zfam92V5qPczZ6z1fACA8KxCPvfPHP75vWIe9aD5HvrOiED78MJA9HazwvELK+T2Y2J+935KmPcv1CD3YkMu9c5LkPcYzX70/Q7m6tp6FvV/6X769dRg+TNJIvg654LxvNi+9DdM4vYY+yj3dxlO9er9HvmXtQD3JsNc9DWfxvCJ5BT1HdLQ7aZPNPbdAz72wuMO960iDPZOeaL51wxs92ynGPUH9B72iKek9lEW4PPZMg77sBHQ7BoUPvevrLzsw5Bs+2oSQvWjcrbzG3Co8uxKcPcvO0DzPNN29l1AKvC95+T1AWTO9Z+wRvsmGtDwd6hg90w9wvh4qk73tLpM9PWajPGO/Ez5YTIc+iepUvrTfML0utW4+XhPkvVkbRD5bAgw+qwzMvFbYvr0TvVW8hFdkvfcwj7yw9po8SI1EPg8kMr5G1qq9HH4IPpBfAb4op2O9v5K4vjq+Fz5MQRm+aassvrCzNb3XMkW8YcKJPXUiRL6nYaa+U1pqvXbNI7zPiGE+Fe5XvUrO2b2hWMA8QlgAvii+lr2iW+W9","d4KPvMTAs71wl8K9lLp4vkgjBL4Plkk+lDn2PRl7Oj6jaQc+3XKZvuWtoL0UY509xmJYPUjm8D2Ywze+Jcn+vd72XD7gLjO+F/RlPpC6j73VJDY92iw+PeFXA74AxJE9L11yvpQwGL4M3rW+d5h0PcCsVL2Q62E9ZfwoPVsTOz15xRa9pkAHvv2xsLpgnJu98yqvPvEJ+b2fq5q+ans5PoX+mzsTgkc+aiMRviItJb2IiRK+gcuJPdV5IT2qzmI+ryUOvtOAyr4QHs++yMHpvJ4SRD7G3lu+6QWVvbQQL71EMTA+eGkbvg9cDr45SJW+J7Fpvk7Okb0PEre9jL61Pc6f3L5SNj++9UDHvpPRarsJl9e9F7ZOvr0QhL0d1B89brdavq8PXT0jxZm9+ZgyPq7bnrw8t2O+kjqkPIdr5717NCC+y5IfPL+ksb0S+HG+OwqEvgjouL2yRPq94gEpvsXYRL6Wx6K9abqGvpeQgb6wxwa/HmksPbhfGbwX9wM8pysaPjnoj74kpTq9L3qOvWUbI74LYkO+uNCSPL1D+7xIMgO/WZE0vo3zkL1/8qG80kMMvhLRHD0eIDK+fn8Evlktv70meAO+XzQqPON59T1zSNW+QK7ZPWKY7L2uF8O+AABhPlZqTL4LkFs+81TrPVPLsL4Osqm+wO+0vE5FsL1oyNK9naV6vYskij6hmzi9P6Q4vXpBPDlBGR29uC3oum9KUT4x+a070V9KvRg24j2Yvfo8PfwTvg7t+b0qhS29oLg8vdSVDb4pOmU9DYItvfz+Yr0+2L69BFJvvTzuG76Tkx69HarSvYLqMT1lKwI8lNcPPcj777xfAcQ9qJSXvDpKXT04L0W90KEfvkkYIr6BnCm+JmhHvdgXI74mHhG9KiIqPUGWdD1UQqG9jFp5vDH/zDx4boK+J6mpPSw1ajwGIJU9DNRDvlWGOzrPO3o9skTSvH+pGD0NiqO84SaZvf4Sw74UrBU+KCJ2vFgdtbx7EZa9a0fBvDYe07yfcx4+","959NvjqVf72pzSa9IaCZPFJjyb265z6+oTiXvNcTHT07Y829Tw2LvqhdPjw3ShU9i214vVb7Mr33wsA9ySThO3j8L71w/BI9giDVvY2+P716sx490+HiPeHq572XExK9TY3xPWaihb2EKqu954+fPCT8nr3DUdC8Yx73utOfnD2gaxo+uQIQvTj5kbqPiUa9WpA2vtw2sLw0AIg+jJy1PaKfqjwV02y9p+VlvQ1lqj3ukZg9Shp4vdawPr7ksAc+++Y1utGFnb18Lvw9yrOLvIVRObtFdeQ8V8zDvHOuzTy5GVm+Zq0su2E+FD0/NdI8s7tUPXpHzrwTCK293kW+vFpJATxXoBG+8ARrvoE8J77B3ru9nwI2vWr0mL4KJCi+fmwVPW2GU758Ji2+izeQvvUddL5sYCK+FENvOUTo9z1v7b+99E3XvfWGaT0HuGK+duGPvmbaGL4AB489/K6aPR2zgrxPhyG+XiievdYoPb6aiES+E8aVvs0Fvj2nhH0792otvop45LydwEC9tAFHvTQYJ760mp2+FOoYPfJ/M77rCr29O+ptvjiQMj2ct0W+TWPHPAM2Lb7jQB2+xTzZvZySOr7DrRU+AH6kPS7PuL3BDaG+xx8bvjvf0r3L9nW9d9ZjPTOhhr7A/6S9cagcPe5hz7yLwIm91lIgvt0mt76fP5c9ojj0PlvXqz4GxF++m5FePojEib0HewM+Q+cHPjvcbT3I0wM/ZxoNvbBmEr7O8f26VFsNvNrmLj9LvrI+RyefPud03j6BuUO9hGzePJWTnz2StHG+I3ClvhOgoz79zPU9VN0WvQo6RL6BIH++LkEWPvfyjz7uRmi9sY7qPRvqKb6L3pc9xqV3Pk3cnD6WEdI+LzEmP7B6LT+MkDs8bgCEviqCoj0SM2s+9OTjPf7YjT4dJNO9OkxsPXdwUr4dvxi+J4BWvW+C6z7+rew+bSfyPeDRYD6JyRc/6NI3PlvoSz61EZc9M2fzPh1FvjydwFW+nBc5vLZN0z603FW9","xAojvfKIGL64Xq8+P3ZdvnNeJz4hDCu+5rXzvV0Vd74LCk++9uy8vg6dSL4Hi7I+YyyCPVZVvTzfEQG9mRO4vbrdYD1Xuyu+RdA2Pr6fqj2Z2ig+v6qmPUy1DD7REBQ+tIFUvnz7zr06ovc9jaGPvm7SsjzinwY9eDOfPkjqDT0DYWW+2GefvqzEzb3b3nW9SIXyPbt63r6jyqe9V3yvvfCqgL43s4U8gHr9PT7mebzTWqc+fdSzPQadvT1oWwI/ec23Pjktkr6a+QG+c3lpvQD1jL017J49GOCaPaqvM7yA5Ea+ogMKPnwSbL5fIDI+U92PPfrRiD7WaGa+k7//PS9kwr0PRpu+0RJTvtcTgr0InMK9QedEvf/yDj6zyjk+GBJMvt4IVr5Edk++PT8/PjD6yb1Zlxy9HmAGPVSTS74EBU++4+EGvqzvmb4gFQi81k8DPeuo7L2WUJo8wauTPSUVMzyb/IU9TF6fve8huL4VwJe9dI2oOsBeZ73lJvS9c8bJPBsl377ZzY+8haImvfEm072t8GC9UqtFPn8S2T2yznO8d78PPrWwqT2MeZw+dEb3va/toD0w4iI+m1jjvQYqpD0cJy++a9q3Prbs3T4PVw0+flLOPRNrUrxfxgQ+sMpdvnXsx7wULJw+DfIIPirUSj1zK+q+ALXyvZDDeLw4Lfa8Ogv2PVbVGz+aIE68gEIkPgAukr7ffmy9feMNP9VzMD2wSoU9KDB3PhXtRb7MjA6+iZUNPmkxCz6NKu4+ul6qPYH0LjtrkO+8pLMRP8o/a751QHc+wmJ6vmIPFD8+Ax8+jFeFPrawCz7Om5s9aOS3vay+Cz9fTiC+j9qhvaaugT0dM7w+Ng6GPX8l9rwc7hQ+z+WjPkDS6T53iX+9QWD9PdKRpj1LlRg+hglFvt5xaj48lbs96mU+P6q5XL0UayW9E6a6PQ4W8j6jOwm9Pv5OPiqhoT5+5x09IQ5wPj9gVz5G3zU/aAccPh3uZb4qGvU+WmE9vSte3j2sKWI+","0bh7vu9XJbsF6KO+/H4JvolCxL7KGOK9+KAuvUqtw7zBV9s8C5mIvtBohr5tRG0+p8P+vLxYvL4sQJu9S1oDvsh/Hb6wj6g9ndzVvm9Tk77Un4y+pPRHvgM6cjyXj8K++6aQvRRlNr7ETC2+w0+lvkU+PL760I+9Wmk6vmbQMzzyq9Q9QMR4vkCVnb08t3k7l3CKvuG+Hr5KvIe+TodmviGdCb+D+Lo9joUkPfE/WT0voSe+v4fEvJPDG76RWQu+77CdvDEIi74HD2u97CIzPhpNiL6u+N48GEILvrtVxb7E0Es9ROscu1DmIL5Y7i+9I6jyPXENq757Eya+ACfgvALI177oQ7K8WqERPlvQw72ScOC9CGr5vVKN3L2B22q++QkTPWotnTyo/wk9v/AnPv6cHT5nf8k8r8UDvlnMx72WZiQ+HZzzuv1ZK71vhMK9qxLXO/7nRLvKKqe+n2omvvaY0by3dJW9rxHLPNoqK71sSLi9RnEwPaFBdT4ZU0Y9AnoAPgJqa7yPlyS+nPkRvpQ0VL6sFS49YdVYvFsf270FeIW9GMpKPG65Yb29b8W9Uldpu02+kr6vgYi9AlqQPcgM+71Ek4k9ywkwvpIbgLzW4e89VJ3Wvdllub2Y0169nb4Lv1RMZD0+ZP69z1H1PIgk670fBeu8NqanvepC5z3hm9C+iAygPQfi0b1KKKc8b00+vhdNR7yUPzM+DL4gPmuhyrvnq7e8/GKGPIJ+Mz0BVC++S2yoPaQhC75rL98945kIPju6FztS/Qe+YU9WPS3i7D3cGdk9RxcdvRLBMT7bFJ68DWdKPX8hhzyN5oY8bpZkOyEdEb0oqPO9pXFmPR1HOr7pDhO8/+j5PdTr0L2FMCm+fchePCmFm72SoO492c5gPRIAA71ME5s8sP/KPYqX2D2CT2E9ODAsvNb9g776KdK7wfwHPcUMFD1TtTE+v9TMPEmHIj4D91Q7hxYCPN7Pxb45Dam9zlb0PYKAOT3kZcA9GkzdvZI1g71PmLe9","RTfkvQqukr36QuI8hrJMvqnUH76thwK9Aj4qvjJhCb63am+8gCAxvq9OALxAQsi9DN6tvdnecL0KUR6+kZInO/Tbjry6ElS9g4KUvYtg7L0Z3Xy+Kc/svUlST71yJqa9vYSQvJtnP74Q25m9GYF2vumEZr2jsYq9Ds7cPIjH+7lRCna+WXUDPXKBXb4Gfug9AQluvgNIlr5AmHy+kFmjvFOsqr2FaOm9YZUEPcpW1b3W01G++nMLvrkQ2L0ctrq9tHjIvDpA/71GPmC+1ICDvZ6yHL51mBW93vqVvXpshL6YSbs95qNhPYryE745eAQ939+gu/C7xrzWj8e+iG+9PMk2nzvB/xu+mt78vRmkkT3qPIK+OgB6vp5Znb4EK4y8FH4QPkoKkr4kiG2+k9YSvob6yb4Ns1S9ZaDevOKLLb9ADiG/MJlUPEymwr5/N9C96vgZPUVGxj1TB3+9SOq+vR9CAT6Y2xg9GG2SvqlmQr7kxUC8x5Yov+F+Mr5MPjK8QdgpvVU5Zb1S/gG+GrLJvaqawr0bGH28uvqavpwzWb7oqxA9dY8gvilkib2Nxx69CcU9vpgx/j2ZexS+RLXJvo/6jD55Z1K9ZECcveRGUr1isfq+arpKO9YN0Lpu5Yy+V/2YPHTn/rsSlne9rDAbvfUIXr48etC7+YD6u+axnL24kx294Cs9vhjtEj388dW9sF2cvjzeCz5EY2e9o+t8PA9kyD03VrQ9YoNAPdudWL7btZw855QIvg/t4L3iAu4+TVM1vqKbKr4My4g9ifnOPfVejLq3Jy2975EJPCZyC74KYpo4ynBwvYLp9L1cnoG9USafvTTsNj0YBPW+ktoNvrGSmz3SPt28Tyz0PB+Bur3hy128SBo1PRThVrtL0r69qfOFvCdB370Zkoe+gXJRPErRurzgZq+9A1P8PJyoozxJfGa+Too+vWQXuL3uSu295kkLvt/tGLwbfAO9IrmPvRoNSzwAuLu92swgvlUQsb03hfq99Wt1vcCpCL3+yng9","d+eRPqQIDLxxVc281w2IPRxuGL1Igw++x8jfPIH8072pGNY9Z6+2vbxccLrKXeG9xOfVvY6UID5yKnG82hPFOYw3GD6lAxM9R3EevaU4qD2MU109g6ErPrSMD718sJS+YF5juoixor0H53q+4wyuPVyyMz3UxR889KHKPD9BAz624ge+9zsrvkNAM77ko8+9zD/Ovb2nSbwcSKW7/E/tvPvdqry9jmS7VMQ8PVenYr3Q2u28msF0PuKv6T2JEhM8fA8mPbgMjD1gqAs9NIvXPUFWsz2PMjY95Z0SPe12HDzBhaC91AGHPZklv7y52009q96XvTCKrT2VRGq9l1kQPpI8p7x8JqQ9vbmRvgHlNzwV+jg9Uw+nPMAmyL1jPDq+BjzSvAVW575WRV69y8Bovim5b75f7369TvYRvrQVCT7fySS9VbLnvAZ1qDncE+88xv9LvmxbH778QjI7K1QqPIcveT28yC2+Ee6nPduZnb49Sma9s2Igvo79X7257uy9+fphvgkxDT2LNZU94eTPvJb9j72d7Gm9YqiKvad2BrzITGs9WGuivSzPDr1gbZs9NjcmvhRbgDf9ueS+L24TvzTZ2zx5mAG9J0eZPfvbbb1QJ5y+5A5API727r0kqBk+EXsEvlGQ8L3zpbg8N7EKPb6jtzzU4qk9XumTveRLq73Xx5u9AiZHPX8jfD5EDs497BYzPpxaPD5nFLc9sai0vVooWz523Hk9XGGNPYN/hT7bQPY8U5AePszomD0binc9firIPYaZTT3ue0U973tePl7Y4D2Zx4s9IDyDPuWJTT68h009tSxLu+gWgz4gfrE9VptOPg2d7T1IuVm9ghqrPRufX70hYiE+wNl5vYXeAj6esv09sbqkPGA/Y7wMWLE+fG5MPg5yVT04ig0+pywdPhVQEj4UWOU9MCFVPtqctj6DJeQ9cBYNPFz9EL58eRE+DOarPuC5Zj0krjU+xsL7PuzHnb2ivYo8KRe3vf4ASD2/AWw+bs6vPlDZH7zZUyQ+","R2bNPeWMPz5eIpY9WJeZO6GhFj1dzBQ9KA5wPFgnuTz5nQu+Ab6UPQ4Tj7zJeS09zfGGPdzIOr0SFJA9HopgPnOMWbzxUfY9lx/zPCC3fL2/8Z49vBpXvSo9vz0RKJ09qdJUPXUjgj1yRqk9cb3qPBSLATyPWbe9micUPoKZDT1zUo89Vd5wPRX977pNw909CXp+vInkRj53j1c+Ht7yvGQC5z3iG9W8ya5FPT4tnz2NDly73nObPUYuCb4v2+k8vsrgvGqBlr05nBM+67nsPUm/hr1rbx28daSJPA0aAD714E4+/DQiPeNEhT3M7lu869nKPKI1rD3CNu099hIVvfHfqj2yxdK9iXAPvY9llj0p5Mm8n/1VPdYR/L1aVGu9+zg4vhJ3PD5Jd1+8RTk2PS+rm73NMJu7VexWPb8HS7yyRz6+22lFPN+iuD13TpE9GchWvdDBrL2Sg+I9ADK1vSPT2DwP/eu7XMhcPV/9nL35Xig9GHjgvT4A7zskUpS9V5ysvUadHz6i7rg9X+uOPSSgGz7V2hg9GFxQvk7Ypr01CrY9AcL/vFAF3rw/eHI93iOOPHIR4z1ZAw0+SAKiveJeaT0oDDu+kdyYvYCnPr0qB4q8wsY1ujdIYj2ZwBc9HMDyPQ1uN70pMZO95iJGPRwEbDy0qfa86RS/PWpW0r0UokS7ivWzPfRTLT48u7M9NBrrPXRDaT6hYZg+L55pPo2HlT0b9tk9ppCcPeMdIz6K3vE96ZqcPh1pKz5LUKo9+19HPd+GMD0vE6k9CLctPtDZkD7OuE8+Yg5HvQD57T33W4E9A4W8PZTijT3VwcU99Ff5PNArwT0PQYc+rOnAu/hQiD7UJRY+CrT8vQcrOj7o0ai8SkcIPo4iHT4VVyY9gQWoPSCoAD6z9o+9K6GsPVZi6D0zKgg9x0KYPRErKT6nPgM+s0wsPXQFmzwXADA+NN8qPsRCmj6Vz1K895tnPcf5Rj4KIU4+KRizPajU8j0/9Fk9w4fLPSFHET5Ytw4+","5Be9PjgRs72jrXs8+z8VPt6WlT0kIwU+anvWvtr7Mz+LblM8MgPdvZyG2L5fva6+dDVcvhturD2QDCs+6RYivkmfTL4bwD29sywAvuJXRr5Xiza+5XsJPS1DTj3/V6w8CeitvSYPL73hK/U9k3XNvSQ0+T0f8me+ad+rvs4fLbzy5t+9/tKlPptQmD3Cqwo+i7W7PUBonr1s7bI8wLhnvXlvOD7xpuc9sEEWPj0n5D1Uhxs+wcUGPuEcPT6n4aq+v9mDPgKwPz7z2TK6XwRrvTeGhL4W5Lm9jjYvPrxRQb4nvvk+ZgBzPU58Hr3fL4q+VBPlvmGnJj0TRUo9jZyGPdBrcL5NFqo9YWcfv+0ZD76RSsI9EjfIPCYU1b0nDZ2+Rl10vICVNL1Brxm+27o1PUr1I7/AZAa+QeGDvNYmir2i2Fy+cVk+Ps5AzrzqUx6+a3HjPKGkKL5D4N09hd21PYwmJ704HZW+mrqKvnlkv7yYF1W+XasKPqATJT48Zeu9yJ6hvcF4Fb41tG++BXeovAJ+DLpgopC+Tf3BPX+AXb4og5a9/LgXvnrfnr3hrpo9UOL5vJgfPT6c2z88wx6GvfXJkT0tKLC96kKtvLt4tr2ryvs8D+iUviR73L0jTw4+Tg+NPi/ZXb56YFA94u9mPv3vgT4K2Cy+MJwtvKRgy70H9c+8geQIvpxfP74hZxg9ytSmvU7pIL0sGgu7iOcbPmAsB7vQ1P88Cd3yvSViEr74VmC9CrT3O37VDL18Txc9PjDCPAbj3T0PLOo9UGabPcsehz0vng283OhOvQYPqj2eWRy8VzydvG5UuT3mxmc+V2jXvEL4nj1wIPc9Po+AvSP0o7yu7kS+5gSMPdHUq7wbA+89X2pNvnXPfT1FF/c9L8EJvUKm+DmWEIK9koenPcmqUzynYh++djkrPZ12Sj6/u3I+VVMFvQkf7L0uB4a92BhRvvHwCz6o0LS92yqpPMZX07xVj1g905mJvURBirwSd4S9VaD5PPk9QD0OmfA9","PWcDvh7URjs1D1++xKZjPYLpfT6tZw49iz0hvh4d+rsq8EM+e5yMPc8BMz2iBdy89p99vkD2Bjzko8G+68mnvi9XSz4yokm9Xpe+PD9FsD1wEyK+/mMEviGuoz0+/tS9sPs7PUWVg77rf6q+D9yevg77QL5YpV09Hr+WPnhomLrgdSo+NoNfPFxJYb7oPlg+uHLxPTYBrz3xZTq+ic1aPpRYf74zo9y9kaCBPmPfqbyQn8i+6olHPtERnr3hhoK+QIGovr/nM74Jsh49tb/hvdDjHb7WdBI+68aTvgg7kb6JGAw+1T9uPggUBb3qo349NseSvQ2w9L37mCa9wxxEPg=="],"bias":["0WavPKMCDj4JJS8+CY/XPdKW3T2hdxk+XqIIPtSWqjyRhhc+zBu5PYWt0j1ztDM9Q3MLPU3QLz6GOgk+FPuhPU7PiDwS1ZE82cu+PX9DKz4JCb49n+vWPcWh/D0+/Gc+US9wPD2HxzpY2lg+wt0gPr3xUz4leBI+mRiIPBIVgT1Wb/e9Sz8LPvMEXj05yhM+koD3Pem15T20yS4+nl9iPhMAjT1XzAA+SMSYPaiYoD0L6PI9BBpcPac2RT4HSlc98IEtPNB68z2Sni09EmWgPYwhmD7cRzY+QwFhPU34gT7t+yc9fWDWPRe/1z11Ybk9VisyPsvNqj6sA/k90iuLPU30gz/egYg/0VF4PylPhD9ffYI/tBqDP8rTgz9oK3w/UWJ/PxlBbz8S7oI/07eBP/AsgD8E5n4/ssWKPwAmhT/U0n8/wBqNP2v6gD/2D4E/xtmHP3kCfT/aAIE/0laAPxpUiz9ecoc/UNd/PzwRhz9cxYI/mSV9P1W/iT+JjoI/ChV5P+BqhT+FBIc/AtGJP7BFgD+nS4Y/p8WHP6uyej8+7IU/3quGP6gkhT/dn4Q/oVmBP1OBiD85rXo/jlaCPy3ehD9OR4M/oL6HP37JhT8O5Xk/xa2EPxA/iD9fX4Y/cQeKP0PodT8Szok/SyKIP1jigj9dsoA/nTKIP+7zfD9u3467XXlqvArUmjxMUm88UZX0PE88Dj1WHZe80cMEvYfPMb1/Pk09EJqkuzyRZTt1gKM82y4/PJ55+Ty2eSy9/wSevY74H7t0FEQ9bAT5OgfPwrxCDsq9mmOLvII0j701AOC6OlQzvOZIhj0Mx/w8W8UCvDq2Z7yvt4889pNjvQiWRL3AuAE9hHZyPPKfqzzzhE88th+2PPF6fL3XZW+9SCMUvCxXZDuGwzo8OU3Gu3vfaTyQnUk9rC8DPd+eyrvAjxM9TBGJO/kq5jthlkO9+LsqvFnpMjuFCm+7d+g3Pa9wiT26+s28fn3jOq+K7zv0CKa8FhoyvX1/TT1gyUO9","vvJxOruRqj2vKK090T2nPdaX6T14eg0+uPLNPc82Nj7GTsA9oYIpPdoDsj0N6+E9IBThPIYJ2D3xs8k8vvEEuyErlD0XngY9IB7gPZkoiz1RQVU9WFFBPqmjljrsNWU9TvkdPciZnzzHeQI9JTvxPb3Fzj1qtOw9RtROPQhrbDuMUUI+oL/QPdLXCD15coI8ficMPlEXFz5houQ9PeaSPU2PjT1faWM99Vq7OxYhSD0BRi89C7dTPSN90j3Fz849vXOePX3sqrzkNZs9Fz2DPdLwhD6NZcw8tD+nPPRlAj3Zj8A9rYHPO54EIz1bQAo+D60EPepBRT3r/X49bzARPg=="]},"lstm_3":{"weights":["j+/6Pa97Mj7ww00+CUKPPTiyiD5T3tE9hWhNPgaAcbw2Dkg+4YEgPl9ckz46DCE+WRnwPb0dgj4l7o4+jBEcPmCsjT6FX3U+LtFOPjnOoj6WCMs99tBqPvViCD598EU+OhWqPShqDj6hEzU/kDqsPQwzXj2eFBw+wid4PswL772pPEw9r7geu/Qw2z0BLyO+flcSPsS/dz47iZg+1zDtPZiimDzGHiY+rAmGPjMjID4e3f09H0uMPlit97sq33Y+NTSCPvXMGz4uAnY9+8J3PTb/cj0GYy4+F7SAu4dflz3Gzy4+4v8APwGctr2WaZE+nvuWPhUTyT4XJM09AtiJPhCwTjwkBzg+NGT/PTaLaj2+eRU+KODvPV1Kyr34Q1K+LLAYPuktGb1qWbs9rVxZPBsOlr1kom0+DOTwvT2VQD4CGeS9eRXQPU7hmj0rEao9U6jju8z6iz1fUje9iuQ2PtXpqjytKkq+Y5N/PSuT4TwGcw++0ICDPqdjnrxek949HOwNvtang70pA4S9WI4tvCCOzb36sdS71YjIPRXhpT2dEi28C4tzPs68Br0KtKM9G0GbPW9iJj5T2JQ91TWCvfiZqr5Da249B71sPvyXir3qY40+4sYavv5pwz1bTOg9LDlmPaO3D73mUNc9cek0PTdXKz4InDs5s9jdPDjmUL12mhe6VmjNPbh/KL2uOvo9pc/cvWcPsjur1lM+8FE6vCwqrr1R8+o8fYykPH6opL3Hjlk8zBT+vRqsprz7Qwc+AcQWO47Gk70dVhe+O2HoPY1HAj6Lwyi+VGoDPkOxJ763ZP69EPsaPQgSQT7wegC+6+ZRPV1fHr0JHxE+DVAaPogIpD0gvgA+RiarPsmye71NZHo9JrFSPp4xCr6NetC9UQsfvpitET4y0QO9oX+0vSh2Jb7xcna9Ch3OPdiwSL40XDm+AZKZPX385b4RMay9RqaIvcmY770xFCe90imVvSN2BT6Cy+E9nflIvl4j5z3WjyC9bxOzPEEM+L0vA1M9","8x4kPl2egz53SxE+1uwGPGL7iD6JPD8+af+dPlqbhT7+ahE+nquVvqZ5wz4rW2s+gEB/PhPrqT768Xs+R9hnPSAAUr09244+X2u4PrktoL5EHTE/2QHkPk7yYL59tv29/iZjvvrnrD63wxY+ezgUPlcb2z5Bk6U+wfuUvsUjFj1HopY+3ceOPslRCT6fY1U+fgsvPkxZkD5rnbM81Mu9PuYjrj7vSVA+y/4ZPKSsnD5QUD2+2ntxPW8S8D65mhK+OtuXPoTNTz0aHLg9sGdtPhkyED4UsOk8JOsuPmg7nz4K310+T8GgPIolgD4Q93Y+TEuwPvdQAT1YGDo+gNjevcGM77uzhn2+0v5jvk8C0b2fEv885IwaPvDF5r1RonC+MmxZvbt6wbxt1gU+H4Ugvgi/i72IDwu+BQ0evhIdl76EsmC+ecoOPv/aRb1NxvI9ndgEv1Yzrr6wnGC+1dm6vVLLrz0+Fpy9YUf4vSC3BL6eXT+9x5jwvXTSjjwfSCW8KpDGvQyZWb5ZbWs8D4z/vQgT470EqVk9tvAVPY92Yb4G7ge9xekMPOdhN74AjpK9rIo8v/lHM712VTK/z7VTvMxAtb3zQB6+2oktPuo2ML1y74W+J75HvsBnLr5eu7w8sjc0vueYiz2bqKS76i3fvL4GQL6kd2O972uxvSahDb06nEC9bjaZuvKAr71Y/J89SLYvPfwUzD1KwV8+UcKPuwgtdTxm+xc9Yc4ZPl3e/byPDrE7gPKDvtlaAL7lR5A9iHqcvWl5q73jA027kExJvrD5yDz/hdS9yDhWPbkGQT5+nke9zGFHvgrehzxwXas+mTIBvRxs7zuVUQM+N4ODvfOiS745J4C9G4wHPn+VzL0bazi98gLAPMHQlT1V6Oi9rTnVvGfWyT097sY98nQ3vqyMsr0FMtO8J9PxPf9pKb92wJq9seRMPSEp7z6cRRg8sv0PvvjLCDxJjbs9grwvPq5KST6tuaE+RWbWurXL/j3ry4E8j43yvMKUJD3eiOK8","yv4lPE/+Mj2TcYE8KbrsvSTYAD5eoNa8fvIYv2evCb+eo409KY35vRQZ5TtyBHs8n9TVPrVJ0L3LZ86+2C45PWOmorw70rO+pSpYPtWExb0SeAI+yLWFPbdOQ7z2fPC7m0n0vYWZSTwK7gU+RXCZvgs0C70pePC9CChOvnNuHL41lcY+0vYSvBaDK73xWKg90hQqPtKHM71OT066w19IPvKokL2m7rM9ng8QPh0/PL7wQkM9s0hhPf6syD2d02g991TWvHBFjL2MfoE+rS6rvDUE2r09HQ09axUcPqEv570fIYA8wszSvit1obpBNh4+J2m4Pf7Kvru8Ee88OM5PPeWJGT6/HkG/tKMgPdCAED9OX3U9GFuxPQO9v73mUNk9yO/nPcXJUjxAI7w8u5Z0vWrjgjwDz9G9Aa4nv3t3Sb7CC1+/jFCsPqGP3b1oTbC+k2yCvd6Xnz34wqa9AbSlvFrEtb6XQy6+IUJ7vhRJ3b38r7M8bgAzP1H8FT4sRKK+rrCQvhO7y73OH5u+jNBnPoWgVr5hW8E8bjkvvzbhEj2xjDO9DF3GvRVKIzpfCwi9Q/EfvTAm/L4DlcY998TAPNY9lb5sGSm+x5SfPlAq/b6hVt2+aTmEvhA/sb66pTy+ZiJDvbdeXb4MvbC9js4LvtHscLwBYhe/x7+bvRFKgj5mR+w9rmF6PhdJdD0lXHS9o5ltPsaa8D2pSLK96UqRvpQLlz0MZs2+y0oEPrnbRj6mu74+hXdkvdq7NL6qAak+MVaUviVTnD5LiUa+ZM6kvrbKVL67vg6+WuVlPkLH1z2d66E9FKiNPdQtuzoZFxc+tSBLPoqQoTo76UU+mE8lPn/CPz4B1qw+TFjbvG07Wr3OvPo+ZA3BPj/OJr5LPlu+N52dPHOytD4Iak8+MiAivnFbHL7yW3A+H7xiOw84xD6VHFq9zVy/Pv4yUT76VCE+XjjGvaiVbz3BwRG/ss5Pvp90jT7PHXc9a0mNvr/WfT4RN5i+VQgHv7uL5j4MIb2+","FVSVO8lR8DwAp0s9d3vbPYiTqL45bNY9vTtdv+DT8T74lpM9emAiPoe7zj657hK+jX2NPHFviTzcIIs9K1eNvZrnkL6eSUO7KTV8vWYRAz+DaEE+DCaGPW7TdL7XId0+N4YuPWcxkj1whcO9uZVSPfO0A75Mm4u+aKUfvjRJKD5Z4tQ9xLyNvjWVKb/cV9U+na8Kv3Q4pL7QNcS+uEYXPmvNoLsnh7m+Kbo9PEyHgr1gqxA+y4wZPlXxvDsOSfe9FLvBO6lzbTrgkY69DUDfvPxoDT6y50Y97A3jPcFzBr1eYA6/qaNAPhU7xz7H9ia+v43EPQQyib6yPty9gPlZPk99iD43MJe9TLYqvl117rtrAwo+4fx5PacpDL7cdtA+KSMBvWcB9z4u6jW+cUvmPZ3Roj4fDSo9KkoTvpzvkL47mZO+Qkbfvtsk+D3ewgI/IU6ePSrCGDxfkTA80QKCviSOxzuH1Oq+/ID3PbzWNT3Z2AO/YyQiP4Tlab4zeVS9nUKdvXYZzL2SKhO/upHOvZuL5j1YfUW9LoyfPkN/h74BaS++98YGvu/VNL3xTC4+NsG7PZPN+j3XWqI+Wd2nPurEhjyejQI9Z+R6PUIEDT5NKuW9NsYTvQGCij47jhY+AwrRPqIXx7133vE9Z9DavuywFTsIiBG5Jl+Wvd36lL76Pme+re2EPjlPpr7Oawy/6AqGviozo76geGu+45XZvGDMYT8ALRM9dmMTvfvMOrv+wvI94KhWv1eTbrvxfic+XTBpvkUAhL4emmu+G5kMPnoJg741ZQc/W4jdPrN0lT70T5C+YL2GvSGVA75vuBY+wWQ2vzQ5Rb5/xli8RvVGPpn92j4Xgoa+whBhviHDEL/KXyS+CpZoPc8nMr5Ql3K+BUwBPwTmCz4zNq4+qgOVvn/2lr4wzYm+RT54vrCcHz7c2oC+Ye6vPscENT9N6x6+WPIvvpktCD+D0Ki9pWYiv4Rm0z4sTV0+nMYJv3ztjr41+ts7FyszvgnDIL/i0io/","XURSPYgFhz6o8LY+fAKtPlWBCT4hIhE+494LPXqJpz6P7H8+N4fRvLvt2T1m4MM+vzoOPjFpBD32ejA+2lqEPh/2bD5lnRQ+JFArPLqFrj16UDw+3qgcPtsHhD6bCWE+5PsTPkBPKz6HD3k+RgGsPpj/lz4ec5E+tOE7PmEYMrwCjF09mmpBPhPmRz45BCu+H9e0PoRzFj+vKUC9XwcsPiO9lz4o+yI+pjmNPll99T18cgk+dpORPuFQfz6zNKI9IN64PX4Yjj7WyAk+lKaKvUV77z1LD6M+OD6MOgUKwD2prWS98v7XPjgYgb3+tdA975ImPgNw7DyrshU9iWFsPiMSpz01q0I9A7jhPS99cj5cAA6+ynbnPA0FeL3Uz0m9fALbPYzYXb0go6o7hYxbvMEKfD1hcFo8IwoQvlO0g75qz/K9QeyTu5Hz3z0jnF08B0gMvg3RGj3K5hA+blXXPdEaPbuRBX68cGSYvmlGGD3nHES9XPCivU/eKj2TMPg9l4WCPK5pDr6QhTa953D6PDzXIb4HHH2+SxnnvZKbMj5DBRW8cPNvvWc+ij1hbCI92cVIPR+gFj6Kugu9FWa8velMyz3Qyp29MWEBPsytnj2VYXy8TPscPqCmDj4+JYO+vaM1PZmIJL1gG4s8rSMaPGvc7bxv0g09Fq07PnClQz5Kv+o9foBiPZyAdb3c9yA9jb84vn1GKD27udo9DIKHvTWK3Lv7aMe8i0jBvTtRB76hvGw9TGbruxfAFT4aCC29c2n8PbNwvj1h5jW+PloNvqaeJL7IUxS+AjTGOyZWvLx72Vo9Q5lpPaynk70w0xs85IgJvUQ01rvHSKS9XN3YvGOkJLxzMxQ+7j44vnyDhb3eLxC99Cruve/TXzycMBY581p9PLVSVb01GoG9CAUAvnDVgr4To7y9wJxaPTpCUjzhk0U+1NGYvSOpr73NG6g9CSYsPbB3Qz2ho389E+Jvvvnt9r27DSq+CBDSvX5dtr1DMDu9feZHPoJqqr0R8RM+","T4E4PpmJhT0F/rO8nad7PXtRhD6lhf69MPzIPYGNwT7ayAQ8/GZPvPagv7r8mXG9MjglPlM1OT4vb8483zC0PnZ/oT1jxta8y1GovdMZzjwLTQ89R/wIPnF8fT79Kpo+Tos1Pr4iDD6qo1g+qJCmPpuHTD6kgo8+ti9TPuogkD4GjHU9cz3DPjUcXT4KpAI+zL4RvR4wBT42714+AE4TvpSiXj4K6EQ++8ZbPsAALD7+luC9DLAFvkMltD1Nldk+LlamPqa/qT3Bxic+qI4APqx3Oj5mFzw+KpU5PsGQbz6HuhA+WqtOP0dSeL1izYU+7DDtPd/8sj2+4KI9UAjEPfNMbz4kAyk9nMtNPkIWuD73hic+a+fSPTUUgz6uhqg+CNfzPYbhBT7drM495DMSP9Thlj4n5Mk8dK5xPoAnMT6gELQ9YsznPUxKBT8wOQe9khBZPqm1zj1ERmk+/7VNPukqqD6xhig+Gi4DPa+STT7bge0+VqZ1PcJilT6suYw+aperPjuL3z1J7Hk+GkJ5Ps88Gz5GVsM+JMpzPqmXLD6DUnI+YcZJPouWXD2et1y9OUsyPpVcYT4iUVg9Xx9jPj5QmLrFLxg/kdDEvBsK3D3bUv099vDAPaLERjzyNHQ9QexePo/lcT7YDZy9M5X/PXzd9j5kHrK+fSGxPfPPY71BM7+9cEEMPaayQD0LJi894MEovS6ij73V5Ni9Y5UTPRs+IbvBFlw9Oj+UvdaXlD5fORM894QRPTskA77qlee9l0kwPKx6p714KxM+ob95vUxJSb4LtSI+tl12vd5Yzr0kbQW+E4ymvXthWL3v/Ty+2iOOveHkOj5JcPi9jgA/vfBQ373qz7a9DsoevjuQnj3Dh929vrrHPeO4VDwULA08Of1VvKx7rT1P0gQ+Bs/rvLxr5ry/2RC+Ejx1vN4OzD0ZwCY+AitKveYs/b3Rbq09F5uKPcMhyz3dbC0+mw1hPbVq6Dwmgg09cYpNvaOE073OoZO+4N7uPdhvh73SL/e9","yHuwPeGlwTv3xM6916Fvvq08Tr0FExo+cMNEPHIprDtZ8CG+YPM4PIev8r10OM8++OA+PWFSv70h84A+dDSEPgY2s70rvBy8Z9IMPj06cz1pM8I9hbCLPcAZQz6+3Qm+A9yLvHV6D73BKKG9lk5bvt0XbT3ophy9g+odPCHvjj5SVYO82+aZvojhEz4le568siWkvaxQlDwk4sG9ysGbvZaDSr1HZGg90c8uvWmFHryxnvq8/ulgPY7j0L2Wqgu+791QvUXIJj4bkwG85VakvkAsUD3JT0i945iKPm7AJL7IPvy9hh59vnrks77htii+A7PpvcT3Fz6l9I++m7fyPZxzib59WWI8P/JBPS30Bz7v+hS+jq2gvgRBwj5MFws/yndCvU6Yoz4aWuu6VPuSPbfDML4koBU+P4eOPpVcpz6SCk8+DzZePuGIl77dWIQ9po1FPuHYxrw84yy+GAOoPc9aPj4Wtj8/TrMRPiW44j62jzM+4XGpPp3nej6EYGU++kxOPvZrED6tpzg+b0FSPIpylT4Aepk+N1cpPqaPqz4OQKG9F/8LPsPeVb4NKri8rEM0vEhIEL8Dodc96qypPSMlkz7zu4M+68v7PYrOBT6mgYA9dm3IvbR6jz5CxBo+6SK3Pl2JAT/mvHE9wuE/PspNXj7gL2+8ePcjPoqQ1T0C9eG86S0NPiekzbw89Y09J8dPPscm8bvdOyw9eQLxvgpY3z04GZQ+UPMCPBpQMT50MWQ+kT8CPcWeB77o0kI9HX1Ovu2FwD4+pqe98ydOvgAN6T3Ic5O+AnS6PZU3FT0GCiU+AE25PkQtAr4Rr3M+VtCLPlh5mb5sYMo9mr5tPX9giT1E1W68KGbCPWgOAz4wHq09E8iEvokgDj5td2c+fZ7DPU8A2T3jXIg+WVyQPcxUDr4lsf08ER6pvDozZz4SL3q9rP4WPm3igz1VAPE9t6xMvAt+3z0yBiw/PRZzPI+AkT6dsq48J+KdPiV+D725BJM9kVdWPksYWj1LLd8+","7c2kvWiV/7y+xg2+fAIQvg2LU77NWz+8pKEBPi7uo7xEIqG9qm2WvmGHfL55BNO+0hgkvgRSwb1Fo5w7csW3PYbDdT5fq0A+5KYdPftG973nxJy+1lMMvWicBj7zMKY9rgTRPc/XNr4jsbK+aFzcPH/a4zz8czq+TbYbvgUA+b3z3VM+RYgvPls6bD3JEAA9RHlFPPC0yj7xym++tWtwvpm4C772wp29TuWqumwSkb0Hh7m71QqCPXFNwL3xfac9cPxlPuSXkb3SDia+LzAYPRrNIL7Chyw8uqkbvryvRj1xdOQ9+mntvbkKcr7+laO9oi4cPRVQ2j2CuYO91uAxvSoUGj4td9W9+g/IvM5uJz4HJ1i9KUe3PT5WM752Wkg9oVMwPelDIb6YZK49nY6evl9PIr5DoNG88n0oPcDSqzz6cQG+w8r4vNH1Ab7uEG+92/DdPQbCh77J2Ny9pCT0vfOWbT3vJPe9XVOVvdv+Br4/w828MiLlvMNByLybI+69d1oZvijHHT0WFnY+CEWLPuCKdz58Ni87MJUZv0g5qj7trMY8KKKOveKB0jxFIoW+heUGPrF1ZT1DVoU+b8ZiPnRiRr7vvEW+oXmqPRL6Qj2M6U89nNqqvccTXr6kUCo+lSYnvq8ST74m2Di9SXRnPGdIar20q7S+KUFiPWCLrT3/R/e99HG6PkGfPD4g1m49j1O0Pn9kvb3az0c+hcaPvYH5DL6WpYA9JY15PnP8Vj4hkDo/vohOPuRRB73CpjE+Wl1lPeG8HL4+IMC+/UwWP7uNpL4UA6u+8laEPaVFlz1KYOI+FkN7PvrUW75tk5A+peXmPazdmL6nXxM+a2ABPV2iFb7n09G+0KkQPcXiCT4THIg9psUzPikRjb2Qpo088EB+viHHE75s+Vo+arq1PtDhBz6AKxQ/IzG9vsscVT9Wgoa9f1ySPTf9CL7WBV6+ewjKPdLIDr9Nxz++kJJjPmaSZL+HAzO9vZ+bPXOv5r7QVyK93gRoPrJHaT4RNDk/","m8AAvo4G6LzyyQs8Ae+0PIxkPr6MI0U+uxH/vRlhNz8I9ey9lM9ivdz01D3EueO9tQCsvpzbMz6rVkk+I3PIPQu4vT5J1dm+RN3Gva8KM76vz0o+U9/BPYMwBL7DAFu9onuKvgYeEb4myI0+IEimvilWzr3PZB698zE6vn2z2rz1cJW+mD9OvI+WjL3DNrE7SqgZvvoDU77UDqG9O+2pvHOq1745CJO+tBY8vcRTKD7CRQY+vvEHvjTXST4HX7K+R2GzvCQ2NL4Pk2q/S0LqvVlvbLyKm/K9dgxNPMER1b2GArG9wJ0lPWcvub4wgMi9y7OMPY/eJb4LDZm+wT0lPhy9Qj4Ctm88DvmZvJVbjT0FuEI/9SVlPWKw1D5LTjY+2bmCPS4bqb3tVa6+nSyvPWFnij4TElW7Xiflvd8X+r6IVM08sX0dv0WcK779Uco+VQ1yvbtLF74ZNRW+6x8yvxwxGbxodB4+7UfmPV+6lz3BbPg9kL38vV27lz6zqKk9OSOovbUl2bw3VPw9AV/DvNvH5zprRie+xiWIvm1LeL7eiX8++o38PjHngT4XF5C99ZdRvqRT7ryHOPi+amVmveM5L74J3c29ppXhPfRm4j1u1XQ+46SmvQFZ5r2lGHM+3ovHvEpvw72SxTI+fK6cvWPzjLuErbW9aZRZPtMshL09+3W+DdWXPgMRoDtczL+9IuR8vgZY2r3mvis/k1h+PpapUz4wLkY+VsxvPjp80T0Ofrg+RBAUPdPXLj4bkY8+6Iy7Ppewtr10aNw90xO2vY36kz3pUXM+N1pmvmNfDT54PUe+4EUKvnLVVz4W2Zs6jky+vUWAaL4g+20+C1sYPe1fuj16jEC9uWXOPp4xGr6RGKG76wAwPcZzYr64hGA+RTsQvrq4aT6Kwjw+MT2CveGhR72tV+y8F4HSvmQR7rwygEE+1tskPl6tlbtnZIe+sjzzPUi2Tz7OeR09oThuvuBwhr0LOaQ+IfQ/PrZ+MT36Thy+hmTYPZ2eCT4uS5k8","dzT2PSFBCL8ou8Y9lxUVP3NXnz2fLI++SvnMvcWr2LzbxRO/zvh7vp8uyD1wAi68NZYJPpMJcr6x3xS8KDysvWYaZ76vKSm+sQgKP612T76d4He+ZMZXPpF0zL0gaTO99zClvDNOib5FwoG97u0vvjmwoztyYaC+Bp5WPvN5w77Avda+kHhyPuf1iT56bwg+WGf8vUMG1D5ZqVQ9LTUvvrMtnb5Sm4G+YyS4uyRNcb4A0K890wpHvv1Rij5pNF+7+6gjPikWrb0mZtq+PM+1PlHrMb3py4q+AvCpOnXtgT5whSO+4fKAvuyhPT4SY0q9zBiOPvfoh723SI29fZlmvnJMNb4Y5RC+WIhzvanOl74Fj0C+BD1YvnYaiT2/MkC9hEh1vSCAar5A8W47m90FvlcSj74IiZ+9dsdYvlcHEb6WSua9ftuqPQBL677tkQO++zzVPXI5zb1A/RS9D/aBvvcyQz2Hr6i+j9VMvnAoAL+kDhO+RJxvPh9ror7aMpq9P45Gvj2g+b0hNOi+0KZ/PZh9Gr7m3Ka+NNg+PH+5x70HGHK+2up/vkTgCb6QZE++J6XXvXsSkbxsvLG+jBHUvrkR1r3pm42+A44XvhVENT3C6ea9UJORvfn5DL6mIK2+p3nLPUBe/778hYk81m1CvmKBNr7yBS+9UqgtvmF9ML4g5YO9IeFRvr8rFLzpynO+070OPqjYxj1/j6S8rllzPsBjEzxoeOe9wYqFvT5WxD27bw+9UifMvGzgLD/Bemu9N+jIPs+3HD5fO/A9gYkLvvWSMr69kSG/wTYePblqej2kxVS9r/1JPgnGwD1zeGY93GEMvUw7Xb50RJ+8boIAvmk4/T0VXSa+DB+rPMHJrD3Wy0E+aRnoPt5Hyj2kPrw9pqGuvR7wzDtsWu29gscGvd5DX70lQwi9RJcZPQ++cDuhXCs+wMd1vmaCgT2hN689SmKIPJVm9jwqmoE+32uVPibMe73qGXe+ealvvsC0jbz2+n++3Y8svqEpIz3vadK9","CPWrvQkqp71dP649F2TUPB4Xxj0Qwiu+OKgMvlR0grwCm8O9ycguPv5Kyz2sMHG+Hn3YvbbSmD2WWcG94HUEvm9lFz4u4J098rBuPoX2mj3aU4y+YPZVvKe9xj1dSNc903oevWO3L7061yk+7y2eveNDSLx6XNi+TbvSvRb/MT3IDwg+sH9NPi4oAr7WtEq+OVcGPjN4pzzqAgc/srISvbxUmT1rbIm8fRSpPRWuHb7Esji+3KkjvMfTbj34biE9akZxPW1scz2Ek0C9QSZIPbC7mz1GDFS9e3w0PmqtkT29+lG7ZCSgPEgavT2VUsS8aCYhO7RkEz/Y+yM+tvjivM9dxz0uhpW+5pkdP/6Isr1BPPA8PnMdvoss6L4NxgI9vqervYAzhz66yoS+4o3OvWXJWL8ShsM+Jw87vtN7yr7CvCO+JvamPvrp5j3snPe9FG2Vvtm1oLvm4369UOLIOsdfu7wddIK/oqQSviK8Q79CAC4+JJiVvmcVXr+tcbG+TFudvCAfMr+AQgG+RlN8vnKO3j2OwNy+9YLfvXsYrbzYHSW/t7c8vzL9Jr9zBk8+UJfoPp9frb5Xwp69OzkuvhbLQ75TmFS+qMRlvrm8ATzTCYy+6IuNPrIKfr4uQaY7q9JbvuTdlr6zgzw+woDovl0FGr6f1LK9VgAOvUVEo75w9pU9WpyGvv+Zq76PiY2+tYIovgHat74UpOq+tudPvnkNbL1mCim+j/mSvXFXRb7GDWq8DH8SPHqbFL8Xpgm/ZWggvjYOXr3KYR6+g/EZv9GoK74hTp6+A4Ntvl+Xnr38V0a9tygPviYwJr+xjqK++8hQPgdRub3Vn2O+AQBnvVDsI76SzJ2+XlGGPR+0j758ET++Ep6FPaAd5r3bBVK+QrOzvSg/Cb5JW3e+xfZNvk/ner3s6D++tHhUvRIcYL4m7e6+vMWjvWRYBb5djI09vkILvaRM7b0O0Ww9kBCTvs3tpDx3GN++N9PLPe+AlL5brpC+0Esxvue8xTzC0h++","cj6PvBfo4r04XZq+ZfNnvXMD2z1s5OA82pOQOxAqjz6ySii+tLtTvtT9mb3KtUA+GRaCPX0M/L3nOu09FeoxPoZyT77/HCa+k7EKPfW2I71DBFW+U8+0vrwSib2f3Ie9zjh/vuBqc71qY3Q9BxkBPrJIhj3C2QG/p4gPPjtUnL765p2+DMCiPoZBCj1blYo9leUWvuts8r047zQ+6kytvKjjxr3psoS+u7KNPJ0FVLwrl20+kUiEvSbVtT09HNS8a04BP0Gqqz2PCpw9K9Q8PuuIUL7JUOe9IOXfvRq3vb0qXK+9C1N+Psk2/b2GzKc9MKY4PYprST4vsKg9amrrvK+v0j37cHK++O8RPbeX3DyShkc8+gWbvf/QsL5ILVu9SePWPaXUnbzqtgU+qBtcPsiZHTuVfcO9RYMlvjN93j2n/0S+hPw0vuF9/j3YM209Xf+1PX/Htz0tq7C9mjfOPRNjVL0X+NE9QjNMvgyrOL62l18990OAPMp8sr0jhxu+foYgvZ7Gwj1jfT6/9oy/u8f7QDxCNVU7GwdcPqP9ir6MGOU9Qy9APamALr5gIh28DgFkPlEe7btuse88c2xiPp03l7xFn4I85odOPmpkxb39B/Q7IwvTveRf7r1PtGm+WKUevi1CMb7QRrU9C+ZnPl1PjT3KCL87P3TcPaZiHb5DkSW+yHb1vufH3r5FgbO+TA+6PagvfD2OXhW/XcqCvsrOkb59CWu9leK6vXIz57u1p9m+OQUwv6CmlL5TWrm+LBFDvZ6EBb9iOcK+ysfYPgh1i76yZhq/uazNPujhOr6bVFa9hx1Juzk8k777wqK+tJbrvax50L4hQZc848kKPHjbj741X1m/G6sJvwWCir4y7qS+UCIuvs3ET74rMOq9hqJevnX7Jb3Sx6a8cTRyvsnBH73zAoE9Mh0Uv80p3b2UlA6/BNgWvcVwET7XIii+Em5+vrhrmL6Nc+u9V2uCvtwPJb6dfqo9N/mrvv74yT7XGX2+3cPHPeY69L5YhEU9","VMHKPegowD6yZAA+XltOPv82NL7DWza9mU9vPjVdrT3FKV8+CUpTPnRbqT75sT4+F0c0PZHryT5TscQ+asMbvumZYz4Pzt49xP3rPPUkaj1/3E4+Lv8CPt0z8z2U+3E9jBSxvtlKej0Flck9aOQfO1H0pT43O40+BKAoveh6vr1rzSe9LJrWPh54rr4v8Ki9kZeFPd6wPL1EVu28DSGxPnoLST0VQ4C9kjIoPqwZRD6PcZ27tTUIPo21uD1dfae8XAzQPmlmkzxOBsG9JqqQPkvdJz4U4Cg9jaJzvkncpT5Uy5A9OZmOPfnkxTxQ12w+ocoZvHU1Kr7Yv04+jf87PfDUDT108RO9eLYvPtR/1j2TfZy9QEZIPdWPLL5uCA+9za/9ve2gGj3rGBs+lyUivQJa6z2INdM6YHQoPlYTdz5pusa+hUFEPvneGT59NGA9ym2gPOzBOb9zfeK8ibXlO9+0KT4+m0C8ba0APoIKvjdpz4U7neOtvvKQij31tcQ8nQOIPnjCc78621g+tQhOPhJzhT2C3bu+Qk07Psnr5j2wRfM827YIvniuL76RrQY+ZpyhPXo8izipbM68NBMGu+uU5r6ukrM+FG5GPgNKr71Tq9u9c5duPaSxtT18Db2+G8m9vbN1LL8rWHe9neCcPJApwb3l91y+pkSCPY+0cjzViRY+NcpRPIcFkz4mdws9j8fEPMvgAjsPfQA9tr3ZPSxgij0GSfU9XbuPOgDlZr7lTiK+le6GPCwsxz7NQS2+PeRCPaBcTz607my+r7RnvLCegL1DREI8T6wzvsgU5D1LXA0+Q6I4vS0tgD6sjBY9ZCr1Po6F1j4Qys++9MbdvD38EL7wKCC+NGMTvrMiDj6CyBI+t4eEvuSdBr5qDzA+TATFPubFyr0FZUs9d32+vRCS9j3LETy+y4XuvIJvOT4j3lc9QFAUPdM4ab5e5tC9DKKmPfKTDTyLZHU+H5ouvvHuFr1v5ao+sYwrvlqcWb76XYm970FUPUBPG72eIbk8","dYITvN+6rL5OHCw9hFTbvdpMWz7qkro+dP24Pmf4Xz5p8oC+VSQNPy4SAL312Fg97ErePeT02b0ttkw+v8rHPe2bID//zGi9DwVTPlBf+D6jDyY+ZawoPONcAj5ME4e9RBMNP30v/j0uJ1I+z/K4PjCDT74Vivi+9ksDv7Uy9L4bYp09KDxXPgivsL3NAVy+Tuv5vdI0srwqDcg+9AugPnQ+D77udPS9tUEFvlz8SL4yL7u+/1MTvy2OHD/UL8Y+KiaVPkU1z73X6G6+rZn/Pm9c17rYit28ivAQPoMw5D3bJEe8+O4Gu3/00TyBRhQ+gtfHPgFlHj9ivbm+C0CXvoTA+T2ER+C7rq83Pmrgmj53Tc48UCQIPzo3CT4OTY0+a3y1PdwYxjzqhDA97SJePnnkwj2pLyU+Pne4Pq8TtT5Anpo8pYtxPfSPBz75Tdg9OJjPPn/Wvz47swg9LDJDPqDxoT6b1Ok7kN/KPskGMD4VrY89tJtWPvMyKD4OJ7o9o513vaQ7ObzyKSk+qyy4PdJmjT145QU/cM8YPt6RCj7cVoc75ckePo5kWj6LMQ8+i7OGPsutK7xRHUI+OW5ePqW/Tz1iFj0+sWpwPb0AST6kWec8BCiFPp8djz7yJtA9qsOPPfrmdD5tAx2+FUc9PrjN1j2zo6M+WusuPi+qGD5h0uw9jyaGPSIUiD0OtrY9hukuvkvM1D2xtCc+rcdavKcfBT5VAJ88hU5QvV8iuT2qJos9OawEPhwYor20JP28/4wKPSsP0z1eiO+8iI8yvhWqHb4HZsG8XjcCPe7KDz4mSE0+699XPFSBBr7XLDC9t30RPi+f7j2kMU+9Xpj2O45OND6Lfja95yBevt8XAT6gcuk9/ksBvvlj1T3tgBq+WFQqPiy8Dj7zd829MuKjPcgrGj7jmJq8Tcm4vRAqB71AKAO+F0YfPspLjTzdrP69/6CaOwWYhLwkJh2+1qTYO7PyiL2wILq8YBRaPkIZRb0A+469oDjPu57strt94xG9","L1iHOwvYtT2uIi+9ftvLvFypAz5Z2rO9xyQFPv0whj4qP+w8G26EPXit1zwKI+W9Nwp6PdT2+r377p09MtD9PEQw0Lynv8c9tuAjvkz1g71IpAO93r3lPOcdPz3yWwA8wJTxPWOnvD1G+WG8IxlMvRZyDz3mDuO9fHANvR0YJD0rBM29LPFvPW3lBL1E0rU9rIG0PANo7L3mh8Q9BpckvSWXKjwM0Pe9DcHQPYEPwT0pnkA9qcewPHR4Gr4lKY++q6NOvVxEDz25SL2+Tb7gvC7rf76bUA4+aKI+PZML0L0iH3c+tCZhPc+eKr4Ehgk8UzOQurGMDj4tnHk9AA/PPdELVT2F+VI+t9MYPfCn9T2N9LQ9oc5fPnRtzj23ZRc+6XgSPmIjqDuGKgq+bvTbvcJKtT2WIkY+fE2lPkF88j7VLqA9IDBLPi6mgj0bse48rMe5vXaW0j6l5Ca9fskwvshdgT5XXRU+6FEyPiFIvj50Bka+6lj2PYLKQT5dE3I+x7uiPnm9Lz/rabg+ItWEPtyVtz11yhA++0tXPqF64T52iRc+MRogPPO9Rz1+Ug0+Wa1zvQprLz4+rPM+BXiGPpIE1z6D6/K8RjRdPEMkmD515A2+KLQyPndnjz6+rpc9aAs2vkzyOz9mSoE+x0CNPljdID8RmlO8fnSoPbsMpz5VK4a8ZU63PruGmT6gxYY+cTuGPmTRCD4BaQE/6NY4PuE+krxxmd89XmD6PUoDbT39PYU+PQWaPVaADj9YFBM+Wsk1Pj38UT7d5qI+a0+4PZiLnT5S24c+fjaNPaoqJj6vKgk+GPWiPZVKvz36MRs/BeFjPr5yxz5udog+ZkiTPmHKZD7jCKk+NJFpPt7lOz5ML6k+HKIKPmIdCD6IBFY+mnAKPhRMjD7t4EM+i2tyPodlgz4UzCo+qCGuPthouDxz+dw+vyQTPj+wG70vDc09MwerPeJDRz2UbmO9pnvtPhpOaz7XXFo+CXyWPf1/Fz8lkDQ+2gIwO7FpRT5MCIM+","3nKbPXSih72f2W4+6SrcvYg+Eb2Vyzm9/caVPaRd8z0yQn09oqYzPb7n7r3mZCW+nlGPPWCvgzo3W9C9cDkTPv5oDD+q7T4+mBiHvoYTST0zuj+8MgcqveraAT77tti8tindPM6Egr0p04K9s7FNvtuAHL3vbNI9LPI0vX9bmD17H2A85P5KvppSMLzaHwI9nXbuvbFhZb7YT7a+rkEPPRkUlzxTI4G99O5vvXN8Dr2W1W09rRVevQFqQr6Snqk7aUXbPYdpt70HO4m98RcgPnUyC72AQr88f2U0Pe70bTr5yjI9msIVvg0N3L0Kdo2+iu24Pfzhnr6lxUa9sZjLu3JJt7xSXzK828PAvB79Rr3vSay9iROHPdDShrulq1o+Dq+dPXMe0z191wA+egicvilcCD55Mve89VHdvX+d2j27hoa9y1CYvXjMxT3LpbA8Qn0NPpWsib393h4+2lpgPbZvU72xnpo9ikGvvazc7T1TZHS8LqRzvW/RcT49iIE94/UmvoPonT7r+mq9F3p6PABndT3qX54+g3t9PlDocD68/yM8bnjwvRF4yD0amSc83qUmvkC8gDxg7KO9DXkSvTEXubwYpLo9EfIrPbZ3cTvbjbE9+gJnPc/BTj0cBx6+BlJjPZOQqj0Fhjq76Df8vQ1MPr1BWE87jiiGvdYtyz3RaCg+U5MgPqF4CT+xQZk9iTJQPRoYqz6wLie7xAbJPmPclj5UhlA9mOEEvs2JJT7Jge+9QaRUP5i3Oj7vZAE+0UwbPuRIFz5HDwo+/ERIPaj4Yb7RFPE9aC1IPS2pljy0dIo9VpvlPgUa3D1XcYA+Qx3GPuBa1r1T/gc8GBSXPo5hVT3Pxg0+desHPj7C3z4lDwQ/x8dlPiZ29z3vTBw+HzEVviuIWr67VYS9E0wVPwdFDD84r7E+opIWPwGdAz3KOwY/1s+IPlc51z1hyC0+kledvht0aD9RcRA/iYLjO7vEhj75wbI+o4qbPiWJ6T2s/Qg/qvLvPT4vHzwnbX89","KGDGPIBDIj6dKYg9J2qCPShhWT61thU9CIhNvkI0D78kPE0+1vFCORJ0872EnwO+x6lsPV4Ikz3PnNs9COtNvlV76LyyaRI+XvO1vhgXu74OWAq9qs0xvqrGdz6jW749C8lPPsM3Bj9qzq4+ZHqCPizlez6CS3Q+3g4VPmP+Fj46ulE8j9yxPo02rj4OcaQ+i82EPnKtIT4cv6o9Ule8PTonoz4EFuI9MIyqPaDoU7x15E2+HjykPMG+XD4RPJs+G8UEPqgfUT6XG7g9HPTHPYZuc71pWeQ9H/B0vtNGhT4zvlU+rN1OPvG1ATwbeT6+mLxTvvyyvT33rCU+z9F/PlNB5z07P829K0cIvY3BSb63UKC+KSYiPuh/er406bQ++jKhvbC38b3zF409/YCjvRtTnT34mI29ME21PMuj8T2eGVW+9UCnPtAzZ72gq5K+YGTlPmIVZz1ugBG+Lb4OvgGz0r2Os2e9mmXOvKaRUb65wtQ+iMkyvswA9Lw+DKS9SmtQPCEB8zwSWcQ79affPU8WLL1HA10872PNvSI0Mb6ZLJS+FvJmvHpEvL2UplG+ffp9PkLhG75sN6I+jamMvsDAiL2OAcc9micEvlwnmLxMYRe9sj2dPbdxgz4vG5g8UUievdMfY74Zrw8+P7WiPhZHWT0Yex69EBjhPvzzAz6mF6U8OAawvrNrVL5er+M9Or5LPgedsr2Uzqg8vAhkPaDOuj2ADMG9y5ASvppOp7xQP8+9xkk4vuAvgj2WEaK+UaWUPA2P972ZBrG9EeUxPi0ilj42K/y+jwY5Ps+oy73nGUM9aKxfPItqcDvtTxy+xBwlvWn5oD6yV+O9N4djvYvDgD7Kmz49mu8hv7+/SzzJtO49062XPQGXH71+CVa90N2QPboaRr5w2qC9C1TSO6BDAr5syyY+J/aaPm5omzwytxK9/7mqvNx1ebwloyc9U9zRvA/+HL5HGsk9lMYFvvsHmj6EGVe+kHSfvcFxv73wzGA8PKQlvVRqFL4tPgM+","F1l2PZvRAr3xSRS/mUSOvrc4Sr7qsKO8GPArPm2dBr2xSVG+6p6oPl7IRT3RfYA+xChZPdbQ+r6WgrW8IgqGPWH1073PNgg94S8jvl2JhD64mwI+HXBhPeqK7z63ZGE+nEyEum0fAz/os9s+2ZCcPjkNqb1Bppk9rYXvPdMLwL1oCBO+M8+HPdgUAz77uz4+uCSGPnoJlb0pnF29AmMmPno90T4jLAk/HlDiPiyKtD3OWZO+4cCMvcZQE757ozY+TMeMvRtyVj62qEq979QNvtMzHT4cWnO+GXZ2PqeSM78kOB0+kLLzPqKMxj0HTAU/2yRPvu7mIL4Ggam+DLBvvkIu1D2bMeA9lUsRPskfxDsMTYE9rFMJPjQaoD5MIY6+leaSPsJUPT65EiQ++RfpPcQFWT6vuyU+QQRFPm4KRD5DJb+8ZMXlPdm69L4HFto+o/B/PKlPoT31v2s7P1YBPjlYRj7z9b8+fuwePq0xhz5T3Us8k3OUPXvHLD5Ro14+yAsUvmWJrT561Gc+qApePppMiT5dORM+PFHqvKokmT7drMw+HP0JPlYMGT6uR14+baXJvHuCbT2+AIw+uoRGPvcleD7jRUg+Yy22vdGHQz2VOTg+3awKPi1YhL8toLk+BmZBu6djmT7Brgg/+wfCPfy2RL6taPi9rXdMPW/93rw+iAW9WrsrvrHeUr7EMI29io4NvkK8O75TV1a+dX2MvqLtRD0QBmK+h0yHvMijK70GMIW+OoGCPZ5VxL3FhME92/zcPXe5+rsb3JE+jL1mPdGFgz07dr29XQo6vrqli72HhhC9fjKPvusDKr1iSmm+wzCOvQTFpb5qLr49QR+CvmB1BL+iZm07rsOivvVseb6rXiW90DuBvo0lHj5Bwxm+tTQGvt6vMj4YuZ29PP5pvUb/dj5CYLo8ia7TvvkaKj2AGVu/BH74vTvucD4q3Bu+ZaqdvKWKOb0jqUA9ihraPZOGGT7Ybwk9Ji+6vkHEYD61+p4+JDdOvuX0vD31C4w9","BWC2PInV8bxCWLi9NtW7vWOujLxXzJ49rE1xvIb7ir2V58a8/u7ZvQkInj5JwlU9VA47vmYuQz0TVVS9Sq3CPJQ4Ub7v6XQ9RcSovgLiYT6UuAy+tgvVvpm0HbxIPPG9YdvEvTZpPb5xER4+GcWJvY471rw5IEc+sFwxvq3IU76Rw/y8KreHvY3tVz0aso4+NvKUvVjEPT6Q3Eq85/z3PVeqibzBe5k9sPrKvGaoRrztTdm9Lh4iPqx1WrwvK3Y+pC0IvvbbUzz93IK/BsgGPrXimr67vR6+i48EPR7OTbwQAy+8YqF6vibWk77W/AM+VE00PAdFHL7cY4K+mLVWvbeVez1DJPc+NLKAvdZKeb7ctrY+Z6MPP+yuBz9Pvhy/LsrpvrboML4M29Y+DVf5PstTCT8Y+b68aEc3vurCHzz7H/Y9GaQ8vugkYL7tc4W/gm0Uv37Lkr7cjaC+XteyvnkhBL+TeXA/BqA6PnbjQD/E/BU/LhFjvvQMJj4gZoc9Ewx9vCbs0zuALLe6U9v+Pn62nz4hMK0+q6hQPdlqrb5Dswq9hMAqP3tqFD7Phlg+oAU2vrvUvD0ZuSs+HvrGPBPZuT2QpoY+sETEvY9uEb5cb2k/1FujPRfENj6wNtc8bGL5PRUOQT6Kp70+s3OxPjL5Gz73YiA9rQttvu+/Z7/hkNg95g6APQzCI75wNvk9MxwJPkVWQz5MZ3g7yDSLvdL8tD0+eIK9AD4HPmbMyzqOO/y9pTlwPchsiT6oNGI/NNGLvZEn1b2at7k91OmAPWQzEj+0B6k+wB17PJHGKD4KrAw7xnUNvc0A7r7P4Y4908uRPJNxlz1HAji+6ZtwPZ2F2r1ATus9sMiYPSgkzT0woq09wbb9PFYlyj37LCk+2yqGPQOOQrvlgBs9ZsrgvNfqPT4s4909utXtPlLKGr2N4tM9nsDsPB8uUz3nqKg9BTmBPhxJzb01i2k8vshbPS8RDb6N2EI+UIsYPuOPCD4UmUs9br9jPeI0ED56ry69","HDbDPQpxsTv0EnI+Q7RSvA2wFj2qHC++s/PvvLFbAb7ggnC93IUYPKCOLz17WbS9e4CXvRorGL7VuQM+vl9JvfHzB76/Azc9ioy4vQiB0D5xxqa8+eSzPU4tn7wBONm9+31CPsqykj4Po8i8AQ/0PZ87jb1i2ku+O7h1vVjpDj5o6xA+evcgvgapKj6ZaS4+PxtHPlagCj2tkVa+1rFfPZ3RKr7v5Ak8JQrtPN8QSz4aMoA9FynDu3SY27v77508AmX1vCt0aT5v3I69kOKRvlDRFTy2hD89TvqEPfpwDb6CfZG90llIPrqs/L08FdS9rKQVPnIZ/L0VcJQ8dZEtPstM1b2BWxE9GR9JPrAazbvxJi28iL+CPfCDXD5otQs/AJo0PT3KkT1LA+C9EQ5HPV+MzL6C1aK+poaFPj2hX70YfJm9K5cBvZVfXz1nRdk+iVANPupOvb2P6rm9X0QWvec3Rj3Mhou+YWiivtHHVDz0ZTA99bMfPu7dprxutao9kR5GvnI/A76OO409gN4ZPl14ez6lBnE+NecmPlKzJb2YLUc+e5xrPaG2hz4mwQ2+bnJ1vfplirx12RC+ZPXKvUAITz0pMY08Ab0mujzQZL61/wM+1CjOPWVdo74FmyG+4Hrpuxuc37xWk2K9SQjmvXK+0r4AgSS8Jnmbvfg3mb6IzbC8R3uMP26Nu73QDQ++H0GIPWBStT1c2Fu+RhK6Pm6+zLzYtVU+I0fZPK3VQz5mAnS9xlMWvXx//TsubU0+V+7mPlKM/L6s4Qw+D/isPtRwxj6Vr7g+0iGoPoSrED7i4ae9xOpMvbLfEz5I8LK9e5ILvoPYyr7PU7E8WTLePY89IDvz2kc88HSePYRkmT4/Mxi+UvECvokTxz50gLI95qTePGFn0j2iicO92ON1vWx+cr2fzFc+6FLRPpBE072R3wI/aeI0PdMUbz4/7Lg+XrMkPj2f/T1ZRuc9xm49PfOcbrzPMeI9t9qkPaCfIj55BYk+ilrQPlxSGz3MNn29","c17DPRE1N73Uap++9ByQvkPhYL5GUiu+fNxYvulCxb2UzYC+VcAtvkpEHL6GR9M8Ybx9vjzKE74/MpK+OwTbvs6Mn77OFkK+XcGNvhWsLr4VIA2+qT6Cvr5Bcz32EWy+3buWvSUw/r3SYBO/V7UrPVJwzL5MSxa7+Qbhvc3nybz/6T++jWVSvroLgbzNld69EmePvpS10ryh2Ru/SunKvoejMr6GLDK+BSjevbRUi74hkBW9UuMyvgwxNb7TE5y+wtCDvvlRKb4qoVi82UpkvKR1CL5TykS9Pw5SO5vCsL0F+VS+M7aevhuoEj1MGRm+hk/GvuQ6070fh4a+Js/dvZyqKD7M3Jw6N7+Mvuf1sL0GTwK9diQoPdZt2L36rbg+8O0evkwecT2ohZE92HHavNv+LT1qkzm8QuFlPdnc/r3Bsa08ZbyMPUPh6D2DQIO9Wr/NPGwxID1p5ei9w4KdPC+bSr3PbJU8NEIcPe7RxD5wcuG9ULDwvW+wdT7qPxe+7quhvEbg57nkuF2+MuMzPdGM+D2Kb48+iBhCPCn2iL6XUNm9xOuKvghH6z3zxui9xdmqvXNLgL75dPo9PWItPh71rz4qJQ8+hLu2vUbOEj67AB6+xrZhvSITXj2wI8M9+N/uvHxZmj3QfhO9T/WMvf8xAD52OoO9jR7LvAbY+L1+xtu9G1Inu81y3jzntB6+ZixNvaqSAr2ua4u9pY0IvqR4Dr6Ni5Q90za0PN/TZz5phAq+M6pMvf4zCD5GczS+chB8PWXmLr54d0G9a0SnPZNVyTwOW1Y9GJkEvsb3MT7W43O91YwOPeIsQr3v8tw9oRvWvTyaRLzpF6y9Pdo0vj2SsD3hXkI9lkEOv5Y+ML5kqWI9YwTXvc5KwL0sHAs+qnAnPRaFsLyTOCG+ZQd1PWzDlbxuovS7eAqyvf1jOr0O0hA+DTpKPSEgAz+/FCg+cE1WPILvE73g8p69rxFtvShu7b0eXRO+iE2iPYuHNz4i4S89G7+0u5mIQL6DXrW8","LafjvaR1lL4IuIa+Y3ODvtsOAj44abK+o8+IvpOv0779K9E9ActqPoGgyr0pTGu+u6AfvkSIKL7ycLw8QWAdvvw2Bb58hqG+ihQDv6Uk6z6dRHa/tve1vnqq4j7ujO+6eFezPvcO977uH7C+NFy3vcVKBb/+v7W+MzBEPpWpab5l57y+NWbVPP9/or5HK9y8Ua8yvdWMCr8K48U88duSvmx4v764fo++nSZKvmGAObyTwH4+Bz1xvo23nL5c5D0+RAsbvvXfAD5m4We9DuqPPNIhn72ewai9Z1M3vszzj74NmPe+5ujyvQxAOL1Gj6a+39HZvq4hPr2w2Yq85ya1PjLCJD3bNpo8PoqNPA+vG77xhKy+O2xbvuhHEL8+DtG9oMcgvipVir7At5q9/qNbvoEP7rzCrP+9Efu7vtKnx77OtDy+ShWyvdv6kb04FpG+xItLvpe3N76/gNa+jAhYvrnIhr7kE3y+TpgGv4QDBb5sjV09EayCvvnrEL7cQjQ9Ga8dvnwGMb4wYW2+mzVFPpeQID4G8ZU8RM7yveuPIr5Gd0e9no6UvRXG/7xWr2m6ttIuvoUeMb1LL8e8u1KRu1Cf5L3Rf8c90ASDPGFOxbynmiG+7J0uvsvURr5f/O69BpQYvlG2RD01v5e9FfNQv+ekr76SU0O+SYvJPfxxmL5imkK8FW/+vbxRqb23PcW9KuuyPcrMrz1EpZy9J3oBPkaNCDqCYOI9lLXvvb/etL0N27W9DVduvqgRRj2JVEW90q4XvhSq+b30vOO8eKydPbXa972vxpu9JGoHvP4wJjvcX6G9jvQyvIfTJD5zDTm9oSlwvYdNy7zC3eu7vJ1fPZOFCj6a6/w9ekcbvkhKhr1g8Sq9Jg/2vSg4Eb6H1Yi8/xnrvHTTg74UqSy+GHShvQkd6D06EZC9FRf9vfg5sT0S4PY8dbJTPvL3iT0PLNQ9sU5Evmljjz0YXgG+PbOePAqfmL1oERY+mBKjPaNbfT4TnAu8ijUHPXDb873qfyu+","YdqzPQN6tb1FvC48I7xzPYRACD0ZWSE9oKSIvf+AXr6dfjo8/GaIPRhpU75LR0I++BQ9Pg8E872A8Vc+iidovalzAL7kWxC+fpzRPF5dYT7o6dW7U3AYPv0isL2s9i6+I2z2vXxHkr2QuSM+vLIjvh74sr1MygS+ZmJEvp4pcb40mHA9wMgHO5qNyr4bQPY9vLaaPab5wT2XOSM+5Uh9vVbhKz39vGc9ARwCvQUJXT3SsCQ95y2gvUoUtL2aMLU9sHsnvTUCHL5Qp489xtjSvcEqdj1FkJW9euMivv+1rT2uAnW9IGubvgZiJD5a2kq+kKUZPfNjkLz0bxc+P6ZVvc6xoL52N5k94MxJvpykob79RYU7sf66vqbUrL4g02g+vXlUvS8BuD2XxBI+9gmuvbWw3ryIpcO+TWyBvd2SxL5XxVU5WQlvvthoKL63BEY+mAIbvexPmb4EG+s9ohAVPHPNnr2pPJe9/BfIvbHaH7xaQY6+KiBUvqK1Prs7ylS+Lh2IvlLnrD0hBsu+WuvCvrigtr497rC+w8hJvoBziL78UtI8ajPvvVGXBD1MI8G+w+gSvk/sQb4XGIi+hr4MvqVrG7/jqr09piqLO4beQL0X4768GlwlPRpj2b22T9q+GfpYvs04nL2nfg27/5BivZj6176usUW+bOmPvPexI77eqJm95VvYPV4wqr28SnI+F/y3PNPXGz7aXEO9qCDYPpMz2z2sfsy8UoLIvE1VHj542V29WFvovIgmqj6kloo9cqdTPvH5rTsMRiM9+oWvvay7dT4cy+o9hf4XPZbPzTzt4Wc8CfkKPhp/8j7LjB08VvwLPmJ2GT7DLge+65lwPsvoBL5J/YQ9lMwKPr/ESLznICa9NssEvly0Dj6B57Y9f34LvjFlqTzudN89FK8SPkaFDLzLOUU+7OiqPmRtEj3eAjk70ECSuqXjaz3NMRk+kqT+PXHtLb06zrw7QsPUPdNISrrQC529+aOAvQr3hrw8IaO8NtR9vRiB0T0IuKY9","h8cUPjS6lz74+sm9fIpkOwW0Bb7tqR8+KgECPu6dOj5yThq+53EuPeqV4LvTkXo9x4Enu0LziD6DN5U+Ez2avo6PbbzIX5U9mtKPPZGRpL5MIQ29nLxLvdejYz7NfVs+4rb8PbD8R71qyLG8NH/uvh/ag70JdDO95pqkPdRt+z2+Hos+uPhAvZttiL3Ul0A+VFkdvlhec71bGG++SrkOvhFwjT7Uzsc9aa7BPcd5SD+OJZc9TGgLvqPPEL8oBTi+Jw1FPr0wQr1a3cc9IeP7PWhdWj7Yl5E9dTctvo8IFT5FIXu8DXd3vr7FFr2WJI08tTIPPl/Dpr3fG4e871H5vTFmi75ayY29YuENvshcLT2E6+88fx7aPFKqJD9BAh48NQ9oPfNDhb54nyC+4CsPvr/xiL4i/hM/FeOVPtRn5bwyvn0+5ASGPs5YfT2qAdw97ys7vXcxJL6wBFi9fn+pPlDClT7kCbS8+l/9PBdhRj7Z7E4+pU2su/yj1j3bTDI+iB0Zvtquor2FIuU9UNTPveE41TxN5Ks9VoDHO/ux0TwLnfs9hdqWPUS7TL2cTDe+aVhDvS1tez1z2fO+if0uvKWFaLtZQYo8ueJqvtB5Rr5p9b+8uerRPeUg+D04UXy9oBWzPWgipD5JAY89cbutvTObib7F/dY9pv4kPpw1yD4ZiwA+19yKPaB5yT2bqPW8owA8vnY1eTxRAh+8JRifvUDTob3A6S0+J+VGPi+GUz609YC9nnphviz3ED/NoWY+gzHjPiPMyLyWLvE9Nx74PmeYl7tq8I89cuscPJZ5J76tV7w+WjZsvSkNyj2CguK9W03avU5OYb5Ifba9pxVxvk2Q371vKKq97o5RPl1s8jz4Xq48NwuQvOiraj72fQE+AhsJPqGnJT4ktVk+I7IJPmPWNr7VPlg+xtGjPnTDajyvi2U9h0GePEL5Ir55ghg/5qSpPhLGrD0D54U+VPK/vV5AXD23q6Q8igKsPuXRTD5I46Y+s5OfPgH8ir0Emha/","ZmH3PTbZdj5DZR8+o2nkPU9vBbzOBuw9ur6OPVPhwT3KnSQ+sBpAPrYgDT6V02Y+7txGPmmCnj6YqFw6aK7hPrukHD7JU0s+wdKLPkGkRb75ouM9r649PgufY7v0OTs+PHRAPp0GPr1Goj0/sdzIvMPgAz67lJ49Kr2DPX/HpLwvjQg+ZLqtPYsstj6Tj3w+P3sTP7ewaj3Sn4Q/t/PsPWM5Gb6TkSk+bRNmPpm4Lz43y2S9EvhFPY3oBz3W9JE+evamPcw3vT3v1jU9/JwDP96gs7vbFzs+xtqpPWZ5bj7T8xo+8QGzPo9QR70DsA6+SKK1PXIW/T4A08g+CkDAPnD9bb3u+B4+6+G6PS5t5D21bws9Qq/fvWBjhb7prIG+PnpnvMcQYz7YEWc9sm7ROynGhLwT2Qk+fgaAvrbDE74O9SU+GiluvQHoUL7bSZO+DbJFPURqUz5umoe99EytPUxm5j2t7u49FNwQvlg4mL6HeaY+Lc9AvsQpjb3UkKK9wPKpvkByWz3pKt++LqUvvVV85bwNWkg+z4fFPVrfOD1U3QY+jAMhPuRS5D0NSyc+AMaePqyLnr1DEBW++SMuPsKLmr48zhm9r7TuPY52Kz6HZMY97xAdPk5efz4JDmC84/oQvqOFIb6Rcvk+bYtPvluvlzyZNnY+gepVvRFVyLskQie+ko0pPahQzj0Rt9q9M28TvU8KU7yBH6k+2gI8PmEhZT2NMUk+NFCFPfTL971+tdY9+tYiPUcIdT41Eh8+uXOvvrWaVT1NQ+89sDcTPnZ5Mr6ghsS9beEOPtU2gj4FItw9hqvsPRBOez1xf4u9CwWgPQAErL3CnlM9XhekPkffqr3iBOQ9F4aFPhRhDT6d27S79f6mvQwlijxY7iG+BMgEvgJoCb4+wQq+Rb0fPpk5A7/oLwQ+oIs+PSHcnb5Kaee8A71avUmsmr0Rb4c9+8DavYtUhryiVCs+t4qCvmN7Fj7Fico9PoAevqx8Rj64Sk4+qAHwPUMdjj2YQWE9","t2YCvUDryT7E3tw9obNOPeE1tz41mzY+kYIoPonoHL6N4vE+chEXv+qMiD4jNdS+JmsKP+gCyD48tmk+wNsuPh9jC74H3Ys+IgePPrDszb6ro9o+nw+tPjPhJL3HPG++n0JLPfQjIj/p9kY9sR1HPozTib4Z06k+gGuvPZwE5z1QUxg+0ir0PqMroT4+OLw+VtmrvoINPD/PsoA913oSPSjXNj749mI93eOZPaPHcr29PyC+Z5K8PfZHIT+PthI9YEL/Plpj2D6PUpK+rqSuPmayBb5oZLQ8+mjhPcq+0b1Pj4S8U8PsvRluLD41Pd0+/Ec0Pvnkw73Kqne+qDjpvTWXXz6qo/a9rrwRvlczYD0oJEm+n/YOvgldgb7Xnrg+Y4b8vAxyKD5KhGs+nGY8vs5ceL6r9iM+0lWCPceMuz7ZwBQ+BP3UvV4Roz45NZ65XCuFvDsWmLwn0wM8IsAZvgpmer6d1Ac+n/XQPi/WSz0fzxa+SUSoviGy0Lzh9si+QFscPi8W9LyTPIc9VbtiPNylQj4eqCU+sMrAPTtyNL3Sjeu9ziaEvp0OAr4wx909yK6KveqNjr5PYOW9KqXsvSg2Hr7/0lc+EBAnPmNzhD57LTM9DEVEPWmgGD7D4TU+GlNwPgAOXb9jFQG9dT1LvvGaaj4F85A+Rfw1vrnbEz7j/jq9BkJEPYHd/T1n6xq7Ni6KvTpiTz4W4Aq8kSUdPvOhiDz/rPI9DrkMPosfmz4kMCK+0GOrvI89mj7sayc/50GlPqWFHT5I7M0+Rg2nvou2kjzidAm9GTWDvmka3D2gN3Y9IXILv55ZIj54D4K+iXCoPmSRnjzpjmO9DHwnPv5ygD6sNMk+agv/PHNr0j0Ajte9WK6QvshoeT7mRp0+zW7OvIz30j4mvwA+2yAuPpIdPj59YDg+0P0hPTR1bb50vEi+4JI2PaalFj4Yjbc8WAhIvesrDL3U0mA+IgDGvmHNDL2YhyG+HHDtPDkORj2Euyc/0FQ4PkujkT4qRco+","n5jgvQVIzj1Fk+88eDiJPfXd6DwwO3a90LdEPpIKbj3A8Ds+BdL8Pbr+urybejw+MG1IPmvZjj7LUIY+ORoFPxQ5Hj2ar2U961XpPhUpsr7I9iq/1T/TPUc7aD59BdM9yh4xPrs/Ij8ot/W8G+zdvWVVNz4EUbC+P9mCPhUb2D0LaZY9ogLvPborMDvRpI690i5bPg7QyTyx0ge+/FeJPdFqkb1OocA9Fw2ZvbK6kz21ZOE8ujRTPSFSqb49cTG+ZYN2viPafb7M9Ae+58ywvWG+SD6IUjE87de0PphVlD16bKC8tqZyPo89sj3VWXe93CGgvuF4JL05QRC81e3JvptkdT2eRfG9AzOAPfGjCD+7ccy85gCfvRNA9L0BXWQ+4gogPaZ2j76uuvq+JtEkvcfYyD2Zp8g9kyWKvl9Qbr4KDCg/h/IJu64bqjsexGe9v/fJPvkhJb6tv4u+fHQdPhFfWL5sPgW+U/6UPg6RMj5KHiA/y6q1vv1kObwIMAK+bR6vvRMSez5wsvU+bySEP27G2T43n0K+gpAGPyAaSz4Hf4g9s3kMP8vknr6x/JE/LwMoP66kJT9Fo9g7YLJpvvqPKb8wuQG+UuosP//byT6qmAs/3AvivQJzUb5C7xY9ZdKmPh2oeb6iRSg/9HMXPkYFHr03xrU+FIt3PdybMb+KMC6+UWlLPc8pNb62rzM+RJu8vk3vBj1feFO97/QaP5jrOL7sxMu9qmyGPQjrsr0WzcO+GcaovX167z3eRrY9TkQgPjc8RDlDxJw9BbxoPXmqZT7g2JU+TecWvi8OB749d/G9qaywPBmb5j2P246+d5oHvufbhT5jHw++YhYtveNmdb37v2s98O9ePUEoKb1SMPS+GV3rvPvsI75sg0e8yIs/vkUEKL63k5W9intuvcFoRj5JAQy+IMDEvWBInb51XiU+MN65vOUppb5yXr+9FyTQvRfJkL3sIfy9PvcQvo77IL5Jzhe87mNFvirhHL2ooko9TjunPou+AL6wCxm+","5PJEPOJYlzwjEPU9cAQovi4CJT4Yo9o+8vSRPmFJHj4KE0Q+0gaDvvh1y74dt789fDsJPoDHjD6/fQQ9+bjzvXVhaD7cEAU+w5Y3Pdu0Hj5yrRw+hoOTPTFV4z63UDY+H5YMvlSM2D5M1yc+pmSnve06Yb10mgi+VUjAPkw7pbv0kvm904/IvWeeWb4pJwY+u/2iPj1zCD6gyZ2+03thPZIpdj60jKc+qGXuvDz9tD2zcSm+3jUFvlo2ab37CYG9mO+HvhzM673OUTy9q17xPbmj1z7WPVs+8voRPmI4cz4cSr0+z/7bvaxyQD5JoUa9ydYpviQtZr2VwiG+3LCpPXt1l76GJBA+pIkcPgNAizumX2O+qSZovic4zj4gmaM+0JjIPclxhL7tcrU9RMqsPDUaaT4rPdM9CtxJPdhcXj3uKuE+Be/GPaK4pD0DcXs9y9wgPm4SZT79lic8RSctP1FFtr3jFPY8tivYvVXMkT2Ncru8WQ3iPVCcfD44Ets9dKoDvpqAlT2V4AM/xMYBvmvQCL64Upq94CQJvhfuEz5apom9L3ztPT/jlD7SaZm+PAKYPbJjIr7gKPS+cl5rvqdM3L5vHTg+4yt9vjF9qL5hpIo+pnzFPaLfHj3inZm+yV7zvd5anz6EDS+9E/6yPnJ9uzz/Tl0+vNGtvXC5XT6Mcq893xhCvgq4Sj6QoGw/hMjFPsv8Gb/9c6C+mvaeO1gr+L4SyoS9mx3/vS1PAb5zO3U+iQ8qvrR4mD7mHhm9DnjHPCi4Oj0x/0E/W5AnvlVnkr7Dezs/q+x+vhf3mb61zMo+opmEvjeZI71bT/i9Wl/CvX+hCT57ohA+TSuqPnPHEr9Pr1I+CE9LPkGDL76cnDK+5TaOPuTdGT4nLd49yn6evtRBkr6v63M8DxSUPOB1mT7hrTm+0FnDPrTYI74NKL4+yvOCPKroVb5YlkM+cHbPvqB1QL4rD7M9B2+AP1uuN70Rfl+9KQjtPtdpgb3+gcs9UfVSPu57Zb1789O9","r9fGvYiUSb06f9C9CxMkvRmaZr5Fv8U92k+qvvzW1D4duZ29qXKHvvMAYj05og29O4+NvsTInL1tyRQ9q8IWvqnHTz3J8Qm+FFLJvu8tnr04BlU8bTYLPZ81971i7EW+RS7fvrXpi76VeoQ8c6t+vh3Nc77GDHW+ZtBDvh+/Cb5qNt292s+tvVIQCb9MOFe+JMqkvnbGlL5se9o9gl+yvcqFvb7ZgTu+kASevgaHsD2rcmy9UHXBvudACr64DYq+pMJfvpa9lr45SAC8Ftw6Pj7lJT0E+8C9tJmhPond9L3NiiQ+RlBwvkQNcr5A7vG+zLPFPCPCUj97ZZy+i3DovZRaDDwKjSU+Zwc1vStf67zb10I+JoHCvQaEYT6GWQi+DhkXPcm8qT6xNqC9uhZnPPXpwT2JY808FyoYPoHX8j005CG9rhsFPLNxQD2HXV69luLBvbUPJruoIbg8NSXLPbb8VL0WkUY+kRm7PYXdWT4r0m68CbOXPcZ7TD4tfQI+TQd2vgC4q70naAU/QAjZvYYhET4iBjc+x0Q1PawoIr68srE+E+EFPtcO7z3Uf+k9MtDNu+lgIj5StIs9QveZunCwcr3Oxx4+KwcFPtcQvrwWbNk875mDvRcP9Dx2vP29GrktvW6Spr11v1O+tQZ9PhtZgT2RYhc+tJezvdMHvD2dHAe9KlB/PtRoQb330p++3+PjPCyVBD1W7AS86urFvQ0G8bwC0dw9impOvviY3j2XbCq9LBknvSpsgb3ONRo+rPiAPjwyET0tv8s90jWuvLmvGb5Ttgc+imp7vQ5Gz7zre8U9ew9KvacXl73+jSU+QUsZu/bx7r6c2dQ9br7PPUxtzz1uRPA9hwGhPTO1Z76+XDo+UvXXvWfmDL4Prdi9atxYvWD92LudbHE9bEkGvqBb470O4k482TWQvJbLHT1P1aQ+AOERvgu0NL4pRw2+CVvOPVkK2D2ofOQ81gShPZeTZbui6WY+v2HpvAk0iT5ovzg9ENEev77M/zz/9Qy+","FcidPcL8+L5ipxm6L0HJvIaOQz658wg/PpqevSzGlr4qvZq+d7yKvo5Uf779qze+RSIhvknqMz7kQo09kwQpvvfmuj3hcwk/SMeEPkNPPb/JBq2+TKazPtrjND4XB2q+Vt8SvF9rYb73TcS+vJk4vqL6GzwQzQE+/FzSvgC9Z76ZJLw9U8GRPUlZgbni9tm8m70Fv4m1vr21Qjw+goOYviKQbb2+yEm+8uKgvhZdw72SOA68cp74PjfyAb5HkFK6M6C4vnqHRL6ZYnk9UoZ2PmVGIj4BYE69nnuqvu3xoz3zLhq+dOilvjr6iL7F/TO+IdLUvXQpTT5unsS9hu2Hvhmluz1jIR69Yb/+PcZHKj6IMD8+cmCUPoK3ND4GbfW767UEPatF0z0PPRg+cEW5Otf73zx0il89GODbPqWkBj/kN2E+OcI7Poyhljy94VI+Z7P/PYHUsz6RW0I+Cp17PvpylD2OdPA9a4jlPkC6ZD1YagE+RLyFPn1mMD2SieO9cSzSPNOPYj3Cr9w89BL/PTuRiz6tBB4+Vo1rPhssJD69bRQ+hoE2PnVCBT5FfmY92BiLPUnBib3N9DE+TdLYPU+w7j2Zk2s88eJHvYT7MD5TkzA+DLnKPIMKcz105Mc9auwtPvdG/z2gEvK7SwpavCtnjT7pCqY+mOZHPgW8ID4RfMc9Wa9YPqwUcz70F9G9ZptavMLPe7z8tFu+BoljvedAw7ywtSw9vZsOvsMNeLyciO+9tvZ/Pvx6lr1wrFI8/IwsPU/zAj6rYVY9NmZBvXeroryO+bk8czwIvQg/Cj5SwHC9UR8SPVRN0D1iTYK9BIe3PfQye7zc7C09qr+VPDYsgD1yOZA7C/eGvaEDsT027Ew7ymdIPtFI+r3DpQk+em0FPXgWuD1pgRE+PU9WPbrUFj5pSbK8iBi+veGTIT7GZ1e+AsjBPZetGz2meSa+BTtuPqYktj1h38g9HXjDvRydpb2CFN29RHlXPmPWJ72Y6Q2+bEOxPeJCx71SXyC7","6zLJPR/mgz7IWe+93Y9wPYvdEb6QioC8K7SwPefrez542vA9W8RyPap8d71oc4U9hjjmPQwBf75iL8y8O30PPR9sCb2Xu4g9q+85vrZtYj1gY0s9jM1svUbHFr3tlC0+qnwnvumbDb7SDxe7X6MjvUhkg70SLnI8e2iaPjtSnz61jGc9X9JOvRn1jD7Fqe09+I8HPjiHRj7xSOE8fS8yPJx9I779XLK9YCp1usmiNzwImgO+CtRFvTESHb5xhoG+eMGFvYJ6sT1UAIS9Xuw2vkAUuT2J5OY8dYGVPZTBZD2efog9Wq8EPlNN1rvU2Ki9MJjSvaFyqL2XAVu9sWAuPinZjj19aRw/aKxoPqM0Kj7bMAo+Ra7TPU7SoD3IkXw+BOsfPrActD2gMVy73pEIvhfvuD6v2mw+3OgbPuO4iz7de3g8+dU0PuHM4D4XloQ9eqd5Phd9Ij83raK8T+MCPkFqqj3y8Yc+jzg3O19xhj6FEf89uXeoPhSFdryJ5bU+CBOBPqE9lD7UdnM+HuCuPj13dT7DMWk+vGtPPNmQlz500Ua9/qrRPYb9L7xbU9c9IqGLPW9qMD1dmdw+hO9KPs2sMj7TDZE9bG4iPieTLz6lAxE+HWsAPp6GkT5Y7ZM+7XKvvD25CD49DA69DiCGPQ9N0D4RE/c9rWSIPbJ2NT64N2k6asPHPECYRb5cZ829ePcavcphRL587BQ9qJ08viqE771EHhu+tFdivs6tlDtqtRS+5Q/wvQL0Z76vUOG+axfYvhCwK7zoury+23IYvm7k/r1hh9K+ou0zvjqtVL74kmK+nL40vngQJL+xGmE8vbkHvcIsQbwwYou+XKQ8voNjOb5M0cq9fgCnPQSgob1jf0i+jP+wvmXQSr466qe+WHv3PNCa473ZMDy+y3YevbYIkb16TyS+xu8jvozIjL7IqZG+tIjevjTaaL07ILG9XE2NvDJokr2uEoG+SzvuvTJmDr4EJ2G9VY0Jvv6SuL0QR2a+s8pavVgLPD1Qewu9","kMULvArMWr3buUa+FHikPc/Aij04QZG8eLrRvPmfEz9vwoG9rsSVvbD5F74VTWg7FeofvuxyXL3ODOw9zVZ4vVnpmb3yTfO9uKjlPCZ/dT1IwCO8q+fAPaRsRbzrEdm9zcbjvUCA0LznrN+8RsGWPmE6Kj6eeKa++jLPPRZNAb3yBIc9khW1Plp/Eb7BvrQ9/AT5veFVlzw4LIo8bJIlvloyuj32h0q+3LEKPoApLr7usSm8S88IvpDsiL17oZ09RwF+PoN6aD7eE4i96Ni4PFJeFr4+0869SZwiPosWcz1LFN29bnjQPaQYu7hl+yw9uQ07vQXjHT0N5aG9WMTTvThR3L1zzzm+VS3NvW/JTj2Tt+C8Z6eBvR0yqb6W5VC9qeUlPQa7mz1vxVg8BlfTPbiGIr2ae7C8N6ESvorlU75nKAQ+9TQrO0i6AT2N7jA+YNrcPXZlFz6BVgI945YnPOictD3l36a9YFoZPh8ykLzfXfK9g7WVvTdE1b0w4YK9Aq8dPvOpnz0LncK+f3Uavj7FIL0MHUE8qO7uvYFwkLt67tK7qOWava4mJr1m7ue6sT67PZxou70YoRA+fqpaPq5Dv71crQS+gDZIPqGJeD3BOoA9bokQvgswg711bAe++udtvifyTb7CHrY9vCoUvjX4TD3ZjRo+bUWUvb8UJL2n4OG+f9SBvunTLb69F8q+4HwhvnqKgb59ak2+N80nvj9+ar3f2Js+0VJbPXZWAL3gcQK/3/lTvY23E77D0C++fcfbvT1BPL4Dyyy+1SO9veOJgL4jyxe/48ZKPhfp2D0GQVi+Lb2ZPa5kcb67qvY8m22Avm6omL6yfxC9NQNQvqMTpL5d//e++zehvrHYIL4yRqa+uSMAv9AYPL2CefW9mfQ+PWzEJTwYT3C928qUvYpLGT7LyqS+E4oTvwRWJb0dm5q+BwIsvmT2n7zGspa+wvAOPo8r4zw9Izm+dKP7vkaio75poz2+4hx7vuOZr755f5G+x0yJPRdmS76544C+","EKJFPRRGIz7iZrK9BNp4PhFa97wP9zI+4BwfPuUkEz5okS8+ZkgQPtJ0Fj2Lbhw+iFVwvQTpoz6D5N0+2WBmvq+oOD201G+8+OP3PZeoLb3Tavc9f51ePgqNjD7u4we9WVF0vQgteD5Z8Ru+Sf/PvOfZoj2bFJ4+BSZWPpfVJD3MALe9KyDWPsgt1z0ImBU+ulKYPmqflT5qLAE+6FHMPjbsujuq8Zs+JBUfPScBwz4RDQk+NiO7PX8mjT1AccY9paphvSv+Ar0kMIa8LrMVvSIYJzuIkpU9m5JPvjHvoj3LV709TvmKPnRGXD5aliE+v6RwPazE0DxTLmS6TIUiPj4qGb22g2u9GrsSPedK6z0GXiE+J+FBvW3M+r18Do29HBBzPbRssz2Ymnm9vC0bvbzFFj6EU5S+p60Fvkc4aL6Kn0+9jxYRvrL8k728ZyQ5guEDPY9PuTyOo4g78oWWvaGscz1zkEm9N+K5OzaCUD9N+yO+sR8svnTwiD5HAgm8XJBbPUHBbL54Lp+6BVMrPo3DF74/Ll2991elvlwcq7yFOlw+cGyovVEPTb3vmYa93lLUPoBd0D3Rkio9K2IwPvQ4Wb7w1ok+zd6BvvHU/L2nm7I9YieaPQvvQz4XIH6+hfL7u8ZuhT5wGbw9vCb9PfhKxT3g5yS+TRv8u6hUXrxX0Bo+0z5Ovkbs+D1GmCm9v5EyPrfKBTz/0CG6BaYrvWNoiDxxytA8xVQlvrTGgz0e+im9AynCvkrGmr59MaO6jPWEPWhyKj6nQoO+LE0FPvpNCb2LQgK+R9/kvNByBD40UwM+T97BvTfImb58boW/BP4Svn+eM70X6+m+YYNGvBsI4LzfXjq+bcWvvStxCL7zQH09iOXMPePU67wRlTE+ERkKP2MXLD30OHg9e4bLPTZJab5V8RC+W6UMPFEB5D7eKne9jxxLPrOsIz6/I8u8MTRDvorh+z0Ty0y9pcEUvRI4frzXqYu+5YEmvs05uD35roQ806aHPjGRlz2xJg8+","HpuBPoSRNb9JKn6+/RgePkIXWr3IuP0++xr2Pd0+uT4E5R6+XePbPvqysz4lBSw+6v7dPVyxGL48j/w9hpGKPamywD5kewe/upYmv/LymT6H75a9Ry6LPma1gD6oWPg9fGKoPT5wrj5agaE9ewUXPujtFT5NzKK+DtBEvcPAF7/PU02+BQMWPnC9Fr+DC5k9x1YHvjcTMT5IrU4+idYDPphYEj4moME+1iyUPUa/m7wP27S+G5zQPgQtAT4hGa09IoZ0PWRC9jwn21C+wYjAPkUTMz9MasI9h8qZviJ0Cr+4sdQ9x4hYPlcCxb13WtY+957zPjF4AD8XqMy9lTH+vctcTT1lXo6+ItKgvrSxmb5eSDu+SQhePdUzRb4OKQK9mOnSvYw1b775BpS9cTizvbrjd76yN2y9LnHgO9Kcyr1pqsm8pwervbj2ab523le+37jLvo9UHL4aLaK90wcRvt4Ihb2PwLG9ZQqOviAzQr5XmtK+rH6BvrjgU75K2S+9cdtbvvh7R77fhEa+rxu3O3uGdb75xHm+h7YAvo69Eb7vEiy+xo6kvQY0Fb4CrSs+ET9EvjHa8710Efe+OeokvhkPcDxC+DS+//UWvsDEuLxj9bE929RyvcTHtT0l5ju++qCZvoTcnL5KrCe+9vBHvqI3Dr4Flz89fvrNvaLz1r0emQQ9ofV7PTbzD72nD287GMukPcSwvr3A2+E9mZL6PXdv3b27fni9ZrADvt+/rb0FBpW+zsgNvr1cZD6tI049SnE8PtMpCL4IJBq9JHXoPTbah71RqjG+HourvG6DtLz6u9o9sAw4Ptwa/T1+YYi980LGvWjrib4Iu4Y9XkC+PA6whz3T7JA9eHqwPKyyeLwalXk8+DwRPh3cRT5JBCW+Yr8bPqyo3LwvHZu9Lp26PVftsb0WW4g7lnzavAWGjr7s09c93UWCPcgLyz07xaa9nyTwvJhqT7y+KSW8Hii2vQYL+bz0IZc92f0IPeIltr30gwY+rh3OPWj6RL4X9bc9","P37rPa2Dhz3RAwg9JJktvX8XMr1WEns9/J4bvhjCUz2+hY09hehQvP7MBT4cZaQ9aQIIv4QbQL33JtU7+KgMvpfIUjyiqyC9D1hTPolVAj5SNme8WFDzPQ7evz0BhbY9cVzfvSQc0D30fVw97b/Cu2AY4T1TlsC9+HTgPDK9Vr7E+gi+OgVivYQuErhzv12+cvYPvr1IsL23ShO+2sKpvSCS1r3I5zo9EOn8PGLsI7255C69vONYPVtU3b3KzSw+EKhIvWk1sr155Ia+lQ8aPjC86b1ii387Yi1POz0sGb1Qugu9XE7sPZG0CT7aCiU8ndALPsrKlD1nsee97tBhvJ4SL74qIgA9ZRW8vQArlr2K9p6+BFy0PLG1qr5jIle+NkeEvqJVQTt+dxe8zqEiPQMFWr6errg9uXaYvs/Cu72wvom+uJ7iOrnFMT17rmi9I30jvmlIh7097uy9DdkfvcYGt73Uh+S9UNNsvf+ti768/0i+Zz2Fvbst8b1/D4K+rii/vR7ViL4Dkw++YSr1PAbYHL43lMy9jGLCPb7ePb4yC5i9yiI8vnJ6kr7l8w2+E9L9PehZnj1Z/Ro+B4a2viXrxb4daOq9FwLMO+r2Jr7ZBcG9no9Du3fG/r6rtLU83MIvvt5JfL6AfKm7YdCvvjIPqb6rQRq+0Ww8vpu0Zr4O1TA9zjoQPhpGPD4B1N8+Q0WtPmT6kT47Emk+7VQmvYPJkD4mmAs+teRnPu4aFT6wOZY+7qYFPnfchz4u/6o+VkcHPva6MD5ISWM+7BqXPpSXAT+y57A+kXwqPsZfuT51pmE9lPA9PGEIVj7c2XY+mXGGPt/8UD7uX8c+2y4dPk26Kz5Dkrg9YsJVPmmCFz59lW89VoVYPkG77D5SDus+v3RiPg1rbD5P9/U91AljPuemNz6GW1w8Ht6fPvACZT69O68+fI0SPh68CD7BPPQ9hdOpPrABzz6TWnE+MyzVPaAOgz6bEZI+9eQNu4+opT4KwME+ccOwPqRGmz4H6fc9","fkSTPQN7gj2Oro89ZkD5PLqbEb1xpL+9CuKqPdjOZL1oDwW9jGC6vb7rr7wS5729rdkMPmqBtz1Vp+O9eRO6O4rkFr5D/oe8buG5vBbu+bowcYY90Jm6vX+TdT2DzKi9G9wDPptEt70iotO997KkPTCLIz6RCxg+Fk+svW66wrxge8Y9mnECPf946r1urlK9ZRz6vMPn2727d249+XEoPsG6x73H1ZU8XGmbPbsI9j3aKgI+OhepPX+SsD0ow2Q9uOz4PRF2l7wvrCU+gw8gvoxmGD1ESra9JNmovCASQz3mMOi90V1OPcg5KL5axtG6dN81vhwjSz11kEQ9AycRPt94+LtI5JG9lOnyvKnEIz2C5MS9/OkkvoUftb0Ehl8+motjvTNRl70ma5k9NRLUvUoilz0dqIK9N+RvPbefYLrhDVM7vKKePWBRWz06Rkm+Iz5ovPTPwb1YG/w7RWqpPVK1UT2DtDK+X81svrw4AL49m0e7+WGJPTTbUT5eLpo+GxCFvtcf+L0+JhI/zWa6vGU8M72W/Sg+cSC3vYi6g72sKve5sji+vXslGDy25RU+T9vsvaXdyD0cuKi8RWY5vqXODj4Idew77Q0Nvh2AN73X66w9vVgAPZTjBTwqssU9hZrLPdM5lD6uQQy8o1UNPoQ7jj16U4u9lVLEvCWN+zt3/7g+0SAVvSoQ3D1rXNw+3KQCP2JHgr61TjI+3TDBPmupRT7R5Au+aUeKPKXRMT5nrIg+s2pVPloK6z3ZDgA/2c5rPliTB775sjY+ueSXPgI8dD7WIyk+ROYxvqa18j3PMjY+CPONPaOWhD4DSgM+hB0NveQebz67sIg+JgHlPje8iz3Qr3w+7ewdPlalMb0tY4Y+rn7qPQwsVT2SR6g+mZDePbM3A70lQ48+tNccPsUMR71jBoU99TXBPsm2Sj7DwwI/sE13PmQpQj3dYbQ9mirNvf8ubD67w78+6ZzEPoLihD0Djp0+eLaBPX+6hT2TO5M+CbNEPUPMqD6WWnM+","E14yvSReoz2d5da9axCEPQASLb4Q2648CeexPVYM7T5zdMy9rAVXvdwIiD5NW649NLy6vff9uT0MnTg+Rqo0P2wH5L6o7PW9hk0WPvtSCb6zZQQ+mnK6PoD6f71PwAM+jYJtvUmLRL0QlTW+u8fUvQuIVLs4/LY8UEBTO1z7iT0rVUQ9YOZ+PclxGj2amAW95htZPdz3vr11A0M+6uCFPmfFq700mpq8zDZIvsrYm738qsY9BIucPflALj7Hrma+900PPavVGD1bQ6a9ehw8vMnavD4Qp0y+NwJGvb+lW71QMR2+Z2YwvcMlxD0SX5k9pSBuPeRjjT1vw469Dk2wvjZauD1KBwO+qmpVPi+ANj0rcAM7Oae/PjJyB71gAyU9y3BNPiQhnr0s/B+8DGZivmlfrj3G8Qi9fyzkPYxwjD7xgkC+aFSqPWQ7+r0JPRY/tJsIvYaBMT1SR0e8zQZMvkvF0boN08c+py6HPKj1J7w6e8O9/GPBvVU87rx8V1M88ACwPLEu2z3qlpM9jWZ+PYCViL0YqUA9wNcTvkNguD1dvqA8X1K2Pntupz3NVBU+tnnxvSOYuj1MJiI8zlpRPWFi/zpGNhU9rjEAPbGW/T3ou/k99cKePaNCej7KouU9aMWzvee+Br0Zi8i99u4xvY916r2v7J692KrlPGVesL25rmm8eWCcPdDdor2PCGy8QmW9vfYbzb2rkR07UoolP7qg+D0/6QC9fnFKPfdoqrwTnpm9eDCgPYstED43+Ik+SRC1PUxAUz5kgDS9CNEIP7pdzz3qQda9V9/jvQLQCj7VMg295Qavvhs7/r2jXGE8mT6LvcuUWrtsKKW9oKBYPqoAGj11DYa+RqOTvW5XzrxMTCw+eO0rPiqQ5T3j8IA8ym+oPXuzHD5nnd09xG0/vsy0jLp+KqC7a/wjvlm3E77/Yms9UwXCPVBCBb/LOBW+xKMxPPjWXD4IqR2+7dbwvWeRSz6JPiy8ByAFvoq5Oj2CKhg+m94pPOZVGz5X0OU8","ktkQPKFoTj4aLuK9Uwu4vrLfCr7UsZ29ZYywPXZsjj0LmVu+4z8APjzbIj5Ciqw9ppeovANKFbvZK1m+T0RzPjPahD4Xqgg/FwxnP8FqBz5wRU8+s06cPqYPqT15Fj4+anfovkSux7zlqkk+HlPrvIylIbw6V1m9Q4ivvZzY/z57l/E9YRmFvXZrfD7RoGw+oUcovm7j4jzxpnM+tgHxPduCL70M96i9TZtGvXIYZj11w+K8miGaPtKGyD4VFcG9QBIuPogSFb7jhUU/+tpkPuY7XD6q02a+NL49PiItyj2Cvcy9LmwbPtzQszwH8k49+f2nPe2Guz5o0RS+BN+EvnVNoz1UH5k9eg3SPSHx1Lyt75s9AKBkvZYjHT4ixYy+ptmTPbNpC7wjOmE9oiHXvELcUj6NL3y9Xm7dvLi35b30BoG+3VslPvHKBr6Y+38+WEbOPsTNg77cJww9zfLLPPa7yj1Pgl89pMXqvV0n4z2Aqrc9YwyhvcmU3z3zkxM+QpwbPk1s9z00uuk9xwapvVkLET8oyfg90iuYPe8ZpbsVlkc+Eh7pPO7iezzo7Sc+d+4aPkfrGT23Ndg9HSkyvvxazT7hVzA+f/LlPR4Gs72NGhY+axc0PhXkk71bd4M+6gHxPd3iAz4BRyA+D6tvvQyFprz8ZJ0+gmmxPWbQ7j7KLQY+8fPvPlLYxj0AJwS+ZAODPeIvIj1c+kK/B5MVvn8REz75FpK++1oXviROFr14CmG+UgBhPNDZ7L0qVBA+NUMIPiKYTL3Sg3s+qCbVvayTwb4Iilk+SMcwvYWDLb4sBbw9ZPuDvlTsEr4rOH++0mpXvjfH2r3bse+96nPOPEWQUb21nDW+0SWavlOPNL1kD2W+YYLavgohH726QcW9hBLgPu2coj2ewkS+EBb1vDXz37zCEdS6jQ2jPVbuoz7vvvW+yD5yPuIsHr496f098EJsP8SnEDoQnAI+IETcvq1FobygtbE8ox/YPpE0l7u0sYq9qAWkPsQc/bxJnv+9","iCCkvvR7Br4VsQm+KiczPptqrjz81Es+wwnSvni4jz2TlOe9Cl27vu/sYT497ZI9U/qgvZoC0b1W7JS7bChsvYxUdz5Ahly9Ey01PR+8ij16nYW+IW5yviBiXj5cI5M+nZvWOwilCD7vDoK9649RvqRWn73RYZ88py0NvtSb4j2QJ0O8DNk7velpxz16D8c9o2VSPhAQ173A19y+Qv9PPoYckr2p5Cm9AO8ovsd157zflb6+t6tCvHNUbz7/aKQ9vjcpvzyAzb7ibwm8QIJcPVRL0T3qL0++1ENSPj2mmDwR3aU9BY27vTEsnD0V7cU8oc2NPI49Cb/5gGO+tYETvK6fVD7GPQg9bhZWPrZ63b2ltwY9OcaZPeRrJz3YQcS9TUYpvsqyDT4ylFE9A9pYviCG/T7Upw0+8jSOPeN9Xz2UHhO9jkFYPhoucb1GjM69qsJEPo6/n77Tf5s9HZmRPS5wtb7p/6Q+EjUKPjzPUT44Cbo+UPeOPrxMrD5BGWk911PaPnR7BD8FCIW9nBOcPmN2XD6WL3w+8mqIvnC/QD47/9k+LdOMPhpduD4zFSI/pQSePmVaBz/oola+eJTaO4FGND1DOAQ9EWvQPHkfVr1iyJM+nnUfPsrqHD6XDug922MQP2Hdbz17byc9ObSAvXuPML7KSgS+z1vTPN+oF74m6ME9MMArvig7uL32fxG+m7Z0vr2Uur1amqO+0SCBvYIllb3ITIO+W+z/vX/LcL7+wZe+hZeTvuRnpL6BksO+Yvf+vdGrsb4Mk7y+L/yQPojLhD0sP4u9FuosvlkkMr4xmim/D3NcvsmvTr7Y/wa+PHJavi9a673iIlm+XX9BPrzOdj2S02m+s/sQPUELIL4Hike+LatAvi2y0DlgCoe+Fo+OvkPo/72buju+aRuBvsHbk77fbKe+3of3vRIO0LvMUG+9ZjHovrsPij5J1Im9Er2RvWokxr1YZ+i+Y/KyvhEr773NccG+1fByvgV6871wHYi+v/fmvuNu0r7bpo6+","jtDoPb2mK71CGO69/ioJPpux+j3UJD0+inKlPOvpBT9cVIK9P6xPvdF2jj0SFQQ9Uc8oPRA5K72HAkU+HU+MvW/0nb3KlhG9SjX3PuQGsT3OWVg8+BkaPbfjJL2hthk+Sc+JvrKgqj08yii9rsdwPjuVWT24ssW+Sk2xPf1i0j2V4n49+XShvqT9rT5682g+RPQRvIaBzD3mXl6+W3v3vJmkpjyRtcu7pd0tPCFeFb47VSs/mcSpvOe7sD3xikk+QUKRvvQgzz5ObAk+l9e1vhUm7r08pWc98COEPqytfz1aynS9hIT9Pb3KGT7iuhw94DG1PU31ET/Dcqg7Pvywve4t5zx2aZc9//rOvIn4Ab35lqG9+nrXPK771j2YeKA885sPPkpXFT7Uy0S+UcrQPfG0uL0uvUc+pF7hvulEj76cmPk9WAllPfKzQT6dr2873R/YvQp3w70XBY2+FiOEvGtdPL3cN509Mh9Sve5xEj7t+529zdayPb5DhL7Bycy+fI0dPp3+S74Xsx+/O9amva3TVT7KJqy960Q5PlvxDT1HDSm9v44GPq2S0r04fnM+VHmMviZ6pT3Jqgc9wsVQPhr29D1i2g298mUBPgIuGTzAGI89xmBXPaeMij54NXa9LwvtvQ0Esb0cTf+7lJy9vtWkAr72Wz4+6ZGYvZCZVb2uToS++yT9vlALjT4W+zC/I84vv/ilAD5+/E+/21z8vlqPEr4JU9g+/uWSv0xsnb1rHmm/Dg5APjkpdL4/CNg8g84svXxKoT4TRxs+kk3GPsxSaz2diTq+6uKuPbi5GL2vz8M+8Ab2vZeEtb67MiC/llLjvVN4T7+VvR6/kZeMvqjf8rw+CjC9KQgevpORnr5pBz8+p6UNvzNaVL6qrRi+GEZEv5IocL6DHB+/jVsAPz57Bj+RsQK/S4Hwvs6sHT55m7O+2uxFvperRj2d27G+OW4BPyT/dD4tz/u+QLQov5wvIL5gkC2+E7W/Pa+2K77tk06++nUovqeVIr+l4dK+","cqVSPdxInT5kkKI+Z3GoPpkNgD4G7Hw+dJ+cPrO+sz5WoBs+2NqVPl8dij7WHW0+iLFKPtxevDwSS8s+5iXHPiYbRz6IjzY92hAWvlLXDj5vrCM+flwzPmhooj0KBMU+mw4ePjRelz06raI+7RAiPsEOdz5e1SU+3qe4PopGhD6gPkE+6gLWPaD3ET4IqnU+vtK6PReg1D2yETQ+MZ2LPquuubx2LZ8+a+iUPkK6Fj5H5TY+xNSiPh2gRj508N893B4cPrMxaT7dKKy9Ibg/PnAakD6blpE+j9mFPf/scz6j+kY+TSG8PoChRb6TPg8+j2qMPjZ1lT0/A5w9fQHVPXZ4iT2xLsk9wy9SPqNwiD1Jym29pun0PfuBh705nva+YXBePc6Q/j1JGp48RcqBvagFxbxTyWw+cFVnvjF/FD58CQK+X/yZPSB4Er45Lgq+pK1WvcH4fT5uqwU+cHN6Pvl5YT0z3Hq9B041vkwtYz5VO389ti/HPdHCBr5N/Ns94ojlPaJhE77MYwM97tc7vS8KLL4s/kI9e3MJv9Kknj0M+sY85P5GPuQtNb5fWD0+3g3nOwyrD74B7Cu9Yyg1vUF2RL0zVQY+oqtXPtUku77GFa0+LQgHvu0zgLyZgOi9RCavvVb0lj4ZyIA91kl5PO2Y3z5oaw2+ISmzPeaQSj0u2WS9rYR2Pg5R+rxR5xa+5aVSvfa9Nj1drbA9ro4MPuuFIj5hmTs9r51ivkDqjrtiwZw+RqCDO9koLr0+q548wuVsvFvO9D1F6/u9tycGvk7v8DqiQKY7kSouvge6cT1jFuS6cJ6ovCALJbwz0u49ty0VPUoIzz2WHjQ+66uFPnuUtb3I1k29AsAdP1Lktj3PGRw9j6kQPY3lTT9aWww+dnJyvU760r3dfx4+GuWiPDF5Or4tr1g+bL/PvH8wpb7JnVy9h5kFPVRu8b69jbW+6FHevHvSBz0CGpi9MIoaPYTdAD4j26s+z9BMPS6kk72/R8g7OhDkPaVg/Tz6HS8+","GA8VP15RnD7Eqpe9cYIWP48uhT7rtl8+YmwMPoKmzT7spng+t54BPh22pD3HsXo+SiQPPl7xjD7sr50+wvPSPl8kXL4gGng+Pwm5PlPmbz6f2sM974jTPratm75lSsY+7e+CPtZLVz5ymhw/8WWUPl6TEL7ocdk9Yp+PPlvbHD48+WQ+4JVuPglDvj4E69E+7TsvPvWsgT5JMWS6iWVTP8vSob18s3s8yEzgPqkh2b6/7Hi+QyjOvQ8ACj8AABg+51pxP64WeT7tUQW9BXhOPghC774Kxmi8JmRGP95+5z5E+9g94FYcPuyAXLy/JRI/dJRaP02YiLweQdM9yCxbPjKakbwd+He93/KdvociHL48Axi+SQuNvsMLwL1WvTG+Z6ARvrteFb5mgRe+Yenxva7UXb6bmG29SdaJvRaYwL48aRS/0O4FvELXwb3sSbC+SSKLPK/rNr5IGli+Fn24vshIGL6SMW6+1fSAvzR8h76c5uG9as1ZvI9Ie72vD2s8wl3IPAMfpb3Lihi9y91LvtIfDb4rCSS9//7Evlg9A77GLq2955OsvbLXwL2XpHA978/vve/bpLzShA+/ad4+vjhjdz1VGRS+AVmmvSrmKb59TTi+dHNLvax9hDy1fjy+9Vhqvovm072LIp67gD/gvd8HSr6Ql3+9q0/APejSrL4D+fk8T/FePB8BH74MU5S9jJq/vdmenT0A/V49ctVuPio9qTsWViK+utJvvn/OMr7lQPI8QPNEvJJoiD0y5Gq+uyMLPtIhaj306YW9h40UPm32Fr0I/Q2809ylvei9Gb7tGBy7POSiPV/jAz5NwW0+1xyEPSSIGr5/TtA9RVOOvsGjET0rLBA8tMCyvUVbnL0dptO9iCTaPWbS7b0v9ha+fW3OPKGB+r4n3SI+l4bVvU7gE77OHqE9M3J0vcQX5jz3fpQ97yfTPOsKEj7aUui8QAKlviBpWD3mibI9MsSAPf2FBr4aT4w9MgoZPg6Ovb3ypMa9el9XveWNIr7wHzi8","0kLLvSbPlL0FyBy+kSodPqHiOT4EorS94w9PvkKd1Ltnewe+ja4TvrXD6b2trxs+tP2dPIIGv70QSxC+o7rQO79o6jxrqrq8SUiIPVdUBD69Gqo9XeaxvWScCT7UP+A9vCOMu2ItkzxAW7o9NTbyPcz48T1F3+Q9vzyxvlhfQL1ecnY9iSP7PMPMp75ASgG9SX4wvXnevLzLYYQ94HF1vUTKmb2CgTY9PXgQvp9yZb0MkAY+OQS+PNvyTr7BE3A+P9kcPa0dCr7xh/I9oiupvYFkQL58Tca9DjosvHZvK77081y9BbK+vpY/ST3M/LW7yx4fvmSoLztv/J+9OCuoPUM/tr6pq4G+uOFyvofhZr6FB6G+k81bvgDzcL5BvT6+lixovf7ioD57UWC+DGM6Pakr9r5zCqq+6mJCPpLotr7825896h6XPek79b4aV84+Bpnbvrf/Q77YCZI+dSutPZ18DD4S4Ue+DNxzvlGnVb7t+MK+c0oxvqyl0D3Tplq9NF23vh1wUr4I+O6+vdySviYyFb6BkoK++r3CvYcasb6zddW9rnN8vqUunr2yGcG8UF+BPdZvpb26+Nq+r9j5Pbyd/74Ixly792oKPnKmrb1L2ym+xTCOvQCeCzz+FrC+HKPqvc9GID1P4IC+9Q5Xvnv7ub3RDPy9T/oKvlBgJD4gVYK+0rkavvIZCr7KYK6+P4t7vtMbyL2aHWC92Z/NPuqkj71/a1G+JRzWPU8lnj2glCy+9O6Lvtoxp75xoi++mijXPZMYmD2uEpS+Qdi9PWLaub3tRwW+urULvvSwG76yutu9mU5fPW2n2r3ZeZu+oeWuPQOjSb6y3om+rYQAPXU1ar7R5mC+60EaPnHX3Dy2WF89knQivrZtAL9leJC92agWvhsCub322SW776uBvYY6/r1bG9I9KOaNvT4Y9j3Wswm/yUJevQyz0T1mWrY8bOj8vITabL7eOzm/RPHWvRyVW77ovxu/1meguxO8w74aihi/t3F2vCGDCz097JY8","4BCWPGN6W75xnwW+16p3PQ6eFT2eNiY+jYdFvnpACj8DWLe8X9zkPSswxD6W57086znBPW05ib0VkZY+X3lnvIK6KL8gRG4+wP/mvSi18734Hj4+4B3APQtl1b1nnCg9UqOMvXFU+z0MRWe86ct4vL1LL70rlki+rLAVPub2UT7KKwG+7tsWv13zFb+j8ZE+1KjzO0D/270j+4U9UTEyPf8IUb7psKU96hdCvREXN76mhWa9iU4qviePDD50mH+8vS9jvhf7f7wynjK+1zn3vXXjzruNxJm9CDehPuyVkr7TllM9jWGEvk4OBr6xj+U8oiLcPh43hj7CijS9IrpJvbidF77d9wO9QByjvMAt+b1YL/i7rssCPS/DkD2viAS+Q37EPb25QD76loK+FMiMvSDRqTwk6g0++fSZPQsyK78kZJG9L/NyPbKEsr5FGF4969NiPX1+hL2SRtG95TycPIJZuD0uSb48M85JPi79PT6ExJq8KU1GP6dFpbxP0oM9suy6vOUFpj0b+D6+6MmzPZIU870MhuO6VuG7PpuURL4W36S9ObqyPRjuIrxdVPA9p8j+vTuKUb2Rkd69MyE/PTfA+DzhZKo8J7XMPhU7Kr2mKRi+FJV1vLdcdT6uU/G9yizPvQZjdT2jpxE+KTFuvVjvvjxUFIg+S92rPe0/mT1ilHa+mPFaPdqUCL8ykXu+2yLsPSk56b0iDNc+8Pqivs4cR73I8vc93Gulvjri5b5xqPk9paaDv7ANsrzr+oq+ogY9vrlv/77l4Vk95/Wmuwi1h76x4tu+rRUYvvbMY741lKY+WJCoPp6Uab540Ik+DJjMvnQds77yOIG9+ZEpvpjSr71hh9M9FEd8PqgynL7jpXW/K6egvs7h8738zWW+6aEHP5jESbq05wo/oGOev2iLnL8U+yu/6c5Kvh4f5D53LyK+AQyUvrdHqL4eIYa+VdyfPUy9Yr7g3Dm+wk/4vU9fsr1obBa+t3+rvrpofr6YEq++Mj6HvtfqjL7FVDa/","4tHRPmEEID5sjpc+tJccPZYVOTwJ4YE+ZXEvvUQSj75xjFA9WaCvPqwuHj6VM0c+dJppPjeUBT/bT4o8bqzLPi+qIz6kI3u9JJeDPKYyob0Q54Y+aJlaPtD387sEXvY9aNbyvVGdIb2hEYU+5X8sPuQDDT4aldQ9/iynO3QkNzzHdXA+gqhNPqVvCT6IAwG9fMmPuyK6rT6j5Pe8vq41PmNsIj5eWNc9a2wYvYjtYT7bUsq9xN4MPi8nXj6HBYc+Fa7VPeue0j6PjxA9oWAZPu+8Mz4oEqw7J5QYPeFqrryB7Ie+TmbPPriXuL0wXGI9qlwzvSWQEb5KPig9WIVIvYxufD51vA4+j8RJPW1ULr3+AfA9w9PTvJPFhD6l7bw9TYqEvOqVV70QBN68C+p2Pf0eqb5jgBS+Y97WPVxNgL7Q+5E7tf//PC5hRzzz434+dSyYOwe8t77Znxa7TSM4vn78Rz52SwM+NiFFPRre2z2Dzk8+dZAwPf07Hz6zgck8V8eGvh4qXr4Rlua9Jm6AvZPydj2PdZC+EKlOvZuvCTxk54s9+hbLvJ7ZNb54XjQ+SWkEPgtlJT5glyu9WQuOPglqI72QJD+9UTm0PnhMCr26AHO+YQ8ovXs8xryeClc+gsQ+vsPJ/bzYtm4+MnxSvZ67Q74ImyO+BMGNPN7Mwj51SZa8zXjwvG7qvr1I2Uk+WkIAvMSwnbxvTDA+UGIKPr2+Hz1qTlS7+ZMJvl6NvLyAwQS/pETQvTXb7Tsh/xg+g6tcPAwGSL4T04O+qHalPldUBr5V66k9jy1uvkoker6SrNU+E0MvvqVnw70cAK69ffHiPnsXIr5zTgi+CNkzPIVIHj6vInc9l77hvWXljz2yTIm9JXpNPQz45Txzr569wxo9Pj6I9z39qxk+Ia4CvpoR6j3vXxk+QNsRvWqpTj4LHpk+kpDQPoa0G77jnSo9HrI1vfJzDD41uui9WaKSPnJQjT1YGFo+jg7AvY/qjL1Dy8e9+7ocvm63wj2MO6G+","TjncvTmtqj7SR8w+uxXwPVkREz77XYM8VHmgPVv6KT7koAc9noCPPcIytL19RzA/8svxO9bJqD4K3II9Pu0+Psjo/LxOAK2+Oi+yPlwmUr5RXNc+3wAeviUNEj55BYU93Ui+vRFjgb5bZBc/Po64vXfhTT5W/K++wqYvPYlMi74J0aC9qjsePtkLnTydtss9dTPTPmIUuz2ew2Y+mfYDPWWxmzzd/C0+ekJTPVw8lD1wBjc+ho+YPTxFcz65/Yy9QVeCvfUKxz1vx/e+H0GhPjQ2nD6qHpg95gAYPqRM1j4/kwc+/nTYPuPu1rz0rne8SaNlPmpX8r0Li829j0aVOmf05z0nsws+ybV0PrsNSz7w3iI+sMwTPcB+j7zrMKo+6peEPr+tQz5/mN49afyIPlvKTz4eefc9ZeQ9Prcqiz38dDM97919vdXFYT5OE1U+2sgdPhD0ij7IOU4+OE/SPXNrGr1ZF7O9CWxuPkp1nD2zC6o+ED4TPnPr9T3H8+Y92kHBPg01lj3xM9o997/uPUVZRj4zMZA+6pPnvdz/Pz4ZbS4+YAbsPXQYfD7pum28NvmLPYycnD3FPlw982SmPthLXz2j+4M+D89JPu9LeL0saQA+FLCWPRuBvr0DeNk98++sPleMTj7BLQe9vao7vdMTPT6DTfo98oFwPv2UYT79EVW9S2qDPuUeNj5Ai4K9P/UlOlNAZ72wsFG9gxYpPbiwLj5MIcc9nTGJPM0w6zukfo49tgL3vetNPr5L0rM9RpL8vTmeAz5eKgM8AdRSPvDz5zzZV4I80CA8PeZqMz4RSrw9YWmCPV8Cpr2d4a299V2LPZu85D12X54962YQPi2lajwOwaq9pw28OwSsCL51c7k8GIXmva3QHr6YQfE8FOPtvVLhI73t9BE9wswvPWL9gz0Gw5u8hvzVvTFKWb16ywU7ZugJvtDrTT5T54i9uV4APuf8Njxk+Fm9YubivcKS97wAIU2++m/YPY0zAr2IJc69AIKTPVTmcD3ip4Q9","sEQCPh3WUj6DFwG+S/uLvYctZDzonMS9WUolvMH02D1vvJA9iKuGPPSCUbuxxeI9t16SPXLrDb2xTcY9xLOIvCMWAT5t6/m89Po3PYK36b39AM69hnlRPubRWT1Sysw9X8Ipu1GAyj0bYIO91gKIvclonrzT0ku9yqkvvWLhzLwD0xu+AZXxvXCOdz2WOh49pM7DPiQx5z0RfqM9z7JVPYGZ9DyIvDM9eIeIOx7AtTsW46m9kPYKPbLN3r0l2LE92SpNPEsYAT6nLpO+h3PmPErxob2xnxK90pnnva3GBT5p7m49TZNZPuNcPT3BGNu9Xr/1vZKgMb0J6Ky8oztHvZaDJz1Bujs+2UibPj1Llz4+OTw+jEHAvW5Xwz7MSc8+qXWBPcufNL5hfI09J2KXPaXrzj5/IqM9ejN0PA2siD5oToE9kIATvvVXhT5Ukz8+Su6WPl5TxD7GRk09+/iLPadvg71Sv20+GAkSPqBupz5BPQw+8Y6IPnjHaT5aOaM+TjC2PI2Ouz7/r7Y9NXg8Pva0TD6yn5o9T8cDPgn0g7wPMVw+M+AsvvmoOT4yQfY9VSdvvpi6er68uX8+UQqhPn3GTT5MsPU9fkIfPXx68D3qS2S+pnYUPnjAKD6k6to+veodvnhVjz3G7lw+Jel2PlzVlT1KAMM99bpqPpTzsz1GTCo+uhwYveE04z29PFU+6s9FPm8NOb0zTqQ9xOUGv2bbNT7Xcre7V41JPuwcjD2vIlA8uzV7Pmb0RrxT/0++G4WGvmYnYz2ZSAI+Fh7RPtosnLyKT0o9YkkjPvWnVj1N/8M95bTsPQfvgDsAWwQ/cV5iPN/D/70LFeg9kmAPPm9Udj7iojo9ZQzDvCfXxT2cuuQ+8kYgPuLqXT6E+oC89cdCPhbYdD6flWM9FRYJPqWfwLlX1F++jJmSvKJRED0OqGA96ECBvticXD0JFf296QulvO6EAj6cz3k+7UEpPoPKej4C0mA+xYqlPtS6or2fmCk+gCKbPocKDT6giD0+","GVQ8vRgTAT9Cwau91dUgv8P1gT1tca29kycPPojjL76DczI9U6QGv9cx7r7rPim+5WXfvZGZwL20pAG/Y5/5PS8qzD7PMFS++fzRPrJYAr7vyCY+ZsbiPrXJ7bwxtwQ+5sCiPpd95r2L3oo+1yQDvqPkm72TcaW9EebIvOYxL77do+a82UQ3PsmfOT6ZNEy94QTMvSz7Fb9pzkc9tL2MvY01pT4dQH27uStSPpDdTT4Q7xu+SQ2qvBnVgr7r0DK87CW8vcRlvD5nXZO8NrXAvZOo8z0iA8G90zuzO3WAqb6wuN29bgievUDegb4Q5zo+5xjcu/geez6Z6OI9TQBTPvd2Kr7/Lb49LsSRPashYj5hYOQ7crOFPc3Cmb4n6oi9hHABvnaBZr8ZNnA+zvtBvrdBjb5dsik+S2IKPnF4+j0A+Bs/ZGnrvSNyH75Oeio8n7k/PNiwDb60KpA+FJ5nPif1xL28HbO9GOFpvnvNs77JAc89neJ0vmiwtrzsrBq9R5b0veQsCjzX81Y9XQULP/6fSj2tomK88eXLvn8dmT4j5rs82fkOvs4SDj2loAy+H1fcPU6gDD6H18Y+VbWDPnz4Cr54JJG+LzUGPvRh77yNu9S90Y2cvdqbdz56mE4+G4j1PDBVj76fGCg9cEGgPrDR6L2NoyO/OiQjvm8fKr0oJsk8+4dPPcwSC75Alko8rvK8PsD1D7zYZqc+SD9uvt3jBL/8RRo+6W+NPtA9dz5SdOE+WCEPP2szqb6XZY09FOicu2Juiz2RuLa+n32VPp29qb16CUa+7MiXPi0jzT078f++d14+P70FXD4z1Dg+iQvHPoAQuz1eh80++iNmPVDElD3kQo4+9ib8PQbzsj3Na609EI2BPubZ3jjFBS2+prdvve9dTT4qQck+Oq/bPgMovz6soDo/Yjb6vnL2TD5qd+G9sKKvPSu8cz4yDiu+KuvYPgmvlr1OlAS92wwPPT8SIL3N30a+p+BfvRccLr70JnG95ZHTPTJ+2z0hSgu+","+DifPS6sUj3PW+Y+w/B5PqcrKD41cRI+YfU4Phpxrz5EgjY+8tKTPZmLHD23CGM+sX64PeexjT7j430+8bgXPrxVCz7taBO8Unk5PrdY2j34OWI+PclBPisSKr22NbI+Ia8MvY5w3D2A+co7KIOlPlqFJT4CpoA+VVfTPquTtD3+Rd26ORE+PjrsnDz8ol8+Kr5FPeDKOD5FCUs93sAnPoS+NT74CLk9GGFnPsPJUD6wfnU+9hiGPkdIlD5LlYw+pIusvZkpqz0JBs88GqhCPg0r/zz3oJQ+AC4jPDOqMz3OlrM9E5O9PlsFuL02oeM9FlEGPge4TT6zPB4946FRPhPEIrvFqXC9RqLEPFhzp7yXlwe9k8YVPo8OiL0gkxe+RA4nPt6tUj52Hmw+yVTcPCo8SL0DLAU9HJ7FvNvuxb0jtxa+vf/+PeKbPb5+K8w9ByQBvmLJmTxiYmQ9SOb+PaBztD1VRpU8fgphvbEyID7UvW493rWGu1FAzb0iAio+f1/TvSBs4r1PFNo8U7/nO1H1JL6f2v89ZmRUvoEIGD4px549FVlhvVPeU72Pyhs+rhGKPkVyzjx+ZcG9lt3nPL+mc73KqAm+V66iPXRSCr1LHhc+JSzYPTtCGL44e3O8kCTjPYs9J752jw4+F8ROvSXooD35vl69bYOSvMIO1D03kpA9eE6XuhYadD38T+A6jDnNPek78LtoPEI94cewPHVHDL6KyOM8pTM4vgqSyz1tTlG9QpptvWrrzz2A5X49VHdvPbTO5T2/sUq+kV8Au/sHub1NsTS+eh44u9rWTL3YTcw9m2ZzvQNH+byEWhK+u6zVveJD4j267Pi9cysQPVJH1b3tK4M9A9fDPe+RyT2JaxI9+oGfvYRvRj6LcA4+vQc7PYpPHD3vQvY92Nl8OpPCI70E6Ps9nDYcvcSd9T1QfeQ92I+PPS83vr3gxZE9rno6vn/CXzy11na+kPZmveGIL77M4IA9DlXYvVV3Br7cZsG9d2VHPk43wL3O1Dy9","3bT9Pc3KLz0WbJC9AMZAPnybpD5EmGO8I15SPe2RPz4qpVA+69xVPc7NCj5TM2Q+9PmgPtF2yjziJR4+9onGPi7zMz6LgAO+ysYHPBIvk7yBDX+94ekbPqeSLj5FKkc+kkrIu0k9rb1WhG+4t2LWPjdrgT7/51m8yhQ1PsRYmj7HhfE8WvaGPRYIjz3ozdm9Rnsfvl+I4D3qwWQ9gIyuPSlX0zxd54Q+SSekvMD+k7zBfkG+HDdoPfo1oz3ClT0+kYpoPgk8kj5JkGI+spofPnblDz6YmrQ6evQJPmCrxbz6ah0+10h6PnOZgD6qTXE9kS4kPqa7Gj6NgJY9/AWmPo4mZz6XkEW9GpcoPiKugj7diZe951UBPk5xjD5bbKU+zaPnPELMGzv4vrW8NWhmPr8PmT72nLw9k+x9PhxeLj7Ov3w+FBJQPcyUJj77GWy+c/SQPm+3VT4qXec+Ta2APjx3ij5CBeI9RoNsPb4hez7tq/8+FL8EP1PUuT4qjVi7E5PdPcNLpT5jbc894QEPPjjzrz6Xroc5rEgovskGlT5wlpU+VTuYPmhYjz5/RZ699jtmPjYHiT4gjnI+xFOovIQt4T5sbcm8lPrjOydCJz65r1i9cbVmPlqIsj43RA4+xyYIPgLt/z65vDY+k5DuPHh4Sj63fgs+rCzOPqdOzz6XBYU9/g/tPbg4Oz0zHQM7X58avc2aCzyllQO+WV3wPXHZAj6LOMI9Vbv+PcQIjj1OlMU9vQAIvdlpmL3t7BG+1QtIPXuOUD3eXBI91FBNvCZmFryWf6M9L1YbvUtbmjzt/ec8khb7vMmCTT15aM29KZlGPtj9YD4TJ+28NX4uPjuA0T0Tfcq9UsZcPWyKbTytH2G+0vqpvOw/cz7X3rI9GIvVPeHUdD3xQje8pyJlPV+WEL7lz4Y8RhKIPYPvT73bvQC8xJcjPpTUY72U/xS+/huMvRJPqjwHhYi9kFt1ulOGhr2elSC9Ljn0vcHwrr3r5ya+Be8GvVC2djz0Xd+8","zSw8Pi28zDzMYfA8s6NWvCvri73J3kW90UmQPQw3Qj66Jrc7pB2kvRoT170jC6u9oL6BvYR9DL14SOk9abaOPfAnOb0wMrY8nGQsPYlUMb62P0y+ZvOTPpcZKj4Z9Zq9ggdlPakmXb0liOC9TygAPDYhVr3xdek8irY/PsV/ZT4+aRE9BGIbvpS9XD6QapI+gEA9vI3vOr5vNQg+ja/PvZgQb7wkXae9sHFUO/HJMz7x9n89FaIFPhx/xj0b/Ne9tOcaPk1xMb6MnSM+4Ablvb9ZNL5OYOO91m+yujmvBT68W7e7MQ5bPipHE74fATG+2pUgvcc7Z76VYBO8LWW4PdaOf7uyWCU9pLNwPnAvQT6NDek9KpKlvf7+wz7l6kk9DmJpvpU9HD1i3r4+Qtz4vaR9HT8KnS09AolFPpyRzD6t5Ew9DGhAvm35lT0iS8U+tYduvunC2zuf8mI+QQLvPSwZWz7JrK8+TB+2PUVQaT6a4Qa9s2DAPp4HeTuBZA4+Gj41PkniGz6YHUI+ufqRPjy5Ij6vWlg/NpQcPuvIHj4mu3E9nswTPdekDD+2cuE9XR5APf2nhj7ZwkA+Y5WWPvAHdT6USWI+c0liPvKl1D3v2wq+8JodPrh3Wz7BA4y9Nx1bvQYfDT9I3DA+xLC+Pl4bpD6lW1Y+XVYGPlt4Ej/wAlS9heQ7vmkUv7627rm+j7SSviN2Fr6YPVy+P4t4vXSnJb7O6Za9lA0gPWIhub0r7ha+qz8gv+kVXb462b29UckUu/OxIL4gpsy+D3hnvM5gvr7XFKu+sS5/vYcdKL4MnlG9XcY1vewX1L0WwsS9NfwZvgHMBr4SRV2+GGVAvgTzQr6iETo9ID73O+7Ymb6VhUS+ATn0vP1xhb5+kp6+vxwKvZZII776E26+LHgBvymVUr1GK669mR2LvskRM74p3t+9n/cWPQGbOz77x7q9mFsqvIk/lL2BKD++JXtEvhs+9jwBdqK+DpNrvSJJpb7rRoi+b0czPZSKkr6fCCi+","k34vPR+a2T2xkBu9JTRGPQrEmb2ne8Y9P8lHPferdj4s5WW8efDfvcSPiLz0yk+8yBqHPndIDD2HoIs9HbihvfRVEb3w0pO7HF2YvFL25Dv3Pcm7wMIwvguF5b0j5HU9S57dvSj8hT3BLSC9naiIvkUmOr6nrZs8MbIbvw/mhb3jgr8+CY5JvTDI7b0Dac49mIGtPZtWAT5HXAs9GJj+vY+p0D1p7x4+S+qFvDk3G76vJ3K+oq97PdpZibwAYOe9R4wgPpwZuLutU6m+CEjnPWNQmT1EVuU6pT8FPv6yrjw73QA+vTQqvibRKbyat3u9+7qcPflldT55FiC+3947vWk6tT3gZYQ9JFhVPSH0f7tpJQk+JpYSPhUsj7s9jtq8goOIva+sdr2yTTM9oamVO8kkkz+ee6A95cisvVW8Xb5RXiu9uXWdvSR/Ob2rKI69QnbUPZeq6D0wvZ29aQnpPfDd7b3tsS8+wfnfPcT/zD79q0a+ZLfAujcuIT8sxw69U/UoPkRvub3ZHgy+Xnz5vRHFI75xEwu+GRzpvY4T0j140Rq+a5BYPJLACT0K6Fs98XhbvXLe/bss4yE+xkIQv9w7Vb5XJp08AFWLPgFICj65wKe9zCuCPGsDHT16Qhe94yr1PVFLNT4dGhM+FDREO992yT103rI9aWZaPZr0bT57m+U9j3OvviFozD2ADqU+P8p/vtnQXj62ll2+r06TvXsICr4YsY09w2GCvWjjjr+QoHK+ld6+vv6CHb4+q7y+f4jUvnUr0D3Q+yS+JdZlva4SP79Bi4Y+6mVGvqnSEzxkhC09YL+uvZFg9r6TgcY81XXYO0w5xL3Qm969/wqRPcKWmbtcCA29VuimPuCow76YIz6+nxJhvBdJW76D6BQ8SEQLvQe6n7zKwze+QpzwvSSryryrZJG+vZrgviCIOr7TeNY9hLaDvq7cBb7Enxa+on2yviLKyb0osoS+8KeNvjUR873+wZe9vf9evktgBb3kIQK+EGCdvgJAvb3v6XG+","oKQtPQYqe76HAkC9GIEwvvv+Ub7qD+q841SMvteD274uGQW+LYegPH/3L72TWie9gZ1+vq2MVb6Dv1C8yEKGvZtIQD6QK0M9/3b8vU9Vjr5RBiY9BT0svrXfKb3TklS9wbMUvvx7Vb6eq3q9Mt2Hvo6diL4I+q+9HmaLvvhzFL8C+SW+dI0Yvjsklb7zx/++tHFWvvw4ML7lZr48bmYcvY+pP76rx3y+TQ5RvnBtlL3HWRy9GmglvpD0+rzkYra+1dMlvnI0rr5zRS69HlDCPvrddLxJHKq914egPrx22L3/pUk+MJYSvrvtwr3cFVO+1y8sPaB3cD4C6Je+l8VUPMy2OT1v+wE+Dy4+PryATj40BOI8PcWdOi8Fqz7/oI49Y9YJPjLJgD7W6TC8E1XdPoS3rD1pD1i9LqH/PXad5bzSZe09vjmMvXEZGT4ecFG9sKunuqwfiD3RVoS+PhuBPAfOXL5rqgQ+pIimPVTdrb0HGqU9hkNuOb8lkL64V009R6t2PRcJmLsPnWA9SFktPknHlDk/1w++/vKcPU15vD3dY1s9XwE7Pk3nOL0Wjz4+xBQJvt5RX768oia+jt0GvpTm7T7n+GQ+lZG8vbPxnr6EkG8+qf8SvsJYrD3KHgW/Sv8TPAI2T74RL9w9diQOvpUrobxpF+I+sh8evUe8Yz6Qidi8mv9iPh0V1jwsH5q+fs/5Pb8bo71lQto9rrdPvuEsFD4tV0U96QefPByrWD5/Rp4+n/VGvgl0gL3Hveq9XebVPL42lD1e1E0+tRw6PJKPw71f+nM+GBC/vXctKj1KJOC9PRsMvBojDT04dNK96qxbvpgIDL9Kw3Y+5DMCPoVwob5BAxE+X6DgPHlVjr5v3dE8+EJgPSeYKLzmQtG9CdEpvsSolb3PCyO9Z78QvsapZb2g0iY+uJXevWoeIL3GsY2+XVKVvKgLWT6h8+W9l3eQPc2v7b0XTEw+Q5m3vcub5j0T+48+I0YSvu+zej5ys/69VX0Uv6U7cD4U4L29","jH4jPig2pb4WnUW+HDGzPZRY1rwFFsA+j6m8vpx6BL4ERkE+OJwNPTy7cr/3ScC+fc29vvIp+D6953u8jQ1Jvsu3yDxocPI+ecYevv6Gn70RVC+8ANjHPuCSSD7IYBK93LWBPl9KlT2G12i+PpxDPU5PnL6Js849h1Xnvd+Twr7Yf1o9wzEVPWvflruc0rA++ldyvlv+kr38mK69NR6vPkVmF74yKPS8yH2XvpmRHT+gl0E/TfYMP2Ww7T4pSaS8/rrZva0vg77v+Ua+DEPcPawuqjxCN26+L0qVvckj571s4Y49fopBvqUfFb6A3rW9+YkpvV49jLzaPRy+nFLdPElSSLw+Esa93991vvOpXL5l4kK+NzjvvYBwZ76ihBG+4XFevkizwr2UnCk9FLVSvh19Ir70OkK+H8d1vXeHU76A2C09Cns6u17EQr4oWAK+Cc5Nvz4YkL5eRAG+gI+cvo13d76QuUm9nFYKvqxj3LzgFaC+20UgvoWsQ74p/HW8+B6YPTJHIL6WjHa+8vguvrWggr0yIJe9ohlSvtuVj76fbSu+jBhlvcR8SL7jr969Ge8Ovvx5KD2jRc++dfmKvtDokr7R05G+yF6hPjJcGr7FegO+ZBd2vVDgGr6F5ku+XaAavoiLj771XFm9/53uvcnI4L7jPou+91jwvfYbFT46XVw9t2jdPJGWzTsDzNu9tMyYvQ3fmjwlPvS9V2QPPlAppD3uIhy808UsPhrvAb6A47K9CgzNvE4ANT54c7K9WJvfPWSuLz2X+0A9bJsoPkZjCj5nBLQ93bqAPDw0yr2WmIu8Hf2yPLogUrxm0Q8+KZwnviHR/718dPw8p32mPQPQAzxKEMg93/FAvoub1Ly1YsC9hvUtOqZHez3dH5u9N1HyPPSSsTzZ17O9DZ4yPklDEj69SQ+8niwAvWQhhr1bcRK+lzRnPlLOT7sRk4u9fCIPvUrhuLyhRZc9q0JOPcWPgb0xerY80u7jPWiz9zyfRwe8MeMjPlfHEL6BO+u8","luFJPTSVOr7ZbNu8Gb/1PAlbkD5SX4q8G1IOPsEdOL40DvC9lnYgPYLcbb5azuK83nYGPULG+b1tHDu87zUxvcsCdL42EyU+vlWnO8g5Ez61yGw71QK+vSjpgz3i4qc9phrFPSytPzyOpLo+uuAOPdOGHj4vEAa+ut0UvKLrHr5sltY83PeIvUFL9T3qT26+7XrFO8kAr72c1kc+yN4qvpW+u7wqr+k8BH70vYOoKLysfW6+yuCRvItXlj2K/yk+v1aBvWe6NL47Em6+t6jJPR6Ex7xkvRk9Kgk8PV0ZkL2/qnk9mFcrvaHgnD1jPLk4Xct0PXXAnj1jDy++o3GVPOnvsb79UQI9GGf3PWX6Mr3B6Dq+R5i0vdjEub4AH/C+wfI7PYIfpr1W/PI940hdvoYPDr3TMqQ9MKyJvk2+i75sP3e+owE/vT7XXD1v7Rc9cL6uvodnh75ThOw7bryyveuV2T2ZoWO+Anq2vK04Rr6pXtW9GHH6vdOwGrzurbe9Ttzlvj5AID7FHQe+R4k1PcgIbL0PGuq9xp6IvhpBprw7t5C+FEymvjprf74X1cM9NsWVPjQObT13GTq+fEE4vt4Uiz1U3ge+6J3CvvPvkr5z2mq9wUqiPZSOAr66xCm+3ZjnvWo8Hb7LH7o9B0DAvVYOyr5j3Sy+i/NRPd7jqrzwpz4+HlYGveNHub1k8TA9LvDEPaOeiz1J0wQ+6Q90PbusFz7GKk4+FNzCvVZ0Er7oYa09bqA8vQUj6L2+Ohy9+5Covudj6rzLIOq9TNeGvseT6LyWsHK+WnVSPUNXMD1zrk4+S1kBPVAdDr5d/BA+WrPRPR/7B746Phw+QjWCvTOotj2Nk868/TunvcdxcT7+ui8+VuSAPVngfT0MYkm9IhWsPkCWrD0nQiO8XsetvcAqHr4Pszs9vTUVvVMXAz4LQK+97WI5O/nENr6NacA8yqG+vYstozwYxSI+e/UBPhPTPb3HAVC+q2vkPbmB7js3XKM94tG+PSbvCD4GcwM+","ttkwvrSwNz2YQYe+aaqOPQLBOr1ppry8qcIVPUHbGT6byHm97ncOviSJP71nAVe9lghovkdzMr0UKFs9DTtdvYLvDb74fv88Tl+iPqx3yb1/X38+6UrhPvY9e77fIkE9FOaIvYA8cL6SWPa9eMiHPQzpNL4WDLC9BSNOvctHgL6xul89c8ulPIvY5D0Thna9CwGXu/wgJj6FM0O+A/JFvu+46z2wol69rcMOvpmntL1FeiE+4u+ePXFxKT5dUs89OrzQvefQk734iUQ9TgcuPC4oDL5islY9+ivZuzktET7AlC++MnQKvtpjEDx5enG+Yh+qvQQUhj0UWXC+J9chPeMyHD03nm08BB/mvRe04D2vBGu9IgfEPkwblb64zfO7j+JmveMN2L1z0VA+IB+zOtRhqb4gE2S9gpMKPTSg4718FQa+V5YRvlj0PDxA1KM7BjwHPFxYdz0nDTs//ZxBvlg0YD28dQs8yd7BPQu1dL1/QTw9PgejPUN7kr0vD5m9ECvIPW8BrL1CCfY80gy9PA8cQr28vnc8tK0Gv9rEZT5gDXk9r7ltvpxFJjz46fi9QhKmvWutPz6KDrY8XFZDPgeEjr48eeW9e11hPqG2Kj1xcgQ8DdoFvbmtQz21Kig8MtWzvHPle75WqeU7T1kwvRIkL70+03K+iTuFvrQ9Vz0JXhU+Q+QQPucetD62SKm+lCRvvN73Jr58y5Y+lRamvkbD/LtJEc88SkqfvJCnED5T7V4+f0MLPsGBWb5HdTA98Jw1vfT2oz7o4Zu998PaPpM7ULzrnIa+zWCiPulkgz73nD0+8X/JPQ1pD710TCw+frysPrW1lT0HlKQ+38AgvmnLPr77rg0+A/+lvUOzFj4JfBQ+dOS7PU81tr7d4Jy+gCekPZRp9D2STAs9KjShPr9UwD1Gm+g9DyOivjcc+D6QaBq/zfL7vXUODr4IDTq+D1pdPqAEgz5zePm9oHcXPTZp2zwDs6O+0Z95vtbJsb4SZYC+Sv1kPf6OKj6Lsfc9","3/CSPSFfYj5tppU+5tKkPqw4hD6e7g8+2ngKPuNCQz47cos+x8MUPptPN7si6lE+EVEVPlYc0D01ldE+C0cLP1W5pT4Q1fg9M24JPhOvmD7DfbQ+t59WPu4Dwz2Xpak+RDVQPvz1jD7XdBg/g++dPZPeHT5ozAW7VoycPkQ75T00+1w9HFffPWDJPT1wpGE9QWx6Pgdrhj4ysKE+NFq5Pk1MEz7ftdU76dXIPZJlUD25YqW8PiifvASwHD7kqWg+F0lru/0DkD3Q6XI8yVLqPVa+Yj5A758+XkB+Pl/v+j08/2Q+94fiPaq+jr3dQWM+P3X7PuAT0D51jRc+Xv1OPrMUNL3QXDG9SSGNPgv9xz03mZ+9yVkQPl9ZOrvER4I7JiCsPeMwhT1SZKW7YruBPdZI+j1ctY8+WZLnvDxmDbyzFRS+ot5Qux6z9L0akM+9E7wSPjBOM72TCoa9UO6OPkRjED2SSOk8XzNBPV1lu70uepg9FiEVPs03vr3Hrwc+9FubvdNTTT05xEe9HSVRvZOj4T1PlAg+3K8QPqygCrsn9IE70SK/Pm/vV7xekCM+lhA8PcFYY70kcwy+YI6uvdUgQb1AhCy++06bPXVL5L2uz7s+kAYtvaf41byN8AE9Sq3yvZxDEr5kZpo9LoK+PAM3qr1ZvMm92UxqPZ6NRD2wg3q7zSMAPn1FtT3pz4+8gd0Nvnq/NDxMNA4+WOYrPsCG0b1zfPy8/f8yvW7IXb0mBIk8S4ntu2fXAz5NUcg9bW48veCq5j1MCaM5O26GvQy92TwiICi9E52vPITkGD3/gHi8Pp2xvQ4lBT6K+Fs83xUKvsASLT2wbQw+XO4sPmGBxr0TPCo+SFvkPl2zQbw12A89p3EGvJiMG70nFt8906zdvVUvODwK3JI8J4aLPAgE9r3wfJu9aQ2AvSZK7L3qddS9e7BUPneLBr6Poh+84NGyPJIlMT6T7u49KHnWPTb9Nj5topw+kUlQvh9MSz1mkRA9DkY3PRSvYD2VpEU7","GPaSPiQ3Cz8iL/c9YrWdPgVMXz5doB4+hLqmPvYwZD6VdHQ7bK4gvZWODLwOjTg+KJagPsaboT1u94Q+AWnhPs5LkD3wJCo9rbrPPhG6iL3rOec+a5HcPpZ7OL501gg+hXooPiMyZD7Pl3Q+qlEAPtYhjT1dYwE/IymaPLnRDD4aAuw+GQh1PiFNBD98gV4+kV8LPkRmWD4TO1Q+T328PpUDdz751nW9VwkbPlj9Dz5TUgA9QZ8cPnJjIz/eKCI9kPApPwr7XD1EEO+68+GEPq3zzrtlm7+9LoycPphe7T74G4g9gPiZPvdmjz1kIKA+gOWaPjeBRz0Gi1o+mohXPi0fwb7xGBI9qzj2PE6W773I3++83f2QvUbgoT1M70c+nc44vkEv0T2Cv5a91+LdvXnQLb6SoQi+DFqMvrdq7LyjP3e+jBgEPq8MJj/nMbs9rSmlvapjK77lWpm+1iHpvSPks72Wb5i+8+Unvkgtc75HhMy9KBAAvnjk9r2d3/++vflNvSqBKT5izbe+SfCdPt3n8r2nTv++SciBPkb/ED1f1e+9yvvzvFbgXL5Rkjq+eOxCvS/vU72JqiO+DppTPRU5lr7cDbe+gNNHPXgNOr6LnVU8k2qsvTuXCj4Q+jC+1IUXPvZk770F/J4+n0WqPkO4H77OGwq/hlcOvWoIRL9vJFK9X3uMvY2czr3n8zq99eIqvvrBRL6bSBk+HetjPR07orw6Muk8fGIlvbhXgb7Rxoe9xrhYvi2HAj9Ee149vaPEPm8Tljznf966Gpsmvko4OT5HyRK8SUJwPJPKXD5KXrm+KwcWPdT2oT7CuIg9df9OPYHlc72dXgG+dF4hPhgZhj5EAsM8jVQEPxW6tL6GO6a+JVsAPSjDrj3AI6o+zBIwPVH9AT2rLjq+yrkuPuDvgr5IXn4+t6VqvXOEUj4wRw+9M/WLu/mnb75iNie98vqUPn2ZwL1PuCa+fRYPvCFdgT3hQrq+1doUv88HZj0Wxh2+jxsbv0hG0j3bDoG+","YuNVPZWaIT7qwyi8BF4bvszYIz1el6M9XNUpvRJuhT3w0Q49yogLvVZUzzsH+hS8LHOAPcE+Ez7PScg9ZCMGPx2YPj3x6ZU8PnEJPuqb8L0zDpC7Sv+6PQ3mpT0HOSk+HSkHvkfxDzzIpzC+mvFEPoi4qr0fvfK9Dg20PQT0Mb2cqYI80/IIvs8Y6r7J/l8+fAENvvXCiTwzx788qpWEPQ6H4b2bN6u8sGdSPT9LC76BsOA9vuuUvn//wD2fQEW+CO3nPksPsz2+DC4/PWlPPkpxoj4WM+K973IePZcUlD4n44a91fyFPj/xQz7LFIS9/44XO+Ismj72Nv49YUOAPoojir5Ddlw9MHhJv0IcpDz53Xy+ipi/uYjR874D4Qo+L0oBPr6WmT0O2NG+NYcQPsLxRL/VP6C/vfTovUH9Bb0A+Ru+Af4Rv2B8vz7sG4E+1hCCPez7hz2sdcO9EmqhvjZEFD59Y5q+QjlFPgrZrb4qav2+jO0vv7LzKL6VkZE9YBh7vuM1t76xqPa8gtLzvuAltL4Nnwa+pLe3vb6X+j4vPwS99mhgvgvdFT56siO/EIQFvxbj8zxvjoG+0OoSPkmgTb5sYUm+ih+BPrybRb0qm46+kpnBvoifcb0nwYW98VJNvTrS3L0WPuC+3eXfvpkpT76UMPC9q/f+vQuXGT9aIui8aMghO2tsJ75AnQS96LKgPTabNDy0wFg+5NrmPKpWTD1KXDM+ln25PU5b2DxOaRI+yDOuPuIikTxqLN8+9dgHv03igr33mgU+T3iAPRRsYj0BbWs9iYMNPr7OKb3pQL29A/Qkvt1g2j779+U75eDKPG8lej3dUNU9fZClvoIimj4Gd6w9vZRFPp3Ytr6lGAs+UtRrPqibV7waHis9a+eDPgor3L1NmEg9SQWHPD70n71vHB696mqtPr4CoD3p23k+NwFhPpeZOL4hsIw9y9/fPQ3mUr0enni9Ymz6PJiRKr6LsXS9ex/KvZZ4Sj0XQFU+nR2NPfUvtbudReg+","pzrsPnB2njxpqAa+/ZxyPU4DJT5rQd8+LCaYPpGU/z30EkY9FV6MPQgz+z3DDQu+5H4bvkihZz37RA29ODsOPkPukT76KFy8IfomP1MaiT3mduk941uVvsvRcb46+0a+PD8GPVMsLzzt+la++7WMPRuKV74TGNi9AEFsvqTOCD4KcKi+1sf+vdO1mL6TGYs+H9uhPkdzqL0D61M/XqJhPiaCOD20J+a8HhglPe01Xbw3mO08OhfhvhvpGz5PiwM+ZmvGvu86TD6G458+bH5iPkyVgj3vrv89ghLkPJ51gL4kOuK+h/cwPmtZhj5bmwE+CrQFvxP1rD4GTZe+LiHTPhg2w77d7uO9IOMQvpCOfj1y9y++O86SvnPOPr4SG+G8336ovWMi1L20ecw9N24yPJikOr6f7h297IBlOtJooDxwrNy9NROyvXNmED0nalE++TJovmegTb20Nac9pxyNPIDX1z3QrhU9V1NMPsYiX7734SY/G7SbvpOhQT1vzeA89faWPNy1TL2BRIw+IdKIPXBPwz1btzO+bhbnvHPFwD0geEs+HDmBvO1uhT2rSDO+xjtCvre+tD04+4u7uKrdPSSqUb6dodC9JfNvvUqdnTwgEf89KMW6PSj6KL4Eoo+7TjjJPlk6c77K9pa9N1ZUPqtSpD3SuYO+uoRnvbiA4D2X7cY8WWy2vTS+hD8WH4c9Sml4veiakT39sYI+/VfOveptfb4bSfk8JCcsPowFCD8NnsI+cFsoPwP2g73Hd7K9boYUvpzujD6rq70+C2EmvjHKlD5S7cm+xb6cvYIPvbvHdOS+R8JYvhdlCT86Vxu9yuzcPmJmuj7pBAo+qd6jvkw4s7zYL9k+eB6JvgcgFj/fwVc+ecgpPv0kGz7tdvs8PmU3PL95Sz7i0p29Mr0jPxDPFD+I5xE/rjYgPolS/r3AhUK+lKlBvsGKHD6Cn489gMqcPvZlHj6JVey9P3qAPnMx+T2t0My97iyvPn/TBr73Bfg9x2c/voJYqz6MvwO+","DyONPdtnTT2gllU+L1bGPVIoDT1Arvs9fMipPdV0Pj557ok+468bPq4hJj7jzqo+Eg81PYOWaz58QcM+5p11Ph40Fj4x1xA+y2JCPmyePD7Ov7M+9meBPvY0o73QdoQ+6c0DPRVWRT4SD8w92kRvPpHGPj6vI9w9Y3WsPT9KMz7Gbpm8pJG9PSGfaj57nKG9f5ymPlSeBT2NGJS9r0hFPgDLK72cFH888kCQPrGvkz19geU9sm+DPnzBMD55npU+KPynPYKevT392/s4pXRQO/4GADybctw92CnbPetKGbsRoRo+e4cAvt54k7tWHvw87myzPiZLLj5zHVM+f/PuPcz6jr0ziN86dxfsPchIiLvwxrW91yVcPNgy97sso7K94aOUOk6WwL3Vxts9dVyrPeRVs708R3g9hNKhPWLf1r01ryU9JLIGPtK7T745LJw8WfP4vaRc+D2DTMW9PC/8PS9s+rxfqBK+523wPPZmsjuPquC8lFGkvQ+ovDwdQwc+zs0LP8eAcL3MSzg9Al1GvZ5rmz2YYZ89iP0wvryMDbxBGly99HKGvX3BiT1na1G9ngIPPnxPgT3zsju+7qk7vbQ27bw2GYY+Qg0ePTZ+yz2/UF29O1gIPnSeGL6MQaa9WndwvWpCU74z/fw9zlMSvBwnIL7eH4M9bcEsvTUvBD2OwZi9RmlmvSIzrTw5ZKA94IDNus1wkTvuWgG9tIFfvC+vqr2IY7k9nCIYPgVrUb3OWMm9NH6vvYxVEL6pvAA9vJggvG2MqTwvh3q9lFclPT3RlrzZnaW8DVfbPAoq2b2JL2u90jTyvQaIHr5vWB09z8aGvW9WaL0DW5k9Eho/voOz+r0ykvc9fr0DPs2HXj6EUUk9LM6+PW2MJL2Mmvo8eM2vPXXMa72SSce9A2BGPY07Mj4xmA0+rTlxvYv6gj3KV3q9V+crPcZgLr7LzSw+tLA5PaOZpz3J/GA8ScmDPVbpvr0GLF4+9ck8Ozp+fb3nTRq9sJDfvA1t27zqOtA9","nG5NPon8tb0pkqY9zPdcPsaW0T34RdM8kSuBPR5eRz6+MTC+tW9YPW9QLT65h7E+IPK9Phr3RbykPhU+obSBPuNUYT7GSXS++iGavRtP+b0lNjE+5aarvRaB2z1oA4O9i7Qpvqlpg70zJBk+sGEpPhtToj3HI8Y9dTJOPvS9SD0UspE8BdP2vdhVuT3FTww+A6u7Oz6euD70kVc+NUvEPWsyVj0HEyM+V7EsPs6gEzzLkX298lugvLEFnT7n18k9aXWrPoILIT6xHiU+Q6AMPspSIz7Xj4A+EfogvIxH4j2j91G8rci8PhxOFD7O28c9V/dQPm5Ucz7GRTQ+biC8PZDwM74w6wQ+3Ih9PUdbwj0sPr4+0LHUvGdjoj7i1oE+Ap6ePhDlcb4ObBA+trqyPVc+4D6JNNs9QH3GPaCAO7ynfRA9p2UpvQqtNb6AXyO9j5S9PXwrXT5oqjQ+hVplPfGNnz4uDX8+rMoKPnN1tj7ipYM+7LF8PmS4nT3EWNE+bTRdPvgC2T42/F686zYEP0L9rT4dIzo+3F/KvhJTxD1cHec9YN7tO835nj6Q33o+whkOvtjqGD9Lcto9CerHPayDhj75fUY+Pu2ZvUtAgT1eRCk+JqbNPQT3CL9UZCY+JLV3PmVWpz673wY/K+aBPkXsy70lOfi+T5/jPqghND5Z/sU8HSPLvbhuWjy+NiW+Nf4VvmjkbD6T+LK9D4L3PVof1z3Kb869zFJavWDpXb5PDSG+OCN/vdECrb2pXbU+Kt8SP0srNj461lk/4AgwPUyz2b6qKwu7iH48vX1+gr0eCsY9o/RQPSo6Jz1A0iE++ZlnPf8tBr7JsI28kgE6PXX3VL4daji+zuaHvhYl6T1c0bi9YfkDv+eJNz/wADC9qqJIvfVn1z1EsdQ8RhW7O8dzhj7sWSU+I+y5voCvw736TOS+DGpbvr/tGD5+ose8ARiQPSrGgD1H9sU9aXslvlLFK769dZG+N6x+vv2Pfz61PxG/FxVVviHJXL2/q8c9","AuKwPHn9gL5mcPi9RoTDPlvGwr2zHwQ+3OR2vUOaXz2X9SS+ACGDvXN/Vj5v4lq/l0fyOoPhOb5cYhI+iAiYvmtiJr7OzEA9C7+9PmAgwD2r65y94UEvPT4+LD55kTY93aLoPKUysTyJOQ++9zsRPjmWG71c0Ia+Qbi4vEJzDD4shc69C2mLPTV3Eb4ZBo69vvUCvksqBD31RCW+2goBPoSAyb3wq9A8izmkPaIcij0sZSi+eKDRuwFwtr0g98K9WD0vvKDrjr2TyW2+voRdvuArMLz1Yaa9BpLYvjJvRb4bmto9kjPvPGpJQ76pebI8FLo9PQjWo73uLWe9oNL1PSmvTL2uHn09szYAPwAN7L2quUs+4ASWPq1Ptj58NBg+eYm0vWnmKj3Q/1u+vXeNPvq+I73s5e8+O75xPY5Pqz5IFJs8+j5APu3b7r3ffZc9WCv5vkLy7js7nMk9D6iNvR4tYL4ljRI+vsXtPektRz26RVs/JD4wvqPBtT7wCuY9fjsGPP5nK76cjv09k9e4PtDXcT8+K2k+8Ey6vTQ1Cj5iUSe/BoZYPJ6zXb+cQEM/Ie88P5kLLL/lWLU+ix7PPgCkyT5dWUc+AMcIPu8bSr0McZw+BwwQP3a41z2XeSc7hkQpPfHAUDulO04/NMS9vdUsVj4fpyW8dIlEPqIBSb9ZTFG9q2VzvYyACb6e+mO+1W1wvsqzWD3A6dK+K6ofvkU8Yb74xwK+1QxLPGrEG74Dl2W+Msk5PWluTb0aQz2+DnUAu7o8Gr2J+cQ+RN+YPQaHQ74MG6S9zzoZv21FoL5aGYe+I80MvqfPUT0Wx06+OYUZvmKI870djyS+3Hy/vMNYGb1X/8a+r9QRvsdEqr4pC4I+8V2Fvb6lAL8c5Hi+UZMkPRaDeL4gNXa+pHAUviZ8K748SyA98QnZvEsMlT3phiG/tFQKvgxair1aDFm+Ro4RPUkyt74kakk9mYmpvX3Ipb4G0E2+0yZjPTfQIb03sd0+XeMkv7LN67lic3C+","A1wEvadskr0tfO69I0l3PBihgL5yn2g+UzumvX+S+L56YRa+nE/fPbKiKD5OSAU9M7evvTAvtb1QZJS+u4clv3LM/7yMcbI8I+xXPqRyNb4Tvlq+J1BRPdaHjL3UO6g9TpQmvoKIjLy3Jic9Rs+kPSJ9JT1IMIk+xjwKvtMi5bzEHQ0+oYgsPoM4Hz4FHso9ppa3vtWZUL6ga6a8naUnPnQ+K773oCq99mfsvanzBL4X9r4+CScevk9Lwr1uiaI9j5HcvoOfJD6x8kW8zfqjvd8sfb0LD727OuDlvcD56TxjHLK9CljxPYtJGT5LrOq8410LPzJxEj6nrQi8ta9OOtrDFL6bBF89vLpCveJjA76DWD48hncCvfxruLt1ohy+bxKDPTF8HLsuG8K9ufF+PkuIY72tHsu92ASgvH7rsj4UVjK+fOsNvN1dZz2AFyg9NTGoPKrXAT4BuoK94tQCPpO8DT3JqWc9HC3JPVcBnj1fZMc9PPwmvnd0hrz6iKK+4K4JPj43jL7pH3O/8IOPvtrCQL4UWBM+knWHPm4+Yb7P+649fzigPSrT6D0hqpg9tCaPvqV0gT2sZMU9rugJvUBUWr62htw9cBBPvoHhBL7rdFu9fXS9vZpqw71JCgM+4vDUPXnK+T2+ZBU+u/nKPOos8T3ildY91+MyPiFtKT3NQGa9AG7ePebt6z594Z6+kcpgv/VtTz9l9ec8GMz4vhwci764LQK/qKHIPpf1u75IzWO+HlAKvrZhdr4UTaS+vsfEvTgTJD3BmY49Wg59vzL26T2DPZm+69UHv2FWm77y45e+G4Ilvju7d76WIIs+D3qrPWIAmb5RSiW86fXrvEFksryrP1e+B/IOvkPXyD76ngY+Q05BvpOn5bu57Ec9EbiWPdTq/D7X2+6+Yz27PZ6Zpj0LrbC9xSnxPDFqfb7numm+SYd8vr2Gxr46naS9G4vuPldbPr4HR2U+fcaTvvAr+71FmuC+T8lHPku/8j6hbmu+3DlbvfXwF78Je1a/","fMi6PUm0qz384Uo+CuUbPptoiD5/Acg999z/PRZE9z2e75c+UypkPvgu/7uoX6E+O8gYPtimnj2U95Q+1YmgPa16ND5EjmI+HZiqvBCTRj54bWQ+seJ+Pv38Oj1cxSE+xZakPess0TwZljQ+gUczPqTMaD5gmW09RtskPrM2ej6vQDQ+WZYbPgi/vT2oP+49frtZPnpGQD4Md049bt+gPlNwgj6viS0+RYWSPrdHpj2G1jG9gugXPk3s9D2gdRo+ceaKPg1NhT525Ns8TbSTPfs1tTwuN18+iuW+vIZD/D0L7H0+boRIPoaPgD4BAuK8wcsqPrCtmT2zGGo+iZCZPQn/fj0XKbS9uCfMPLkLKboqfMu9bILCPaWAv72RMju8L2DxPcppIb3i3AE+vLoMPoTxBjyz5Qc+wUS9vNpg57tpyC08e9a9veZbHDw8PJO7eHsPvhjUI73nLSK9mQNive7d3z2Dp1Y90tsMvp69Jr5WnR0+eSSXvR80C73JlwA+mmyfPJKy+zyIo4u9D9PAPG/GE71w0yG+ZJG1vBozRz6TZzE7xLo0u8XH7j0SIpE9cZloPT2Drbx2J6u98fmVvRujkz1wLKY9SnyiOnu15r1OEWe95auGvdpsd7yZ/wy+JMSpubOHJT37eH49g87iPeFNvby02ie9adk7PnEXED4AicC8gfoIPeYRHr7p0u094QMqvrUJabw2p+U9q9ijvdbvFb1oR7O9KNb9O4qMSr0S4LM8OIeWvTmp/D2aGis+kJY0PpY8jz32Nxa+Kb86vnseEb5supO8rMiXPdkZAj3Bq6c9jQ+wPW1Rqr65eiy9eaanPfStHT6tobg9AdLVPVSaob2tWro9yt2NPdnP6j2c07A9CIgaPhJL4j2rPg4+kYsbvpa+u73au4690i+Kvt7+l71dqPc9wefVvRlBWL1vb+09nZi2PaRRqj1NWXE9bNEpve15ab3m3zg9UmZdvZZms70lG6y96v38u91v7r2wJKC9Dd44vL6T5r1oGQo9","gNUFPo/vez1u/Wc91h6BPRvoBby7MH08/iCMO4p+dT4Tbwg9l8pLPEhj3r2+FD29qnVMPsP89TzOCZ4+Y4iaPjVA8LslU4E9qV62PTXGn7zKzLW8eraQvTXtzz1KVEo+wyQiPuYPdbz/4Ao+W2riPeQjQzv2z1E4T7gWPsQ9Fz53Nvo9u7BKPkrrqz1LNOi9nrJCPZv8bDwRlJa8XzOWPqAnrD1e6FE+3ewiPitzwj3VlhO6/7YOPi3Vqj3iz38+v5GAPkTCQD4gR2s9lZw8Pg10MD4asCM+XseAPXT1dztk9TM+raCNPmFlNT5kO3M9UnGLPtqzXz44TDE87quQPrzWUj76oAs+megmPkfxRT4aBYU9OjzuvE4hkD5tlnA9xn5NPlISkz762RU+w1QHPoBGtj2W6og9jMSWPhAemT4DEq0+dabYPR92Sz6rkr8+gHVGPrr07D074r0+CIIVPqhnA70j8yQ+4CsuP8ZnlD779r69gAEePSnD2T02Eps9ANYXPbrMfT1dkyo+yEycPLTzET+vngI82bzAPkuEBT5IiS+9PCvrPSlkIj6ymoU+5PUXPr4WSb1/tzo+pOMKPlLliz3Ydm49pIl/vdNxQ72sNSc+kESYPnP/Gr0ObIA+TCWvPWqYRz7hpiQ9FguNPeGHXj77xxM/sVkyveIGXz6WtYS9Tj5cPjah3j0haOK93gOru7ZoCb4CR9g9Y7Tcvg6D8j2vh7u9chFWvKO8+LwIVFs8HkY7uvLhjrx4ta89UyMxPurLMT30b4w+PfEHvvrXXz24gvY90AEHPqr04D2lSqU9znOvPEgrSr1Hbaq8+U0OvqHXEj4NjK296SBDvX0/rLzcL6o9JIlEvqGMG74TNTo7UPgsu87+Xz4hpSa8dEfXPVuUcT5Xogw9er85O73XcD3E0nS9DcHHPWG1y729/Yq+/uhvvEkvVT0y93S9cG64Pp32Db4nLtY9oAc3vFg9Jj6MgY6+PMXXPFc9Dz7oPpa9N+pDPjvgDj1MYCG8","9rqPPcZnDj4YSx89YOcLPESFpT1S7aA9eLXJPM8DBj0Bovg9jAJdPY9J/7w36AM9O5kfPr0JJb5NPAY+oMNuvQFgkT2m/D497kKBPX/bFL1zCKs6cLqyvbYoxD1Ie989qEP+PF4y1z2OUPA9DCq0PCtZ0D0Y+bk9FE4JPl5sQj3Y4hy+OVHZPeatKT/bIEI++AfaPLGlLj50KQK+KUWpvUyOjb0/Dew9mOstvf66Sr28B3K9PWcRPlGxET7egyK+TmPnvRyhoj00Hgm/tCXBPfCm8z07x7w8V4OBPSKAQTzRpUs+gswuPZniT71QWzw7OgW9vCEJcr4uuXE9wHD+vbzkaT7dE9s98mbtPdd8cT7huuI9hKZPPgIK+j4YAB0+LzinvVf2qr5fqK4+gdO8PYT/AD+oLQ0/SQE1PpGF2z1jOMc9WIkgPgqumz1x4E++gJpMPkyurz7tVz29dRuBPVKSB70NorU+0szZPqcqoz26XLI+7xukPqAp7T3nU3M+iQoePiIHLrzbl5M+3Y0lPkR+VT641YS9iIhTPv/mxz7/xcA+fwVDPuMzPj4Upqw+3C/ePZ7IlT7qKn4+zL7hPeTYoT2h2VK9dkToO8IQMj63WP66a50/PY9vUz5Os58+LHEsPjVJuD6DFzi9yb0CPurgQT5Any4+VKuGPpzg4DqdM5e9+x6GvkKk9D0jWxu+sqA3PNwMAryIxA2+7K9lvrWwHD3pp8q9x8mOve4VKL5WoTC+JiMLvm0GUb5gYSI9YtURvjlE072j09i9cPEGPo5phT3GoqS+ct4FO1VfCb7mZ6s9caZJvhPuyjxQaHs8WnJRvmhMc77sX8U7+23Cvhs1UT3OzQC+q+igvYebJL7t1NG9VlK2PSMRQb6oBpC+KKcXvLI8vT0x5pe7BlN+vftwO74MWWq+k+mQvTEx0bzTxTa+aAmsPf7m9j06qWu+ZY+8vYxiwr351ce9p+1OPePPuzx6GC+81I7avQRdrb1KMfW9focgvjLkuL0G+Ls9","cKkDPYy8wjug1IQ8zEmxvAwb972mCzU9+R2avYDgND35wY0+i7bvPYqgkT1aZOk8gQuevUlaAT2tqdS+kmW9PdAY7D3l+s279EYWvbGlsz1UXUI+/xVLPgcQlz05OeI9IJHFvmt7lT1GdQi+D8l8PoOPz7zFRI89eY1nvsb55r0EGnO+8+ulPZcUDL7Tzkq+mHshPgqqaj4QxU0+F/Movtm1MTwawhy92BwxPiJUj77MSHE77Ecyvt339jzv1fa7k3cBvuYnDb7SS1e+HF2/vJ3SBj7ra6U8xRdWPomo3D3IBbY9OUFWPg+fcr1CZI69qhyePsuNQ70fEAc9rIYmvtfbUTu6lW0+2dw4vo0+eT269Aw8J7HQvfvUhr5j4rc9+D6NPYDdTbyYRw4+1O4cPc6T7z3r1Si+L1M4vzjU2709Dzm+MESbvYU6Cz4x73Y9SV7Wvd3PYT0PNkS8yMgGvJCtiL5NuTw+aPQEPgpJW749y+K9BUbKvYghAj4FWYq+CjbtPixMFT6kcGO96CgePVg4Yr0Sfxm+SkoXPSKVt72Q5b6+N9W1PQF8L71061M+vpGOvIj+OzsJXmK8VZAav86HH7440kW9l/64PVESBz0lEJy9fLfNvL0OQz5bDTk9Cv0YPl9Zrb7LcT4+BCHevdBS3rzsegS95pirvaA8gT5uqhW+wVaUPYLAHj02XIQ+1uI0PfuMeTxnjPo9kybmPRLVST6JeNq+EUaDvinLq77GFxQ+7k5rPrJgRb6dsRG+W+9vv3xz5j7sfUY9/O0Fv+1ue7xEjWQ+inKUvmjQxDs1bjm/he2sPW4IKr6L6lW+HLAyPl71fz5LNgm+M4vvPnhYOj7EWxG+gW4WPuET0T3Tp4a+Uj4QPj9A4b7Hs5q+JcVYvpERH70JZAa+32UOvi4hFL1sDBO/6EJkvtr6mr7o3oe+1rfEvSq0ED6WwDu/JWthvsovEb5dI+S+L+NyvcULpz3+7HC+kRB3vV8Rnr4n8GW+6JI3v22mBL3YXAQ9","aK0Evme5WT3WJ0m+wfC2vvsHhb591o49aA/7PNkQor4VflO+rf7JvbPBDr5sBiO+BsssvmvhUr2l72q+u03lvA1WNbwqw9y9l+IRvQywjL6m0lS+shWpvgaKRL4s1Bi+Em3Cvsl4v72lvEe+Tu9RvW8OTL6D3IW+ikC0vp3U2b0VEMe9i1qWvtvbOL6jQje+hMugvqmeJ74zqsa+RI6HvqOag77JX++9fR4Vvq6M0L0Pkge+QnAGvmX7HL2GTy++AeQ+vqmfYb1L95Q7VvrHvr1XFb5zlNS9vPy7vfn0Yr7/bWK+Trq5vUoGyL1/kpQ9LkHHPNqQkb68Qiu+NW6gvhymrL04d3I9NbYcPWOIxD1PYYq9T9cYPXiI1b0Yi38+BsYJvi580DylANa9GNzDvIR0jL0VC9a9RfQIPozl8D3j+Ew8WzTpvX6ZJbxQ+wW+0m//vVCelzy6RMu9iOQqPNDz5LyatLc9qAUyvWs/ILxWieY8i3G7Pe6n6DwbvPc9rdtsPhINRT7lWy2+jbqQvTBBwLzfN+g9ZBwxPoHMLr6HUTO9IfFSvaLn9r27X269MUEnPjbr7DpGqoC9ekiFPXdbLL6DtrM9at7cPcORTL31ywC+wZYOPXmwfTyN2l0+366vvB0U9T3qRy6+yCGFvTuhCr3S2nq9B5W1PJqe/T3jGoO9aqzSPTG2tr2XxX68g1A4Pk+HQL5TL1M8xgp8PW6Uxrz+es666py8u/voaT3voQe+2HEAPiYWQL6q1mQ9W8+zvaa1pj3+kgI+5e8xPs+hhb3c0zk+cHGxPatkRbxZdhc9ixPgvVlJnr3ic1S9lNzgO0A9Qr5ipBm8OkNivLZsnT1pzic+Q3ICPXPUi75v+DQ9kC+nvQkfbzsBKda9qnTePUvRBj55BX092XhgPcvnBz2g34699UIZPVqHa7wQkd09DgJqvZsFR72QUNy9eo9tvVpjG71tN/C8ECvZPFIN5TwUkPQ9SywVviyhPzwjaqK8TAcZu4rk1r3E/qc9","vGxxvlZIoTzEd/49m5ivPc3Iy72sEfM6w3h2vsEyCD0/KoQ+lPjJvdKJUr4DkJu9qZqVvjAoAr1Z7Zq+ry1ovg19671GyR6+CEv+vFN30j3DcoW+1dubvJse+LylFEa+G95kvl4kx77mzwm+HR4jvinn5b08yau+Aeoovrkblb78zwa+y5yUvgIx3r32ACq+Z0XuO7G3db9ebk++OLfEvT/VH767ica8u6aYvugWPj2Pzxs9Mr3wPVBEqr3CNGC+w06evuJ7Ir4Oag++jiXOveqzpTwQk1++VGOuvUKWDT4UimC+tnO0vkU1Hz3AvH2+/MtovR7ml71P/0S+A23ZvRZjfj5fPFc++tx2PUp3WjzCvLC9uGCVvOLfHD0dH8w+tET8Pb7jCT7STKg9bN+AvAGb7T01j8k+bpukPU9vR72HVPc+bXHnvW0Tfj42hzE+q3PcPdkERD5CK/897rBUvT82DT41x0s+B+vYPok83z5mhY89YfyOPoTcZz1jRS4+drXfPZvNGz+zDRE/ioCPPYrbVT52fJI+EHNLvq4YujtexZA+teUdPpjGMz6l214+MzhTvWSGJj4vkqs9/U/PPqxMcj6IeYG9styTPuiBKLwqZMU9RUbMPeoxhr/8xMU+jy5pvqtWbD0esZk+K5rQPnzmIT4oa62+cCrJvCmnxD6MOLA9j3o7viBZtz1sdyg9k1VHO44kzD1SnXA9/IbWPhXlt7y/oO88gkuWPSAqWj2PO9C9XukGPMlTFT2o5iy/qS5nvqCiu73Dtv49cEUDvrxHjj7iKIm9Y+Q9vj5IL70Rp+E+gR8VPuB3Zj7izWq9iQZTvpyRnT6cQDQ+A26zO+aq0b4KAPK+aQpdvq+AeT4X3+w+HCoJv7oeez5Id7A93MBcvtUgmD0O7ww+9tsEvswQdj6pE+K93Qt+vjgT0Dwe3CC/5AylPGUjiD6G4ty+0ksTvigz/rwuzbY+9jMDvpyWdL7p6eo96284vOFDSD4FoWm+1N6gPBCPE77WocU9","L3t6vYddgb6NZ8I8o0VQPQz33LwVSvK8MOkcP8XDG76qCt09MH+WPpV/QT5ybEw+tYRuvh0K1j2gr528/gisvqxdGL4Ujmi8VGHCvotnoL7O8m49mR+wPJgOLz4AdRG9i8aPvCBl7L2Cq1k9y0IPvoqRiT1BWJu8IHdPvmPitL7OXKS8AD/yPJpYoj10JxU+VIVvvuPFxj3bkAu/CIKTOVowBz7+RZk8jIsfPZFePz7Pbou+ahqIvTvx6zzm1c68Ph8jvssdBL2nyUu/P8ZMvhM3a726hLa9hSdWPsJmqr7Isg++pouKvbmjsryyy4w+WqzYPfeT5b0SmXG9LiNOvikkV70QHK89D9X5vAZbyD3lli++ZEC9Pg+qhD6jRKu9Q2ZOv8bKP77xVPi8Rl9MPiWR7D7O+1++DxrWvRb7Pj65L969UcDUvWE8ET4xRRi9vqztPbBWLz4D3vy+7SycvgnU8r7pmYc/m5W3PWQhpj6uTFM/yd6rvlBPET8iBDq+ydA1uwKUZD/HhRs+ZFdPP3GIJj/BGy4+DWELvBfGjD2FPQk+ccQuP1cxHj7/ekE+v8EHPsJkKr6w06c+Jkg+PqhsLL45laG+ipt7vqaNGT0s8Sk/AAp3PnAnuD64U5g9MnffPvAVOD5GoDk/p3fDPpjLgT0J0QK9VrCZvdIOrb8ft4y9/HJ1vdTCjz2s13o+WooMvnNzED45eca8whuMPtMst73CRTs9j/spPjq3bj0PiyK+iN4PvY+svTswtDY+z3b7PYX6gz2TUVg+sgflvlph2T3Frq8+TgYYPQJsUb7xZqA9qIQ0vUDQTj5f/i6+x9SnvWzWyjtmQlC9q5s4vVAiozxpUpQ80+3HvSHPzT0DKtO9GdEPPGsrSDwk5lI+82IGvrvGZL7Xq369eXjUPUMlAT8WVc67nMF8vXq8/bxWE0s+TcdnPPs8N70nUpA+ngZaPuMIMb79xY28L9g1vSdIgb7mHgY+U0KtvRj6az49DAY+h2WmPcP3rLtaipK+","8DacPfm/xj0xrOU+ECPYPR25LL9tITI+sQxJviKZKb2cn6o9AymgvTzAnj39jC898PGaPpzfoT4D+p498HWXPyYcA72a34+9K5FyPbIE0L4QdZG9kr4uvin6DT74GbU/1/iJvddCC78lCzo+chZmvQKhRz/HdC6+MolRPhVFmjyA0OM8P4KovYCDJj7MSzA9qlfuvRgoAT1Bhti8MOobvj/vIT6oPGg9Ek/sPUpBPz6vg/48CLn/vdROqT7N1na+MK+GPvdPnj0Dsh+9tqXIPeWqpj1CyE89Cg3fvcwPibxwZ729AU2YvneQoD3HWse8S0ZKvQAUC7618yQ/iYxxPsRMZr6rLju8Zau7PvWXIDoepvO9bua0vi/K6L78Wam9A9R6Pm80vTvfsM27JzHLvf0zmz7TMd09bJzxPYjRkj5vJcE9i4mLvR6CQjvmQwG+OJOyPabJ6TvB+2+94lLOPvPJoT6gSBU/bhc/Pf/gBD2nIVK+CFuAvMgjXT00N4I+S3aWvdcCNL7+Xci90TIcPgqsED6DzsO7Q3aGPUybz7w/4wW+2qe0Pd3OXj6lchg8WYIlvr/t871pNI8+0qmmvuA2gz51uq89hPOIvnXlEr5Ld0K9pftbPtD5NzwJqa++/zy8PSc0kT6kX+q+JW0zvDYkLLy9iC4+blW/PjoGd77wFDU71bxRPtJ+Cr2CaeI+1DsZvmGyoL3Wp1S+qPKDPlzTQr45xFo+H8bivRKrBb6BQEi9+EcbvR4utz6FW/k9EHEHvm8ss76VbkS+Pf47vYF32b2vQ3o/KmnRvq8fVL4lT1A9Cra/PJsjA71J0sq7V6UHvqmR9j6OefK9sErlPna3fD+we4G+wm2aPd+6Gr2rAKc9fxxtumapmT7AaDM+GF/NOldVb75UcaK8w4U/vI4MjD5wUvG8wvMGP1ujJr0qhG0/tbP+PYLFuT9LQB8+XmFUve4QbL6nwYU+0016vTnfOr6Acgc+r5ECPjA2iD65HEc+IU0MPWAzi752Q9o+","RgWXPA4FO75kzqQ942uQvXQwiL3/ccy8bHtwPahlWz7spiu8HGRuPdgPyr7G7rK9nfqKu7m8jb2TnB2+3UIUv3N9QL5H6/A8DkjVvvZEi7xuiZq+dHEivleM/T2CtJG9DQZ/PZCTl71/j4Q9JDlVPdHw7z045ao9FL9LPhSRJT3Gh0c+01+oPIGL/rvKhrQ9Qk5GvgiQAr66qT6+6nfVvStyG72O3pa9oisjPsyHDL10Pc+8zwIRvp+9or4E0o088HhGvpgCkDyptO89KU+4Op7zy70jdkI9up/3PSYyrL2FFSQ8fEjdPCHFg71d/a08fCHhva5FRL4nOZ89Dj4TPiuiDLy4iwG+nQXiu3AYXD1sd6Q+vggTPbUfCT5sh9u8HOf2vektXz5iWTu9alsMPjGrkzyTA5e7i4ZZPdoYRT1cSGQ+IvHkPeI5NT4a2ME+fTaKPJzrZzrWEkk+wiFCv4Rdw73ARDc+N7UevRZ7mrxaXjc+ie0GvkuCcr1KT5S965CmPRJ4Pr4aOUQ90xHEPDfH2rz8hza+bWVcPC4pjz2GKCO+pGA+PixYOL6FzcK9qiv5vYtovL1ihjQ+VY2uvmIDhz50pIm9FSabvTj+QrwjPao+wqKUvv51075V39y9cFIvPloUZL3HVR6+Fm/+PS3/Vb+Xk/O8rgKqPWSOkr7RtTg9xqiDvmBO1708L5O9QyWkvcPWEz64Xcy+yJrMvmJihb72lAu+9F8/PQA3UDzYR849Wo0SPpFYFr4BJlC+xKVLPb19E75cBHs9LrndPdx4C764Smw92Le1vR7VBT54uDW+1kgbvaDKUz4+8kc+X8pAvFchSb27xei8lKsePIuscD7TkKI9hOIbPRIUsz1sUEK+ZFpdvl6WIDyiLjI9d+P+vGnDT77pCl2+EqJ8PnYktz6NtnW9yFEGP/M7oz1rXUS+6V3BvcB8FT5c+ps+1tyHvUj9Nr2Y11o+48d2PiyfZjxFFuE8n5/APET8vL1cboI/cAByvfpXjT2rbzE+","wIgzvkK0K79gPlM+JjhSvw5XZz0YXYs96MwhveJqsb3A9YE++eWuPWSZtL4qM3e+6tiYvgzr5b0MaAG8WRBYvbmTer4ZrqO+2tOivj1tuT1nBIu+JJMwv6OU9T3I8wa9Cva+PlXta755hHu9pMYsvkAWtz1hFMY+GrzAPUp6j77VOve8TefXPfQXx77B5qO/aa2CvWCKCL5KIu+9aYaqvVH4Qb5xdYG+67S2PQ109b2nc/g955PRvgM2Yb72oKE+UB+bvsdEZz6GLDy/QT4rvtema76WlN48L/A6vl9Qy7znqvu9sR6BPf0Rir8IFzq+mo8gvVYmqb7irrs9xpTJPiJJhj0ve+w9GgzZPcDeJz6LQAA+fgEBPi4DozwX1Ey9Gg0RPsHP0Dz3+VC+QyWAPiVgULt0XWo+wpLZPfGuqj7hPHS8bjxTvZD/Rz70J/I8n7G6Po4DjD7ApXE+JVh6Pu7txL28AV0+yX0BPl1UIb0QtBA+wdoOPl/bED5WOFM+on0+vc5eRD5kJg+8blSbPeUZrz07LGM9d5/pPQXz1j3II5s+vIhhPGGWlj5gWEs7C1gGP9ZdBj6RKss+IMVSPlknVz7FWTE+wrZfvyvJMDzZOts9Aq+UPdwjJ77aZG07mprOPDPlOj5Es0s+54bkPWd2iT2vMoY+tZUePitXtr7aZi++gW/EPUHf4L30rdQ9TDUXPWI+KL5KgU09XQiYPeywwT0twL+9ao7hPMQtYL7tlBe+Rj0TPrXwvj1y+I6+qDJFvrabYr7JB1k9PndqPmaAFb6Wbom+IDPmPFY4Lr4WTrG9eZRpPBf9mb00X0e+rHZfPXweeD66WQ6+r6obPbna9r1Sdre9lCwTPgFM2zu6AuI95k3XvVZirD3Hm7884B32vMxYHD5WnJA92xaeO4jAlT1a1oo8zMV1vgXnhz6ybfE8ngZIvnIIHL7ONRO9NJJhPY+dED27DeI91g4jPoDJ4z0Jvia+DnssvDgcJT6VhS29xyH6vfGzbz39KTc+","O4B9PqgnEr572hG9K/IsPip/hr4jAkQ+36x/PiYIPT0vDAO90rJXvY7Mzz1s40++JQTzvgdEQjwrvgY+Iy04vpmAdL5q5Zw8IjD+PKrvCL7oNw6+je69Onyw3j2q1329rcQAvWqAZD3bFLq9wL8wPoZhkb0DaMI9MjDmPKsqsD4ogRq+WN6ovS/zk77alDc+TpAIPuxD2D0hhaO8LrX9vTbYxLw6+vC92804Pdpgbb5ehTs8Tu1NPvJWjrxaizq+UZMGPhyZIbxuwSy+5BN+vS12tjt8kFS88p2IvX8Enj323ak9STipPrLQ6D2aQe+93HVUvjFQijxJLwo+AiwHvX/b8j0ChiE84sigvVcacr4RxKi+mzAKPlG/BD6PN/g9G86gvgrDUr3KZPc9h+NsvPAIiTzw7ge8LgnGPTnrkT4rfY4+i/hZvocXW77d9+s9q9MIP5DGpr4ck0691TWYvZyKcb7sk1I+l5dsPotmljyHlVU+xJ1EvnrXe771sRw+GR7GvRZNCL++lGU+gyLbPD2mMT6bmVY9v6I8PhiXKr30lsQ9G8mpPtYYxz2hcC29fP3lvXAq0Tt9Jg++ArByvvhITD1kM7q89dH0vbT0bT4vtQA/G3SMvFgjRz3OEAQ+RBypPsI8IT6To5A9jGOOPV96PD75XNg+eyOavZqaCr/wSas+2QX5PYRBcz4KQbE+Xm/jPbJefj79WXk9+dS9PWMdTT4v0rA+weLavAwwvj46BXU+ba+kOdZlAT6w+Dc+9ZF0Puip7j0eqvU9dMd0PtZF1D4ZPiA+w5+aPrm7uT767lg8d4GGPs4Z1j1sVes9e8/2PbRRGzvXzYc+iSeWvYP+hzyO5AK+H6WSPljbQz3IxqI9ZWpwPt9EKj65/Yg+WRdQPuHsrj6lOKo+Na0APiiV/z3w9ts9/UO5PiwULD5vOYQ96NE2P3W0br3I+b09qJO0PdJ4qz6s+Ky+jWWUPgxh2rzbdq8+Q8Rnvfvs5j0kzw4+Lbm1PYMuED4ccrM9","ozP+u1cPvD3yN3w9huYyPVzzDD1PwLQ91I2FPU96y7wM4aG9U06FPUvgBb7Y0Le9TyucPbiOcjyl/ek91GPcvUY38z1KC8k9u6YUvdG4iD2NkKQ+hxGpPJ4w4j1PuDK99R8BvW/Ej73dyoS9mdftvTxVqT144mI54cbuPbBvXz0wUqc99KibvbJOqT1/OwK9oPMTvqarPL5gUL+9BFfTPOy0LbjYORQ+p2QnPqTH3j33aTW+CncOvE6Tvr0Tx6w8H8OOPH0eM75Jxds5vDIFvd8uAr7QqZE9y255PsjwEb7ZGwK9GMiSve4VkD05VrM9kgeOPcgQIb1/9CI+oeQRPCXzZ7y54Qo9n/HmvXJaYD1s4LU8GDkSPjtmeztsRdc9YK34Pd8MbD0QGN890pMhvCwiP748PdY8JTi1veHdD74Bfeg9XM0QPVrXbrxnSDC+CeG1Pj/3GL6O7F28uMQLPbUWPD1LI5+98lNBvu8RF75jZcW9dUhOPub8MzzziLO9Ey1mvW/7iL547fo9htbEPQjeYj19idY9A6x0Pi+ZJz6Og1E8/ZvAPdYIOj0yC6i9hGZDugclpj10doS9AZ0ZPSoiyb1A/cg9bqQMvmXjKr2ENI49OY/7PXZ8mD6GyeY9EdcDvis4hr24h2++xp+dPRxyQD3vMsO+j17iPdKfKTynjJA9vcLrvL9Hyr02bHU9VkEUPQG6gb0d53w+HgaWva498T0CxI6+Yi/RPe+5Wz4Ot/u91Q1yvdqqKj6WcOI+ZxuqPrlo+L39tIO+ip9pvc9gFz5677y9Ft5qPto07b2Fxbw7Fpj+PuqLsLy3nM0+gWv+vZiNOb0M/YA98hwZPvYfGTvCUr29MmFQPh+Z872Z/gi+02GKPt3aTD7anyK+eE20Pu20zT4b8oM+ztVZvq+ncb7c1M4+axyYvcGWgz5SK2i93M2aPu1nSD6WQYw8axCkPgq2Zzy2b8k+H4wjPYjGwz6r99o+qQ+pvkBtgzrUios+SkxUPg+2Fb5jOec+","IC+jPb/xQT1ToY+9XuA1vsEMCr7pRra9W7ICvuqwQj61A/G9k+orvltpPrtoC2Q9tdUJvghnB76zBFm+2FSVvAGgBz7loUO94Oo6vjVoYb5/9Fu+euBCvA+PC75UMdU80+VovvFcfL5oOLy8932fvgjDODzoWW++InKzvcYrjLwqr8O+6m8ivsf7t77b+ei+55Ewvz4FdL6CE3i9n2cXvjbSpL6yEyq+SmmUvriLtr2LlJo7UGgXvcIIIL3G9g++3Ty+vlpYtT1vzc+8Uk6nPBl6OL5EOZS8Eu2TvZclZL4Lwgs+Zz3KvYaNBr+buHi+YUy/vunsIL+d6Se+0Ystvk0BIz2Ei42+Ly3MvcAH4z7+dIK9s5SGvYS2+LxxYe0+JzhKOQ6ykz7ppzw+d1d7vcYivj0ngaA9fyGIPZXjmb5Ntx++YMPavOZtUL96xUO9sb6fPKp0VT1cZR+9oH2NPWY6Bz79mDQ+tkmAvmDxlT2bOec9+izgvYss2j1ySCY+Kb4FvWvOI772Kmy+AwbKPsQYlj6UCe8+Q3hhv0/ESz67bsa9jR8CvTXA6r3JoOu9bXxAvt/IXL6K4HM96wKJPYeZPD8OpLm9WkgDvmiCwL3f/Lc9/Tlvvex5SrzZOqU+YZiRvQQQXD1Z3Rk/VCo4vgasDj5AArm+uHiHPfP5f702uZk9FiCHvakaFz7Z4dK+rR3+PdOuYr35PxM+CHCzPQIlzb2tUW0+K8YTvjYyCD4/Lh4+NR6hvGrLyz34ooE+0xJmPfMwvz1C9Zk96YuevL8zaj0iRaA9ZZJlvpSNz72yL9Y9dJqWvQaboLwAzsC9Ld6fvQwpdz44iMc93EiovWZkyTxHGok8zsV2vq5mJr4ohQ++CMr6veM4gj6gA7U8MCIcPNYKdzztGsw9w3PVvVsKxT086x29JTOVvZsGPL31oxM+gEtoPnI8Zz+AswS9HkATPjvQAD4PURe+c4UMvUu5B74xWhg+vwS1PTgZu74vyNC9oQUyP+JCMD5nffI7","I4RtvoVoUr4mXKO+0Kq5vebc6zx3Rei+JIeRvmVCxDysTmi8HMWRPmWqjL6BQLa+u1bHvkr+qr63ub09K+iTvc4jmz3Sg4U9Kbdcvi/dMz7nTqS+GPB7PuIheT4T+kg+8REDP1mp777IeBC/mKGKPayPDr8PJri9DF+Hvio3/L2BlM89j1+VvlY1Sz0Bsb++8RSlvpu9g75MjGK53mBHvunwpr7qZOq+4Y6nvu0gLb9Ed86+C9uLv10lR72ByLo9l4AivHeT6DzpY7w8HilGPgSKyb7rpjK+6fYavtKRYr4Q44m+uSSGPFJ6rL3/LjG+lwFzvpMRYL1EFLo99tfCPlzdMb3aUky+LKgmvbTWVb0Tvjq9O2SYvvfQxLwmbda9YEY3vbT5Br7MuS89211ivsgd1b2UKnm+FcHavpYT4j4nosa9mFqpPC1npL5PQrM9rJFwvozMpr4t7fe9zxS1vTfI9D3uApU9eY0OvVzLtb3upWW+O/YtvkaZNb0zPLu9Sh/LvoSFOb78trW9b/U0vTssvL30QHo9nix+vVGOS77V4Bg+d9pmvctF8TxHbZ6+hSLNvpi3sb1+p1E+PVB3vZC2TL6rRRu+lb/SvdXKnrx0slm+H+8GunF8hD2VzSK+3ABEPQ7Jt73TZPm7733uvWuwtr5i/oQ9WWRLPWRCBz0aFAS+wq/pvHG+ir6C7nM97iEsvEKeDrxeAvs933LpvPiEBD6XnUQ8FJPVPfrYGr7otg4+52bnvRbpoL0ZsHQ+ki2JvStw/jwV6wc+ewm5Pakbab3Fafq9zfhFvoTNMz6PLZm+9vaCPasXmD13xQQ91wXRvMIfAD5gBg2+QGnEO62ZUb6tnyg+3AaqPKLlaj3s6Lk9MOrGPKyMCj6aI0M8R/2APWtVgb3Ekec9VD/hPUMogb2qEgm+KXZHPlGYiD3wlTG+pDATvartkb419LS9TfAhvRnn7TxobQ0+mZDaPPGcej0VoQI+IWMwvEKXpDuJxHE+vjuWvUHIfj3nPiE8","ReeXvKRI5z2i0uy+Al2Yvc3n2TtvFZ296stgvTD5eTyy0Q2+ugcGPh7CLD304Ai+ShravYwETL4j31i+FIxMvP1AkTuuDIO9/usYPuoB7D3kKJq9RIGDvgWXrz0yCkQ7Vk2Vvo1qF72CjuO900EJPagXG79VCLC8acR4PlVprb68nJM9vHYQPJ4hpL6CJNq7nsjcPdD0+D2QKbW9JdrpvZuBBb/0GmQ9X/cwPSMIn73tdw6+5kLNPH6m9z2xnkS+b/bPvNejzzpKJf49nLcuPkC1gD2kviS+1hiHPfKt5b0k86M98O2MvlPHBb3BXvU9ILTQPKi5lj3vijW+n2A7PtPiOL4EJtw+FwKHvVCJVLz+jhM8+GU0vgoJ5Dz4rY2+R0YNvfsmBL/yUq++Bybyvmit1L3Rk6q9aIWRvvAQNL48lqO+EDiRPuQkzjr1KuS+0w0SPUedIbtrIpO+sVJPvVEG2b5XRbI9LAcHvibPOL7O+q89NpNBPjOT/L0oHNk+3+t4vr7ajL41o7o+EJKlPeYPFL6VGeG9ax6YvYus5b4nZHy+cwFMPZdJXL2D6/G921CDPn6UA76BZJ2+Q/wSPVLhf74zZ8a9zzYBPozgOr/R1Va+VkckvsXKmr5NFpO91/asvCW2/L2Enw8+wugDvg7OCL8qdVe+UhEdvsUXQb5M+Fq+9bQGPoFfAD543L48oEkPvdoVJz2KlKo+/C7JPz9nOD79qCg92C/yvbW/Nb4Bq30+A9DHvb8VYz5Ip9C+s3UcPqZIdT62w+U+Tts8P8Jalj4UpXc++KpvPtZ/CD7/aci94BwjPqkCtT5x2hS9XrrQPp1FJT7HdEk+QmCFvqQh9T6z25Q9RqLTvhIf6T7Xo/+79LwTPkA/lT46v8U++4NcvXXpCz75llk+Y+IEPm2ssz67Oo++b+pNPjWLuT6kfK8+z5xAvJqCb7xcXzS+mpzNPo4DVj6YA4m+tAAgPg+4Pz9rPxs+WGGSPqtJkT6CPLQ+xQhYv92MmT265cG+","M28Wv937TL4YUUc8LUoBPlDWkj72vdm+tdhivfsUz774HPQ94tsIPme+AT9uWbQ+JseOPcrfdr5iUXQ+sDFRP94wGD64RfE9MWVGPZ8gFD8A80e9xOAbvn8jqD6xmvO+5U1BvhBxo759d4Q8zu1/PL9glL5NJiu+4GwuOxPQ1r3y/KG9MNZtvuSovD6eObG+ghaEvqL5Wb78Cmo+yz7hPUZNCb4W3EY+WQ+pvRUR9r1ktXw+sJMNP6GTrb1DwbU+a0V5vilpkr1AaiC+bIUNPlR/gj1WzMK9IpehvKBGAT7xJL4+SbV8vkiy8r7DJEI+DxnHPXqDyL5uXIO+I+X8vW0qej6JVzO+ZQM5vPCNbb1dCpi+EYkAP5TpPD5PzZY+zfmXPJ7yxr3euMO7wQobPUSgWD5p9mU+I/DoOzTjvT6cfU6+6gtGvdVfJz5P6TG+4QB1vpqrzb2V6NQ9yK4Jv7CX3L2XFJ4+/zmJvSBpRz6VgDe+yt5GPdr0OD5T0Fs+vTWZPXzsbj6CQie+qAj+PFdODr88pGS+p0VwPvt3Hj1oAzi+1HqLvV43Cb4UUgA+sYt/voBHp75WwvW+gsiwvQvKgT119uG9NdkhPkuLST46Kte9e682vbEYSj68IZg+SK+/vpmfBT2UmgI9e6/KPGWYBL5wiBO9x9povjeycT0Wk7Y8iYGiPb3rUb0rUZA+EJGyPVKXdj5wDaI9KNmkPdrtmz66IVG/1vqIvg98Rj5PBWk9UdmAv4jLNz+ntwo+Eu6cvaNUjb/xIra++4NlPkQod77WKp67opOjvlDwGL7fgQ69QN8YPdGXxT6mg+E9BbCIPwkMjj4efwE+pt41viQvTT9IeCG/CRgZPgzr176EYBC+mXAVPn1qoT69v9++1tkWPyRYCz/X7lo+08i1vuwFob4XFjE/b6bmvRzDVz09hem9etMLPppYIj5Mh0o+FOorPrbJkD5JCnS9sJitvS6nDT+r8vQ9bbr1vaUktb1NhdA9nb1GPlHROD47nR2/","G7xvPXb6ML2JIXS+uSSivh/Fhb2qG2G+g2mvvcTTcr6omoe+xnTovVVzoj2/zLC+yLOwvl5tkTyeTgm+iroBvik8uDxhhmy9D/XEvEtHqL2eKcG+SiG2vmwSGr4MbzS+BTsMvgifPr6HB1g+mEs1vnYWIL5S7zM8bBTEvuRhor7lPoK+AUpQvmzWV77/6Wi+cd6bvJSIRb64XjO+ML3Mvn5z7r1WNKG9hSeZvnVcQz0uqQS+DT5jvejCm770Rgy+yOrXvWswML5Mhiu9NgS7vR3pHLz64h6+6KvnPextuL2knIW+48O6vs7QBzy+Tl6+7xcSv2MfhT7jA6e9adtrvvxZv735vQa+PQg3PrgMxr0PPMg9AL+wPXhUwr3J68M8Pl3NPeSlCb00M+U7Ukn+vQM6Kb4q6jS+vAM1PgeWLz3T5eA99B97PODyaj59AfI8TmvbvZRsaj2pZs69OfbDvTotxrwxIfu9sewtvMj7ND6edlW+Aoc7uwYt9L3/gRa+NVnRPTIYtj2kdpO+EvSjvfVGiD7rE5Y+szetPtlI5r1hjfw9uYWCPD6yAT08fII9Lkn+vckXIL6qZqi90a8fviuLmb5Wjve9qvVRvaYKirup0RA9MCxXvSx7uj3SC4O9VWi7vRRF7r08SYc+GuWDPeo/kj66W2s+ifeMvaoKAb6sjKa9wgEXPscDfT3d70O+TYfvvJxEND1D/mg9WL0XvEna4j1nVQQ+ZhC5vbv1AL4u53G8cvlguxKsBr7Z6tk9ukSXvUUFID4xDEi+b+xdvXq0HT3T9AO9f3MQvkoduT0J3++9RqdTvfF+9j3OWUq9DrxJvfKrsT0Oqfi82ZdAPRJFAL2+sz++mr0uPgAMCb5Sc/y7gEguvXuxWL5mxS6+53zrvdolgr39QYo9Lc0buir+uzzZqjG+zS6kPX5C9zyuS188QASpvW/eOb6OX7Q8N7w6PTasRr1nI609VlJyPbv0M71NGSI+vrkDPVE/HD5UP8K9+gy9vV9iET1ffZ49","ng97PbR0iTx1Nq4+A1rPvfq03rxtiFU+Agh9Ptw+br4D/dQ9m1Mdv9YNobx5WZy+/vO9PvRmJT7V94u+0c/cvs/D6b1KKLG7fXB1PrGHIb9pYKc9ousDPndo6b7B8I2+U2gWvs6srj3DvkO+LsfRvqeLuT0wykc+ileBvhCCe74u4Ey80+HUPecNRr4hcKy9vGNPvvY56r287Oa9x0Vxvo1uAD6Pxz09lneevQ9iZz6PmUw8BcjZPZxyWj5udey+s5tbPhwKfr7yqpK+3y6XPcTmBj6Wq7u+yWF3vUtUpz3BlRw9BJ4NvxXCzr0pR8086fVqvSMPor7pxoA9bTCovi0zK76++M++Ti6tvg8JCL5IuaG+zI5UvOZ6n76SEJy+VgXwvThZZ75sSoU8tsqJvhxvn754OJK9uiCUvtL2wb2YUES+vd22OzVjhT1C9sK9982SvkgCmL7IKWK9I3I5vtHhXL2sG1a9/f8OvjkAu74FQyi+4r2RvooMGr6xTU2+MlmNvt39Ab4RJHi+YRaaugvoKb6MLfw93KGyvlAFg77bcDW+pLsxvs8DBb6Ixko9FWMyvjyX/b29JaO+lO6yvcpyUb6S4NK9+6o2vvSyd74DPN677tjPvdeeW76raZe9TIRHvXZbur7tKry9GGA9vguQZL4Yli69PPcTvnuI2r1744u99kprPRm9Ib4DPd69nlgQPRQJ973wGes9mimUPeIh/jung7c8R8DTvWhczr3KUD69s/FYvv0dCT4OiE8++PxlPiyXwTzYKB0++DVQvf67Ir5FrVa+1gV/vSOGH77iBKy9EXPXPX+hPz792og9zMXAveu1FT6n6si8YUNCvtzhcj0yGUa+Jto3PLHcuzvRsBw+T6iDvrZaeD2qSMO9qU6DPDpYnr1Oq769aLwAPsxEhT5Iv3A9lFiiPM9EmzyYB5W+Eg+fPUl+kLvjsEu9t04bPL206L1BjZ498aZavhHMEb3FenW+YypXOwUrub397BY+5Nb3PWWkhr0028I8","EUDVvNuG1j0toOO8i6Sfveis4j0pGry9vZMXvTlIhb3/M0g91xR/vX2ctjzsgps9ExBSvuwGjLy3ThQ+tDQDPgNYEz2FjDK9seT4vK5Pnj3lKB6+/H2TPfCoD76qjCu+nwUGvmKQkD21Zuc9kTwovhlLIj7SsAO9RFIHvabNRb7ZGys9lRsZvZbjmr0p6zu+ibQhPTBxOL1tUu+9BtCQPasJUb2Zo3e9Xoi4vOFQEr40/jc9qSF6vRIDG71+h7s9TzquvViH4T1rsfw8nUFPPYBcG75HwIw9orK4Pfuzpj3sMbY9JDyyPVGHID6brSE+aowBPrT9BrzT0y+9pHx1Paq6Krs/WAe+N6iXPdyZc7x19bg82ZREPtNbnb0PGhA+lutNvu9lKT737SC+U2aUvv4pGr6jTrY9jziGvvI/nr6TzzS+491ovv2Isb3Ut6u+riOmu8lqy73emM+9b3S4vcOlk77kKx8+l4AAvjLLyb7qCCY+IMOOvnJ8u77Jyom+QzFOvtk1770I6Iu+2YxCPCE+q72XXHy+WqVxvuuJlr2vxRu+AqSHvbN/Mr617EA8WMHIu6ytwr58O3y+JdmKvuiIHL7g5CC+x45HvCPXDr71ahM+KivXvU1+tL02sXA9wbQPvmNixr5LVZ+95C5LPqYCO74zr4m+0GFdPTqC8L4N40k9PRdyPWFZcD49Pn8+XopfPYJXaz6+3sa8uMiNPv11DD5/SdM940alPUlv8rxWVpK9zpglPmiAmD4EIcK+I4U8PjcESD2TnNw9N2h4vREJhD1KTcY+Q/4UPlGsZD5+XrU9v2Q/Poje+73mkaA9R2YgPr2Fxz2r1sA9538xPqEHnL0z7yM+sFsvPjhbNbwiJCQ+at7UPQmXAT6gF0I+7tXVvF8Ztru/Rwc9TDL/PIfNcT3oGoI9/b1zPgRslrw6mI89IsXWPbx/Ib4jkAY++siuPbJwCj2r+Rk+7karPRD1ET47HkA+gUGyu467bz6EKYg+rNosPs7UlT3YW6U9","6+Y6PcWe8z3A/Y4+2//ZPU4BPT6DsuY9yzNyPaGkNb4W+YG9qVp5vAGIkDucubi9LnGzvfKfgb7tkBQ+eMguvuhNFb5WujA+9dwfPdk+2TrQmjy81aeeO5YmBj6/5IC9+WAAPphSKL7uB9M85vTBPYOEhr2+N588qjfYvOf9ET6M0zI+TiJPvEZndT19QFK9Y9eIvRjsXDu6e0m+InPmOy/Q/L0g3lc9DBW9vYFigz6TvFi9EB8nPimyX74UpKO9tUbEvZjhkz2OLs4+p34evpZair1+6rw8sYARuz44NL5d3sg8ifUTvZDflz28h9+8CnJ2vbWjD76FYAq+FOglPe9wPjwWlG89oOXQPTaI3z39zaE+Odz4vcfsPT3MeeM9XCH+vCeT9j19kro90m8IvtmgZT5U1qY+Qt31PvMbyj15FXC9+xFwvImMNbzOMdC9HLhNPU8XpD1Yf588m/rkvHGu+Dze1p0909ryviQ6qL5s3RU+4iEkPifHQb78l7q99z8qvkJhdj2kNps7l2vDPQ8ISz4V7As+KP6jvEHBRz3JNAY+0Y5rvQqjTDw7IcC9arOjPZOxnDzS8Bk9uruVPW2+VrtZUzw+0XwLvmOMmb1Jjqm8WMDnPISqxb0Tr7C9bBRuPUo5nL4ryVu9M+e3vWLNAT1kojA9UNiOPcv5O72qOnw+yYvyviR5lrxedHw+GWcYO3BNPD4at5W8HVJxP1PEg726SQo/2F8VP/s0ID40W7s9+NzRvR8nTj66KYM+tHSNPshPB7/EiUe+LkWbPqnuJL7FpSA/Lk5vPuNlRj5WhIg+og8RPnk+ez0IfXM8GWxGPXPfA77R4ZS94Wagvu3UGD0MY7A9+LdWviPRr75XITc+ncwaPtufqT5Dnos+mBoiPiC9lDraB+A9lBrMvdRyub4m+WE++ZiQPgfKDT2Z4oW8K5c4PUd0GL5uP9s+M2PPPY2X/T1+/m4+9eqePQsrKb1voiY+48zfvdxFJz5lpH0++fy5PhIrrTyrflU+","ExQmvjPMoL6KSz2+DBzHvbK7cr1Os/49UJy4vuPDwz77H669HBrWPZ6gHL1y2iu+LtDHvjFeKL5O2Fk83veuvfaIkj1y52g8p+X3PTfkZL7Xyc29QaLnvZg8O75904G+FIOivnOatr6G6AS+3+W0vuRfHL6pEk6+SXx8voOK6b7PYtU7E+1TvZSaNr75zyy8bOFbvrOqBb7cm/q9Ps1BvYuhtb5P24S+UWNhviAGIz2bdbG991KMvpKbSL5eeci+5h2Bvm5tuL7jF8O9TEm6PPoWLL0JtyE9ITmNP2mzoL5zIA0+cc2lviyNOr6uTk8+OXjYPWyptj7P3ku9qAHPPbTStT0thVA+IVLIvfaVXT6L+Mg9JSj+verZjz5gteO+vVmhvRNGCT7LXpM9D02cPmDvJT4SAoe9+fEIPpNIjz2gW+28ig6GvSdHGD5IZRs+p5Slvt7iED5jwQQ+2ND0PTIiMzzsMYA+aniDPopbrj7fM+s9RJmzPfb2KD7JKSu9v1d6vTQeZT7w+bM+78ucvvwhmr4bAFq9MqTsPpKapz3sFSY+/1d3PeY8g72h72u9NsKEvtL8cz7u3T6+Ac0CPa1XA745v+U9LaROPYU1iDyU2vS8eDNavenorr6RO5e+al2HvQw1qDwf6sS+Z6wxPvJGs75vuMc+tcsUPtKvjb5FkIo7RQVBPp2DCr3tgVq+YjEAPiWohr50OFE8Y5uxvZEjPr0eTLs+Glt4vmmt3T7Hqxm9lOvpPVzEjD1Hui8+5fnyOyMxRbueaWg+pLanPTaJCb5ZsZ8+zUahvT53Jj4ilQE+hSCrvSQFTT1T3K29xMgfvgPLHb/BOcs8FF/LPUgpLL7nxPu9OABPvd5Fyr2bxCA++o2PvVxfOL3i3OA8RiuDPFMBSj64/Q09uFoCvoemyz2cEhW9YXbkPMQsuT3/EgQ/tLKHPH0vAj7ULiw8ANbqPXOZFT4n5ZY9s58IvTmcML4cJIo+lm60vYiGgj56qrc5pm4Bv6ErRT48UBY+","UYeMPTG8Hr4G80e+uuMyvScU3j0IV1c+OXwvvpu1kj6wIQw9qn2Fvb3ur75QofY8JyKZvj0cmL0IMIO9ZjuKviiOYj17G5A+vsAIPpgq3r7/brg9cYhnPkLTDD5rk2Q9n13GPhvgi76ISkW+natWvenla75Xlvw7AvTZvsvgoL4cHM68FJxAvrhgM70Azua9uNGAvfVODr+XINi8zZOXvd6nF75sNRq/pn2+PKpwXjzoNMW+kX+xPgd2rb2qVYk+lJNuvnZDrr5u27A85FtPPsKkZj5Pmoe+dGCYvmdlFzyCMLu+btZJvjTqob4JjWW+xUArvtY3fb1m2l++Mml/Pg=="],"recurrent_weights":["RpeTPTDXH77A/kC80sEuvlNHbjwX0hq+35xZvisnN7xeQZG9nnS0PPI1DL1DqIq9QuVsPUCgq71DDzu+Z2Ewv9v0AL7S7Lg9Pt2OvX+q1D27fWI+0NmkvuEMR75aW8G9IwsHu5GrgjzItTE+p3pCPilvGL2+maW+up4+PnbrnL4oOJC9C7LCvsKtF74ZMyU9L1BvvRgxTD1lrRa8qCduvoYjkT2TKCe9gTsGvQ+aIz2P5+i9G/nPvSAPo77JuAU+2wSbvtMzGjw1v7w8B9UUvRDYBT5qjZ89MJz9PB1uGb5opMI9wBPcvUOMjL16hiO+8dW5vQUykr4ZNRc8mvzFvOBJUT17oy89ggzlvZNOmL3WPai9cH0lvZREvTzuy2O8kq0uvrNGqjtZPAq9BoBnPqMtgj3Z6qI909gUvY4EALzw9+A+TtXKPajP27uI1ow9UykCvUyrsj787P49x8kKvVC37r11ko89SU+DPHRMWz2PKIo+agIQPpr+NT2Vpuu8MVUdPrzvjz0wDss9uAYou2BF9r0Jc0S+2m5ZPtYcjTzB/cc8YQkrvkBJU74GCqa+4RswPAB0Ob7SdPY9n6yEvl5rgj4l4Q2+j7kUvbrD6D0apLQ9Lid5vtb9hL68ewO7aR7nvZQpCz3rFEK9zz6BPVzGpr6uto89n7zZPTpYZb4WCRE9Pf7bvH6xJr7Qzz+7WmoNPKw7IbyH+BG+FrwwvjCPsL13pZY9DvA6vHH1jD2AD/c9JQ4fPURzwjwtKbu8nSm0Oqz8qr1IhVo82QyUvVxcJL5LrZ48vW7bPa4MU74HMaq+hj4IPuUhST5lSM49irVcPbGzGD0k9i+9HmkjPRtFxT0D3QA+kgaSvKJLIL0P0KC82rBOvm2Y1TyQ03a9+56HO4yjzbzwe5i+cNaVPrz6HjxIIxG+NrlfPoezJj5KLTi9ztREvedbBj5H5Wc+O8povXBkWr6yywk+xJxhPgh+GD5nlPI90cRyPQtGlb26UZo+kjX5vLh0GL7zgSe+","CMW3vcz8hjx8zD09AkQNvoXt5by62PC+7F4ivkMK972xJmA9q3yQvSfdNL6E/uW9074aPmeHRr7FXby9pQu8vQdSUL3Bbfy+5C8Iv5i9Nz53vV++kuv3vdPvUrzfjJY96JhXPt/iZb7KlS68xghVPBv0uL1M7Im+5w2WPHC7Fb8xatg9AierPKXvh77LXUy/h3iQvqxZ3r0d5Re99ZDzuxCu/L2+aCQ+jX3BPeGpmT0wRbS9+lL5vbrV6b42iVu+V22YvvJniTw/qh6/5PFIPdxyCz6XiBg+zrwZvljiPr7HTic9gIgnvNV6fr7xtZW8tdiwvfbPeL6CUk0+SCX4PUWb0r3PB3K8MjkfPS0aub1pxlM8Mkq0vaM2Cz7GijY8f23MvbAjpj15P449+CUuPrfEBr1+hA6+DrhIPB8TpL7nPw0/1UXOvYdSq74VzsK+kB1CPQzFqD7zLfO9ORD6vVwhhTwgqWQ+kyMHP/SzuDrf5lC+J0FWPUm7Gb48Aac9MKYBvhE1kr6Rl4O7q2xLvEdGGD1OXE++wR1EPg+hzT0bZp47KPlUvtWeYb6zeZ89rm5dPTNV+73JCFi9cLs9vps1rry1eg6+B96BvjiJRL7m4lA9MqmIPafblD3kBMS92QcgvnZPDL5uIXI+iAWsvTMABz6mzZY9+bVfvso3Kr6ywgK+QdIrvoUlwD6nnzM+b/aQPpX5fz4CJqK+0rQsPrzQYj1XKZU+LRLuvStUgr7ZwQM/Cd+wPg8Gfz0i4lk+MPWAPeTCLz5HNW4+QnXhPssFLTz1LCq/e9QaPfR1vT54pSS+yefGvjd/tLwyBS6/dbD1PoN/hD0ciLU+FHRevUDgOz6hCGS9i92Wvh30cj0pzYS+1QazPTkwTL1KvGY8y6y0PZtGkrxwUEO+FZrxvQkiTT5XChm+1+rRPR8GwD3xAia+hmmkPUV4CT6HBr8+epQFP+cuczuzzVI+zAh9PUUjiL1WRCW9x74OvhQ18b3XykM+G0mmPX1AuD6k69i7","ObYBvkw10TsxgYS9Tkx8vrMtib3f25u9GDs/PBIAyLxlqdY9lUSyPvhiIbzfuYS8lqg9PuBBWb0bQDy9aHSbPox40T1DqJc9RAZWvIbVZr62zIY9GLi7vtGhhb25rok+UIuVPhdPej5ciaY90HTNvYHsob1UUcy8qcwdPInowD1T7Ra7q9J/PeiPhTzMwmK9S+BDPkA0oj2RJLm9aDt2uhC0jz6s3IU++6pxPTkwBT1C+z49iznFvYHFnT3TsCE9clMhvlE9272qmD6+okO6vSGZjz76CVE8R2zhu/CFgr4rJoU9JtQrPi61lL1gFki9lU4DPnqsIz/UAB4+CBinPBfOyLzu8kc/LImVPCvigr5NBmQ9PzD1vv0N9b1WFVE9i2mBPV6wxjsyzGa9w0y2Pk+kJj9Y+bO+3zIUP3FcgL7CVl4+CN76vltLwb4OnNM+Wurlu7O1hT8S2j48c7eUvj4Bij9P8CG+cZAKPkF9yj19WW2+5PUOP9oOjL42He29Mr4xv2SIP71Yvwy/vq8Uv1H5nb6c/+i+Jb6qvq4c5T7sAny+XAh4PWKYtD1npBW8so7NvRvTVj90Pg8/P/E9v7aWG79xOh2+4MxIv/4G7r1dCU4/gd3nvlOypj0DMkg9GgScvrUyV72cMbs9MYkQPxVy4D3Ic5c+P8Ygvnqgyj7ribI9d2qKPbAjjj6xge67idCAPgtpXD6Slpg9wEcnP0H04D1y9sc9F2S0vG8jBz5TRwo+jyMXPZP5HT76Xpe9sLE5PrmWAD6unWK+hIBVPuZWdT5KCQU+DGZ3PuPcfT7vb9G9ig9fvBq1Nz53cSa6nHXXPZ3HED4hVBQ+QtcgPumpaz76P7G6gjP0PeWvEb3Pm3i9NxwuPh+7yz1RFv49TqKfPWh1MD11Ias+ohYEPQ3iAj7o1vq7txiRPcQXgj2IriU+Fz+fPWVeFD4Wn8U8HBSOPHJQjT5HAtE+oJo1PfLstj2Mwvs96UtQPa/zWTzunv86Ppx0Pae92j2vM2c+","aMpFPfsQB76hFl29o/HIPZjbxz0M8Rk9nROBPsA2Ej6ryBC+Rt16PXGtFT6XLXc+87TsPV70qj74pDe+ngkQvaJR7rz8fZM987BGumI6pD0LhyK7MkOyPrQOSL0q6y89e9L1O3rldT2Lrre9+829PVWOdD1mrik+41vQPiz+HL1TxDE+XhS6vQvTv7ssqX09yHpyPbVO/b1f9Ao+t1LfPcHMBb5LUT++jnfWPECFRT39XDY+SJyhPe4RlLyDs0I+RRvvPUlGSL3RTxC9MldRPQUWNbzFWr69SQzyOi88oD0CRAa9CFlEPbHRFT57+My7nEq9vlB5jz0Mf3Q9BimePWAC0b1HjRO9dU+4PPImNz3+0Ak8WtQbPbZViD1rL2g+iQiLvLaCP70wUU48QUncObhZgb4PKC89My+DvHo2oj0ctIK+495zvaMaTT3DXiC+xn0NvVFysDzYR7E9qwmJPmDqirxWViM+jBpbvjDL770tt4A9zV0IPW+1gL7KgYy9tiCJvpRekb3l2bQ+tcmTPKkjlL3L7l09nd0evsSDtT1B2gw+CTIJPMP1F75nrRc9KDGzvVHQeD1Jn9y9wmWoPqHmIT2Rcwa8Vw1wPewhE76fDci97ioavQNPl737sQO9nZhMPoubgb5ilAe/H0jZPb2RS76Qsf292NKlvQEu/j0MJpQ9NB8ivzqeL73bnnG+vEaqPtX7cr7Zc+o8e/6zvsYCwbzrAUW+1T3JvqnYpj5hLwM+3uVTu6rrnT5SV7E+mlp8Ple/fT5NbMY9TRZHP+Pyij0hrJe+ZMLBPlNRozwMOi292rxQvhkoHT55tDI8oocsvdLc2L6bXYs+ad3Evollpr5VCcS9eMhnvmwiS76utTG92WdBvRYAPr2YUe29DqgmPfF+Cj4r3fe9jkQXvjgkvr0SZPY9rsZmPjAcgL4NmK++wgKtPjJCL76/Zo++z6CtPsRMAT/7hKG822YOPQUBXj6Yyb0+zgFIPk9uTD6DNj08VJ+3PuI2KD6kLwa/","zF8bPvCpQzxbJrg8oHMKPZa1dj2FzJM7+I7kvHqKIz4O0mQ9yMVDPK9lkb151ei9TpcGPmLgLzzSsVU+getsvhDoz76tAwm8C/isvU4c0D4bFAy9Fvolvo5aKT5U2dg9jUeZPXI8Hz4JJKm+W0w4vdYjCD6UHmy+WIzyPb9a37xHCHi9DeppvURSdj1Kf729Qs8zPcVVfr3RIdg6AW+XvVi4BDpXf0M+BBpmPeFyBz7HGxa+A67XPTC0Tr5o0M89JY99vCz+w7y0PAQ+vbOgPJ695z1aJD09VTBRPG7FaT2REJ88Pc0dvXPsZT2PksS98jwlvQXIN74d7ty8nYekPaKHIT2sH0q+L4F0PWF45Dxe6tu+UGwgvev1oz4obH4+pq6lvGR2Ij2z8NO9u7kcPXH9C7/ImMq9VI6qPULXIb2m6pm+tSH6vCowbD0P9+e+3v74PfJvmj6GHMk8dYkrPqRcejxMxxA+ZUBzvbnJJj4WUIq+zFV9PkZQMb0g6m47bbD3vFiiqz3ktzw+6B0RvdJxmz0tK6M9xXMgvSvBDb563ty9IlRwvl3ziD3D7Ts+f58Pvec5cj26cbY+PwsuvRGikjySATc94AnjPSeCKb3bFmi+n+iZvdhcg7ysTa+9QO5yvu/1rr2KYHw96jyvvnPttj0uxgs+Ii+9vjVjpT36t/A9wPInvPLogzxOliQ9nU8uPgRECT61YV6+yzhBPW1nu70f9oW9euYnPZMvrD1ZIya+WxgHvqaOHjzs9w29AU2UvqRvX7075SS72oBtPGwRlL2SBze9H9psPsZdBj6Pxzq+e/eqPjLIrz23iaA7M8eTPoxPhL0q24W9wAcKvdXVhz2/E5o8iOc5PAekHz3S1ta9OSnePImhK76v++O8YNBMvFR8Lr4PFp29znjHPdXe3rzJZWu9DgwHvr+40z0uDm++UO63t+ytJb0Ki709HJQCvlTbL713zbI8N2dfPhyYvL094kq+0hBxvQ2cL77YZ7k9ib8cvqY35b0J58I9","GA/+PbYJZjwMnqK7wuJDvvCIOL3tssE+WuKePWxbV7z9zH0+6kZ2vcHitT2AJB8+7dMrPWdMlj7sXAG+l/HLvWMgUb4jfQU/qw4QPzNGoT4V9949DOC4Pobzgj1458M+XRZuvhrC9z2NAnq90T1jPeI7GD6YBXM+Hc9uPrG7t75QC5C+k9w7PlC+4r57b/k9lxsAvn53Cj1sM9893gCiPUnPQbx2cNw9txC/PXgRUD1sRs0+fQ9ZPaYdsj2xnuA9WyMNPA7aob3bKFS/doqLPWJreLxRKMA+6aM9u4dxjz3FFvA9g8PyvL/mD73uJyE9OYuhPWqB/L02HNg9Ye+wvhVgXD3y8z09yqLCvuOeKr6P69a9WaWIvQMsTb4JTrY+EUG6vB/LPb0ryI692c2PviKMVb52BQ69fAHDPeW6Zb9aXIU+IOsivM0OHr38zfE9E+ydvug3QL64UAm9b0p/vmy3T71Swqo9hKMLP5ujbb5dYRy+HxJVvh/tRb5gNoC+eL95vbppnL619AI88X6OvH8JkL24qfi9XGxIvkLA173troA9E9W0vausUb2B4wA9Y1iyvsnBZzzhVZO+WceVvaZsvL6yQjY9FOXAPYdi37zFjBe+RYtCPPAkfD4V1IS9bcskvX05nb4h4K29d1ElvoYDPD5hfqK+m/3EvYmQiL2QDFW9uw5ePqwX4T4/wDw94JGjPXm9cL4yqRm+6MkHPmG9Cb+e1hM9eKsGvv1U/jw21xQ+KkHJvnJWdj0Vx4K+5r58Pqr0cD5wdUc9/bIEvuZSKb72TqY+2lqwvXZ/i73+vke+eX/ivV6Ujb006Iw9scUbvC9FZT53P9u9++yfvcVuyj4VxL+9vUKPPcqo8bxkQHI82P0bvhApOz4QSQi+mnt6PPGyC79lTI++pYoSPcXbcj0UWGu9ulb5PXouCb9Oefi89WAivh13lz64onW+NxRmvXS+jr5vhUG9DVKhvWOX07xEs7c+x7iKvTWGFD4dsNI93wC6PUFpqz1rgsu9","2RccPg/EaT6NCoI+9mbbvGaQjD2nDgg+5fYlvoyetb0EOpe+FANBvSVN1r0Mmb8845zFPp2tPD6b9KS9VN8vvfzE+DxHPsY9VxVBvGDDQr6/66W9vkJmvcXYLL3i2eK9aHYLPnTsWj0Z8XI+qpAkvWGLAr5n3Ra+QQl3vmG3Zb0uXIC+Ra/IPSGkJ71w7MS8+qaBvb2+C77+yIm9rv5JPLyMmj1jnII8hs9qvil7Zz58h5M9h/kPvoh4d71JPAA7XRYKvil4x7xtALM8dartPha9tjzFCx6+Jd06PtMcqj4Js5q8iLVAO1H6kL2SAYA9pNrzPXzHDT4DqeO8Km6svWyPZD6tqPm+kdHBPlB4vb3SIRI9gTkePrlaYb5SHT2+mCTcPQ0BBj7mklc94H2uPtFN3rmAAbu922E9vsCNxr5tK2K+EXBBvxFqEr5W/IQ+TOM1vgM7lT3TAxa9ytyJvejPiDu+mpO+sgPMPe+ys70rVEG+itHUvqE4S7wGBSS/YHoEvqDsFz5+IyS/Rgb7vrOrkr6bs4+9iai6vW16/D5LGas95HnEPcJ6kLkAWz49esEGv436kT7l5Kg+eBOZvjvwvz6nYoq+SePBvkseE73DVIW9BhIpPthEFT6Ed3m+lIbfPF2HAL7lApi+sb0LP+LfirwdUR6+xV1vPa930j6ZmYE940QvPML0jrw6Z+m9nWyuPJYiGr52s9W9WiMcPkOsvjwkY005UaWKvD19HDw/ghW98OUnumQUZj1FOje/iQJMPlzFYzynNg2/nFYYvp7izb3YeBu+ZK/uvd182TxA4IA9O5cDPjW1wj7Ww5695P4qPnFL/D0T6228M5x/vpelvLwFoOU9vOfkve3s/z0rrXA9v72BPfqPRDuVHQW+9fMbPvLTLzqS7is+QYXdvR9ogr1ZzMo9NigYvtei0z3mGjG+t8tjPcf50j3JpzW+ojYdvURCtz3VnRs+mZmEvFHvL7yuqB+95YARviwRir4zi7i8iOKyvmscEj3ecsu9","FmqcvavkWbt3PWG9CHoEPZm7Bb2m8EE+uJxCvs+qjb77YRy9uB6CPpW96L2iQBO9wyQQPr7g+z0UqAU/T8dRPoFYqj7gpNW9oJCYvk5XNb1ixl4+7UOMPXwLTj59Qw69FGkQvjgmnb7O/Js+afqBvaEWSj4h4CO/OexqPnovqr1qWbs+l33+vY0Sm73es+28mQE/vv5cCb6LLFE6WHPqvQ6IoDwGNb29T+9AvmuJWr1vJoU7jqs3vlPgbr3U/zW9LlEXP8r/8L1pa8W97WIWP2jrlD4QIHm+qbZhvposwD0AsdI8unPkPVrgHr5rKM49MpFJvvoTpz4xlz8+uklGvmDGBj4y47C9enruPeFqw7y7sKq9SbrWPB8peT56So09dyx5vQIjKD5IU0G+OPJAvbrqJb3PVvE7yLLDvTG5Or6jwtc+Bf5jvnaw7T0ugOi+R+R2vM+aML0EhEw9uN3pPMuah770ATc9hHMOPhKX+j1H2hE9Ry4uPvaVML0Z8ZE8CO+8PbIQIj9qO6y+DQumvmX+hb0677e+FUQVvfNqvL3KxAm+DuqvPRVsqr1HJm4+YsqMPo8Tajy4cSw9Mh/nPee8s7vXj7a98XwOPvjHoD3zo+w9jC5Yvm4YNT5Zzj0+1PYlvvyMOzx4+uk90vzzPTFYCD7PcQA+xUpkPfSrt715Weq9m6tgP3x/IT9OP3K+pPCzvtXA3r6/kA0+QRbGvSUs0z7ds/m9H45evJwDPT6I10M9LejAPbLcHD+SbHG9xHvKPaf31r40nCq/Agmrvostoj0c1iC+FJmSvejrsr1aWBc/EKQBPQNs0r10pKq9qpjQPiBxajz9bZu9o/iCvhZ0PL4WgeU9FTjqvuSVTb9/JZy8YR+GPt7Wir4gNQK+JAAhP+rsgT26sxA/iWdBvq8BjTzmOl0+BKWHvoqRHT/gdnS+/8B8PhRpQr/Tj4q+kaFrPf43UL98IIq+Cl4avkScej5MtgY+04BTvjnz8r4PWOK84SDkvaBtzL4fHjA/","M/A4PrLlhz5ZWBC+3sWcvej4sDxJMO08qs/OvegjCb3g0Dy83M8zPjjyDr5f2kG+HEHUPQt2OT7T3pg9QDKJvb7p7T0GAok+CfQBvwge6706ftC9gMz8vUPdhL6lFnG8Yr8ivvC8ZDyr/fG+YLcuvlHbn73eUJ89RbrOvSpuID7uoaC9QV7YPYEvhr56uAs+Oq/ePW0AW76nkOy9mNX0vVqSIT3Bzfu8v5KwvFh3Bj6wS7M9Ko4DPdCbST67A/A8jfHOvWvgFj0hdr2+E+QZPRudtj1DU4e9fIOqvhkrWT5gf789Su4DvlJT1j2//2S5gyw0vBYrYb7ezR49VjYFvimvwLwhhKe+bXEavgTPfL6P+Q++IDT5PMnNPD6vPni+dMQTvY+L4L6bmNK+jKsGvhf1vL3YbU29zQjpPgTorr7gjaG+sPHuvboLbr3gd2K+mxzKPdY8Kj8VOui8x3LgPT75Jj6mPu095y+IPtJtI72pIGQ90+ZpPue5xzxsdqi+yluhPkgyEr3zp4s+//FRvrsWCr2rSQE+nHJBvvyVCr/4PsO++qoDvmeNBL6wHt4+yQ9hPMJnLT2k+pS+GzeQvizFIrwxAbc7UU4zvVdBqj4l5Xa+lNq5vbaWg73814S+Eldcvilcir4tl4y+jYpcvVMW0j2QT4g9RSgFvJBTyT303Ay9CeyIvv5937z/sgg+53wDvWLlPj7o3Y07cS6YPkWzirrSeIA9mKe3vQRv+Dw6uIe+oOGlPlufoL2yJN2+fQy0viH3mz3O7QM9aeX8vZgEJr6kFZs+gIMwvhmdvT4dyXO+r4b+Pa4jqb76R+E9W++Mu6AxlLzurqo+RTW+vdd2xr3mpY89A1nivAlZb70cv1s9UysovmHVI7+dJbS8OOlBu2MpnDwyIay9VVYuPXALob5qNjY9pwwJvrt0Fb4Mf4K+71mMPa9H2L0UrAm7pburvlW5gL19OtO7B4pSvlOX872ZIPW+bFizuvJHMb0fTL4+xwqpvtzrA75Ke5Y9","wZ6BPtTrZD644I4+QQ3RPlZfkr2onkQ/aNDpPqzmgj6toqE+J4mzPkphpT7KQBu+qnuBviyoKj+Icu88mMk4vg7XPz6ANAg/dmzkvQNOVT5FQOS+V2KjvuFdYr7j5Jm+NsrKPmq7+T5LCp89CmAwvn+Bgj49PoG/rOizvh9gqz7Sg5G/chBWvH5cJz7Vlmo++cNGPqZPb76b2KQ+y3eUPXz8mT59inY8LISzvYdjHb4OJYY+XTQCPvHRIb9qt5w/fldgPeNO8L0AoUq+DPXoPmY4kr2V27m+67ubPsULnb57dHs+WQ2nvqIPfT7Rhu6+zp7zPH4Mrj12Akq+42JWvycOdz2FblE+0MxAvcJq/j2UJ7E9Bb2tPXtNkj5VeRc/fhySPQT1iDzWZQq+R7JGPtAOjT1Jlwy9MbjnPsOEjz7oYr2+EYmgPmaZGj6xRKA+wiFlPtg0Try4vqI+HgRWPqovdD46URc+dXatvRasiD4MNey5x0KLPiZzjD2f9k0+5DYIvu0HVjybH4s+Lp89vfdLrT6w7hI+OxASPSiMfz4WFmE+/dTGPR5Bnz0OLRk+8sKlPWhLTz7/9vc+Z91YvSieqj5nksU+NMqAvslcyDxrB0E+YvZXPWq2Ar22Ho8+caATvAYrPT68NkA9bZImPuUjpT6isDW+rYEQvegnGD6I85y850+2vdYwy7sIdFI+blAwvryxeT2lvkq+953lPuF1ub2gaGc+GNwsPpXpFb56Mqm+H96OPQN0p77U1do+QALaPlnMmz55S/g9DSw0PrDnOb5YPa0+kWoOvWprzL7c9SU+waYcPT1Xgb5bIqO+3kT/vcMnST9h3T49kPAfP2Xwt74aJ4K+YlNHPjmPW768BAy+1n6RvrJUND6MMcI+pWUbPRC36Lx/Yza+FwYfvibGl77RYR49z42UvmXCEb7ZXwY+PQs/u9bNLT3o5K++m/TYvdYRnL7wNKq9LYSOvsFNvTvnEd49OGpAvVNZabyqR42+K/iKvjT8/b0ShNU+","5/i+PJMq1L1NKL6+K+QRPo28GD4M48g9LK1EPNJT6L1fLBS+j64APWouO734D9S9MHwaPxiOor1Ftjm9g8nePQJhWL74cCI+cDTKvqyslz0SheE9wFpFvsVBGz71Jyu+X6DJPfXOgr4venc+KmItvkyDtD03jSK+NEguvhupnr5/ho69VVefO2GMID4BCMw+7LWFvYYd/j3rVVm9LWY8Pl5zoTu1jkI9bmvGvU+v7rugf5c9djQwPXDYBb7pBRW+nCWIvdkakL6NdBW+ocoePuDFCr1rB++9HlqJvf+5sj3fC649Nj+ZPQoRDD019So7sXL0vC+6/b3/Idg8s+K9PE6D7j0Nrpo+irA4v0gfkz5ELdE+zC2SPs+a4b4lKfi9OYgKv1TXBz+nU5W+WPWVPoHx5L2HN/c9BayGPr0U3T0dvQO+0E70PjQgqj6mqR0/nxgPvggLaj+S+ss+5oSFPnr4Cj9/vJW+8DLSPvg74j5pMHO+daCnvkwqVj51LOO+Gqr5PkK+675hqWy+1rWKPjCx1T6QpRi+sa28PDFPsj4KzAS/jk4KPplAUr6WDes+i+MHPqKhED6FbN0+r9wZvydeCb4T7ds8AQDNPdH0Fb681F+62FKhPpOUaj5xUwM/kvRHvqwPOT3RG5w+h181P3k3iT3QdkS9RNaKPsy7T75ooVM9mlS1PQbpJDxQMQQ9YLmLO913cT5H6tG9iwoIvu0jOD3vmRM9EpdcvjWdAD5VIbu9jG3qvEcTBD0h3P4+c41Ovv96DDwkzbK9znibvUWsNz577Rc+isb/vOTQBz62ELK9W6vGvRUYHL+ZgUm+Ne4fPS4XYT0Rgp+9qL9/vDBoBT8CwZG9UNa2vaqPAj5OfDa+98SpvSC1wD114CU+DTN/vtamib3ig/G9wUPtvIF4FDxlDSy9C2whO+4e2b1uEGw+jfdyPfs/Pb7JE4U+1CXuu0oDOL5ILTO98PYAO9FNpb14brg8fJI8vnytTD29wco8f6+CPv+JpDwUP4a9","/D5wuzA1H72VKq49H7NsPa4gcju50gy+aRETPis9hL7ajde93hlMviZNKT5QbtE+H92EvgNOh75u0BG9OqIQPsMi8b6IxVG88o6hPJRRyj6o9Jq84PmbvoNJ9j1sn+e7cDibvMKzRj6JB7i+qv1FPnLT0r56huQ9AodtvYca7bzvyv+9n1TvPrH7jj39Jlm9LLGRPYZV3D6iXMG9hoE+PuTMDL1FmW0+jpiCPix7bz4Z7ao9vSWoPamGhD6jH+w+/DLovdphhT4B3v672XfhvlxGXL45Q1A+bv+GPjMHdD3JmJo9paCMvuEpzz0okQ29RXHxPlNBg76XYzm++pKUvVDCaj3Asm093XzpPQoBuL3FqOo9SalBvHq4yb7qjyy+UpAwvQUasT0/xH++gvv7vKbGgD3dMBy+eeETPbL8nb3/pq2+uGMTvfkJoT6zPPA+EmZ1PffQEr4+BvE77GuHPHtdkTxc1QU9r4LevU/pub3avJU8yiiWPZxThL1mB+y8pzvBvZ6Vwb6wv0Y5HR9Ovee1kD2cpM09cHmDvYZZTz2gqH09KlUGvS76Ij6uXhq+QW7rvfM1gj3EXUy9nL6cvAPMfz5E3xu8xCxNOxpHcr3YAj2+A3VcPo4tXb41rAo+qMOEvkVPpb1woYS9xtIBvi3cDb+dXx6+mabuvV6zWDqjxR4+qkp9vxmgPj6Fjvy9hsHGPvUh9D7NZLi8SUajvTHnrT6dh4u+r7YPvoC6ED7al86+6EEoPlB+Vr9mm5E9yfpaPqUv1D6+Zzo/6YmKPtE4BT7/sEq/ToHNPWPRs70i5UK/bdZqvf59Gr6Ntyi+smJevVJORL9pfR+9jWRRvDKpkj5JRti8DQeJPi1KDT8eEt08JE1AvWx0wD5YRwM+kSmkvbkDzT1XgA2/nd82vgflJT6NFSU+b/6MPt0VDb9Fvpo+CFSNvVGvBD/DLX0+gRRUPB9pZz4jtls9xZ1XPrFi8j790d88iyRivcRuHrzwDnw+49WKPgeTor1RrdS+","4d7GvRo5pjwf0SA+TeAQPi+cfLxSY0G9g1HXPXp0Fr9Usku81QfDurQu2z1YbaE++qifvMziDL66p1S+d4EgPVmRnT72EB6+CRs8vuRZCb+Vm7Y9X6gEPnPiF76D9fu9WcSSvdyjC727mTQ+IgLFvVzCAb64WDI+BUP2vQM3tb0Tr1e9UmkzviCDpD3/hPk968scvR4xOj6D7IW+25gEPe/8OT3RHzu92v2kPaD2Zj3SFrM9JdEOvsYPmj1tVZ27Rw1PvS32Qz6K5B8+HIc0vVZFkTyT1Vw7h/ykvuU+l70hYT++ohsLPTva573D38+8AyW+vSbwOr7pfTu96QHUvAJ4tb3MXF0+Iwg+vs33xbvRbKk+8DQAPKnlfb7E83O+zhJCPs8ga71XCGQ90mbXvYQP3D4TZVY+2a1BvkZWgz1K15Y+47M4vQqEIb4Hz84+y786PGzrBL9yHYc+NKgCPkvAQT2h4DG99RY1PnrGWr5MBns+H42AvvqBxD2r3Bm+47sYvntr5TwdIo+9vFB6PYbStL2W7Fq5QphEvalleT0WYnA+rVEPPs+aLj5CJdm8NfrhvXFclz6jwuC9rsWRPv++vTxzYoa9P5MCPDUOeD7F3Ac+3KeLPV6cnrySU429dWLrPUEwK76FK7G9EqiWPg+j2D31Hzm+NIrPPlyenT13lr079DAgPrfJir3cPQu+K3VevuHIM77P/GE+L3mIvMNtBD62R9k9u8R/vswmgz26TU4+2CnFPaxF0z3cEyY+sGTyPrPzCr3S5JK8bh3bvXndb76+jYQ9RulOvRt1CTyGDb89GsNeviuztb3XnNm8hZt6vjgqaLwXDHg+lE1+PcdupD0h40A9t/dgvUiWwD2xBZQ+wrYIviI5wT4shQW9cRFfvebSTj6s0268/4+HvC1+GbxuR8O9Cqj7u88q6r0XSRc+XdxFPsTyAL5+gcG9SiIWPCoioj3+Diq+cMiEvf4tuz2ELre82036PdY9Vj4BWTK9BtnFPkM8bz4afZw8","B9yBvp/AKz4Sq0E+Z5t5Puj0qL6LvbS+pZ/0PWMbGz+Gs46+cu+YPP4WlrwRKwO/CU6BPmbhlL7vnoA+n/z6PTf+mb2Jgzy/uF88v/K6yr5X3GE+QeQZv4ULyL00EkW+y57/PIc1k74BFSm+7cPsvXpKXz7lDFW+GTfmvgPA4j4o21w+fP03vbbVDT/5K9C9AFZhvgKa8z3R/Cc+yp7Bvqf97jyf7R69CYFwvJbqNL5tDIa+CewavowzhL4lw2S+SMejvtR3RLzZgFk/5RuLvet6mT5SJba+++j3Ozz2B71XJBK9DkCavb1rrz0xqcq+u/TcvYDxib3ZpEE9mT2hPtn+AD44Tdc9mI4Kvb5mKr2QB/q8Jh4MPvuxRL0ydP0+kqBmPIFwB73fVF2+Mh9Bvlp9Vr31vHI94FWEPpU10zzCCae+2GZJvd1F5r6qWiE+GFR8vhkJ171kz0I+d3ezPW9XnDwTwwu9mXpuvb5W/jyG0e48HqWAvcASbb2d2ii9TCNvPbbeMz7GRVI7820AvqarsrtpQyU+JuVxvTCxDTqYh4A9HA8iPq0Y1T1j3Yk9I3pPvJ9U3j0VbQK+GAzBPfgHk73WJLW7OnbQvbRLsL35nYE9uqMoPRYxfL04crg9inYbPgLEXj3yjpm9+i+xPZBPZL6C3JQ6e0qbPbsKUz4rOqE9oYY/vsRD/z0wuGU9qalYvm5JZL7uXZc+BTScPmKtJL4laz88aIrZPZFegj3yRhC+DQo/vgiT4j0/IXa8elievntMcb7cvp29DreJPfwIiz6GEAI/s9iUvVrIsb1e1AO8FrAiPlGEOz3Lrgk+jOwQPkeLij44gXW9tmoFPmecuT1AyL29LX2PPp8tBD27hai7bf8wvu3mDz3r4q29MD1CvshxFL7oHB6+kOEfPa1EZTyDuHe+gA9wPsk2qb5DffQ+pywhvii9Z73jLc6+0lPBvmXYV76xdhq+1FnovXfDEr44vU4+bZOAPpaHJr6Q6d48hjY+PvFARb3/3Tm+","lD9PvXQL6zwJ+mW93bhJPkU8Nj49dz0+evhwvm4Idj3ZigW+sSgJvZeiiT0ylh09FeZ5PVJHAL1vGCK+ULeavoftxL4eek++h9xKPZMYTLxiFRk+pwkuPiUbFD53uV49U4iKviFB1D0zw5w81DkHPkoILz100DA9n3alvgm2Kz7yovK9FNkhPkDOAz3rtk29BhO3vOZaFj2ahSK/tcp9PqTEBb1blhG+4mYwvi+ZkL3VDIY9++WAPfnQab4dn6i8iPOgPTbgez1edSc9ead2Pqy0A77EgFy+VCi5PcZ0mL0E2Uc+TuKdPMAAar07Z7u9GvzwPAt+Er9WJaW9MB49Pr0gtz4W79G9UeYtPj0FZ70GPNI88W9xPxghxTxyFc+91qfQPp3Ms7xpLR89oEQ4PpZWEb+KMsE+YTZnvd9bXTwJQ6w9xWRuPxn5Hz+ZLOG9uvervkbE+z4w4Pc9S+AWPf3AlLzi0Do+XNTDPSOmlb2yFrQ9U+yWvlbvqD0Jp7q+VvrlvReYeb6OxgK//c6XPhrSbj69EaO7+xcAPL1vOz7obYU+TtB+vUe+Mz38gMq8NjX2va1inb4Lftg+8aFAPz1cMz2b+1S9ZJ0fv2laGD0oOwS/wl9pPowrOj4bBc++I0w0PsZicL7gjuq+4A26vc+UtrxeYnA+VCZqvjaoAr9gWi2+I68WvthEW70TI2W+Cyj/vdPJzztfXAK+ZwmgvR2VLb7r4Ce+zRyzPJVobzq3jm2+N50RPZ1nPry5IOM9SgKMPW34lL7abfs9VFMLvg1sobwTzWE+fDMSvdHLwL0f3um9VxUrvllDkj4yNje+RuUvvtC/YD5NRtG9G8WKPWqlmbyg7LE9L/VDPm9cVr1wgM679EzqvYl8wrzafqM9V0Revjsxtb0p2C29UB+QPUkVdD2WgjY9bSDRPZ8ZnL7PJBY+AqI8vWOABL5I9/u8V3IAvi+AizuCp8O9Iy2tPBNDDb4yEGy9ctu2vQIuNr4TFr69WoTcvaQJ6r063Qu+","Tc+PPXodtz74Qsk9O8IIPxdUdT79sdE9h0yyvQ6O4b2EPKA99Qy9PscCiT5xzBE9HvUpPjoPMT0SOzO+k7cWPLivjj6i5BG+BcWXvV4deD6OQC89MdYvvkElVzn2Xho+Cjs2PpzC4705lrC9BsqYuTFz0T2WExq+gkkiPnuT8j6RMI47G9plvuOwQT3jggy8uvAUvbwuBb6YwsM7XqPYPvM6vT6rJA8+pvNLPp/Rtrxgocm61wH/POv1o76mcuc9NvW/vUxY4b3VOrk9UecxvWHrrT6dLKo9XDmUvXE2Ib2VHMk9faDaPgApgb5uv2o9hruBveoT3j1jeiY+FJNyvQKTzb3CqO49DrejPEYpmr4nW429HQ+BvkL6VD7vxJw9RVukPIhgAj5x8qa9lLFFPY0TnD5PL/s9WOaaPbqVIj454fY+IPbAvbQw7Tr9Jha91WEjPm16Tz1VmcK89ds3PgZ14bzUqG++qImbPQ6+LjxT8D+87L3MPSR7ED7jz6Y8SvQSvced5TzfRBi+x9IcvDSSvz1NKq89VgOvPk91QT6mbOq7D/awPbtLMz2U8FE8j25pPWMyGb7SdzQ9pU+SvbVKDT+fI1g9IjD2vfoNLr4l8rA+OJDKPcrpzb2CTI06aHGGvGlSFT88lGO+C4hlPtfJiL0Xibw+IaMaPjVG4zxBaAC97x1uvkmLur4Y6CY9w0+1vYllAL8bRCy+dYSiPmPwu76Kgje+OCLYPWenCz60cIW+Rsm/vid2yL0WgG+9LIINvIIZ1r4XcVg/T8havqbxAr40SmE/6oPTvcu3Y72rv129y0iiPNSGBz32BYq8BGWrvUdsd7wtDBw8WnprPhoE/T4QQty9KiAHPtb2+jpKBiY+ng/IvV9wHz4hCqU8II9pvXRVab51lG8+KAk2Po1Ky76gdja+1w2/PQ0vDD4DA9U985L/PC2d6j3Qh6s9G3Luvc7cAj45mcM9dWCkPcsrs76t1s69hg9SPvlRSb27UYs9JwBsvZavA79HCao9","DIx0vE56P70a2x0+2EypuzjefD1xoR6+NnbYPaA4nr6qAhg+DDqvvXqkuD2a3Ys9BwhPPut9+72jIU6+ZhsRP+pxLr5BLgW9v04GPWavVb4USF8+LmIoPvs43b0YQwI94MziPb5CSz0aDTA/wr04Pq5Niz4kkJQ+l0uoPUWvJr0tiAK+1vQaPcx9lzzHbzA+tVA9PuZy7z20OKY7GWzEPfYM+T1tDmG9WbcKOZGMW70sB/E9nJa6vVIbxD3bID28Ew6wPcn3AD2cI3A8038AvukGFT5HVjY9ZBAdvqtsFb2XtTm+rtQWPXp4xbki0gW+SI5Lvim5ZD/V9mc+pp8vPQWgDT2JCoi+5Y2/vUAjAr0Tp609EP0fPh40vTyjZwC+YhbZPqmhfL0lYm0+M0fqvcuetT1DpwY/7JS3vUXFLj8UhpO8QardPdxBB75aKJg9nZvOvCXgjL6p+pk9MTLVPjKtQDtu/jk9CZeEPtStJL45WIa7v1k3vjQ2Rz+bTbS8OD8QvA/UGz4xEAq+DJQYvoLcRz2Caxk++V2svcDquLweIy69fztKPtBPPT4TMra9JACCPa1fCT3RRHA+P3imPiv67j4ibxk9fCqwvbeL/D5L8/08+s+9PS8TA71Q82M+gHgJPmpfsr4juvW9ORWLvTJmDj5hZ0e9EZ7xPVkIMz6jyH29jt6UvRns+LyEP9a9EhDTvohJwb1F6AU+AC32vALafT4PC329YEn8vHaeUTxe7Qm+Dn+SPWrMXr76KsU862pNPCPl2rt10fy87kmWPeiv+TqH0D69hO8bvVQeNz/bdSY+1XZgPmSUEj9Q9Aw9QLAIvl5AzD0VHAO+bVwGvZDRpT3DspE798/lPsXGojya3kM8ip1kvET+1D0KpAW9N1vPPR5PKD6Voho+L8s0vcS8Hr6K/SC9XFV+PtMJ171vt20++8xBPQ4dFr0Mb4S+OyZCu8uZJ7205b+9gmpmv+aBBL4wpVk+Za/CvQucND6LNjW+7iwsPQx6cr16P689","Ul5hvu9H2z4pgu8+kXqIvpgCZT12uKy+1QNcPlSQZL877Go+8Md1vj1Jcr5DFXU+CYm0PfMUh75Uh54/49guPrCZqb2vLhs+6W5SP9e/BT4Vesc+Jwdav6JToL6csQ0+O2ZavyVRvT21VF+8k8EiPZYjrz4xYzk/IWljPYlrFL7ezcU98cmBPLB5vD2HMwg/lHVfPEAWtj4Cgwo90xaLPp3SwD5vOYC+t32FPm1qVLzPvEo/rf22PvY8jj7l6AM9LOsMPxWanD2Fmm496CnAvrQnr75OrQu+HW3GvlmqAj+mgpE9xjCwPllWz72fIvY8slarvSYa4r2Kmbk9mZWLPHol/LwBbKA9RLlsPOHjOj5hhxc+Xy/1PUXC0DqELwy/2NqjPLPYjT4CvCU+4U4RuxKiWbxjwuC9fCSPPl5gvD6ytE69JqiGPd0kh74B80G9toFcPlykhz708LE9QFDTPQDq5T1Tjyq99lCQvTGEnz2Nvf88hBtTPDXOLT5awUI9T1P+vJgd870RZAU+zmOCPHdzJL4Wfv698C3BPpsLhz6UEJm9ih2GPVZ5mj1L8Os9/OgNPlCqVj6E9Rw+9FjLvUaMWz5XTkc+eFP7vSxTQD0b3T8+0msTuiODyD2REsG7PcbJvcydjbvzaG27eLWaPM5SL76vKTw/yGUvvq1I6j3nt/o9olwFPhWQPz4ZnVS9gRJNvak+kz7UXHQ+tdE7PhiQDj/j7hw+pYXmPfZ6Fj3aEj++cvQlPVLzvT1HI7U9YaxHvuB0iDuSAQ29Q4g7Pq5Np71cn9Y+INdDPkGcxT05ZR0+NWsjPnHtWz3negC+rN6OPSWbvr3Ijg2+4tpBPRM2vj5hqn0+QMdovfbeHD4OzJS9vu/XPFrjdb2zSvU9GIdQPmvpCz87TNa+4/JtPpPqWT3j/0U98VaZPoYigT7HOBa9QCxHvoNokrt37xy+nPIovk1RBL9DyK69JibEvfuB8j1zSGG+XzvAPclHlrwrQNo8aEqivV2WGL5TgyM+","0epbve4Jgjz+eBq+pHEnvdPVQz3hdKa6xAJTvl+X170atSQ+CkMSvlwUlbzUQbe9a4qbvTNaBb5r3e49gZABvhPgVjwA0aw+kkWQPS7smr2AG5W9BURlPXMv6bqlpfy8snf8Pb14Oj3XQja/VhQvPk4i9T38EPE95Z8aPu96rL36L8S+vSHxvQlZsj4uUas7iViZvqZeBz58zAm+RH8kPTTSuz2p+SE+TcjHPTmSnb4fApa8T1b8PJvED74ogVs+hMYCPjRn0L2TnM69nGx1PcM1N77NegE9mMRCvUpe4D2Bj/W88+ZlvrLNe70ftGo9SB+kvuuXCL4oIiC+t4/9vUl8cj7qriY/Pl+5vsr0jz6lsgA/u/KdviSWNDsmj0o84pejvTO4KD0dIRO+1jEQPjZXCD5ncTm9maDHPulsNz6Tgic/bgxHP1snlr5QWxo/ocPrvq/WJD+heC8+Ss0LvALjHT+G9Zi8RdY6PeLKuLx/yHy9LDlKP05sCz5G1FE/amDKPh4JDzyCmQU/h1g1Pdth4L34gra9opzpPk2tVL5xMec7ZE1mvK7Y9bz13709YCYKPi+G6z6/jZ09TSGRvgKfMr/Kji08ecelvgYfBz/oc10+KxiHPgdR3T6FrKY+/OIbvBbujr2QaWe+iUFRvFiiwryKXgA/KopuPizsK76zDUA+E/M2u5KaGb60Jag9ULuxPfhRgz2/Mf09GKCCvsWr+rpf3lU+IBf4PQISfD3IZYM9cy4TPhrpzz2SiEI82tO2PW9dTj26Va89dtuRvbH7Xzu46by7SavouxLgt7zpeYs+HpBuPs9LOr4rdKE+5INRPT7xjr3yM8C9Vq9+Pn1Rjr6TKiU+tGvePpVuwj3+zXI+TaeSPiiP8z1NTvA9BA6WPnPEJbsRPAY+nDZBPlaQQz406+M9/4rmPqL0br3ki4g+4C91PsJrjL4TvU6+GW51Pt/98bvHfxo+cLHxPoV9c77E39q9S3MsPzEMBj53cqW8gOE3vcn7xbypzcM9","jUTqvLRhjb6lWaY+9NPKvYVBDzyz8qY8JvNGvdgM5zyQmlI8+B5vvlCYvL2o5Sm+9jjOPRMRxD7pKtU9sj3oPc+GeD6dffo9Wb0ivpn8hr66ToK901iRPmdPz72rkQ8+AVeLvVeRL77fPnW+zloBvzyvaby2/Qm+qjxSPm+yRb6SUKs+F+D+vMjk8b4J7So9ipLRvscR976ceik9VRobvtz5lL0IJc487zekvnzX6r0ZF8w9k1eIvYllyr7DVwK/NIqcPebcib6bh0i+ZNHyPlp/97zzSZm+dhpsvFLYwz29GAi+9VyJPVhjlr5Bwn+9BHTLuwNbyj1L/QC+BjzQPX8IzD3o50Y7SaolvQgZoz3ZYBw+DwwQPiTjHD2JXj4/OaS4PQzlMr5BaGI+dsG7PHa6pz2NhYc+bt2JvU/tfz3b55y9XROaPTo/Wr5KE/S+TqHNPHv197229RA+L3xZvIF4J72FQWu8DCqOvrkHRztE7xk7RObkPLu78z2+EeA9J4hPPVL0UTzdHIM+rlxpPr1LWD2r20y9r+hPPiEXpz2DARo+H0wCvSeDHb2nfcM9ec/hvJUTUT7AtWS+6COPPhGY0ryWgEW+8eZWvrajqb7QZFS+ddfHvXCzcD1dZd89/srfPF0GN75OKoc8CEnwvR/6kT74Nd4+ptbWvUdiFr46Y5g+e608P1akQL/00Kq9WjsBPDhfyT130Oo+8iJfvoovAr44/OY+ogHcPu1v6z6bvq09UCnkPDOXd779vy09OqxRPlC21r00Ima+ivlnvvj+7r5qUiS/yUzovfQ0BT7jq848QxgDP4mp2T7VdyA+cYMpPZFJRL/d9rA9b4yVPe936L6C/R+82NKevpQIOz4TUpI+xkIHPjCv3T7HOMk9hr8xPj1Mab5dnLI+l2rXPbADGL3SQ3y95ClpPuGz2z6VCl++3/HXPMYbQ7/6LpO+oY0BPsqu2z06a9A+8Z9Ivj9N877nL5y+QpkqPxnigj54xQy+2s+SPup1gT77kiu/","8+ekvj+Mhj2mowa+0tD7PQgh971sNbK94+B2PU/PLb9XP5q9vJ8nPmI21j5bIBA98pLvO2iRGb6kG109DEPWPWqKnr2IjhS+OGTbPb56bb6AT/y8loIiPZL2Qr4EmHm90L6gPgZ6gT6NHam9/oSSPcU8i76ZTk8+4OtLvmncmD6+QpK+SMFOPvtK6z0R2Vm+riYfvecEpTuL/Bi/s7cePijngz5Ucr29u+XwvLIeuD3gHc49Ph7/PPhQkDxYtoy+c6eNvCckeT6LjLe9l/G3vrpcoj7pjyY8wqmSPrbTXj6o2si+tI0cPoieJz7gpz0+Qx/DPRACtr2hOdu8rc2Evf/9jr162kW+Bi+VPsyaH77N0ko+S0t3PrlBAD4QGCU+Q/PKPucIwr3q8is9ceWSveH58DwlJvs+zsTvPd66Cj5i9nw96LGMvnJhML5Ui5a+hCT5PMCXGb55pI09A3bNPezwTD5RIPm+4u0uPXveqb4qfJu9THwxvfJ43T5Ns+S9hgT9Pt+gnb3T8Ww8HW1UPerxw70u7zy+g4uiveXhqbsz+GK+kTCpPoCMML5W7Nk7hHqjvWQDmz2ehq69dBsWPluoDzsEc7G+YTeSPZqY+T67Cdi9zS2jvRk/ED6SVL++rWOAPuyOvrtw6i++N6MJui48JTzlvtK9S7hcvjOb+rzO5uK8XNuwPKfTlr0okES8MqHKPYxm/by9wlo+I92JPk1aNz6qFIm9hdbVvQomAz8igZ09My+JPpFikDukgBI+vgoVvv8gcj2Ar2W+pTbWvX7kJ71+bqQ9WkiZvebNWT6vli49EjF9O0t5kjwd3829xAWOPvSc+T2dUe48kSuIPWov3L1e5SK+Ch31PMSAhD1HGG4/7oHSu1jOoT6aQhE+gyZtPsFy8z2pOJI8SfkovIeWej2TElK+Bgv6vWtzgryOSJ+8DtF1PgH0Ib7rtsm+yDG4vZLAlj3AkJ49SmFUvnilc73RKIu96hVbu7aUGb5EnJQ8Qw65PVMMaLvWCzy+","Wi31vu4RhT6tEwG/AzvgPn6k+jyrLDQ+OlSDPvTDxL6DdAO/FSw9P/r9hr1iupY+JQsGvghDxT0yGaK9ZCiCvUxTUb3ntpS9EEZxvng68D5AQMC+28NpvhZLV73f0JS+qW1SP/nmcr04acw997uNvjA9Tz1L7EO/GRxZvMOBa75HlyO/IlgqPsItPr5UkKE+Ex98PiNYMr/VMoO+0E47P2K4O70iYju+Vtu6Plw88r2xOXM+0ibOvifGLT43tGc+GbaNPme4M76wDRO/PYPBvsz1AT7ayKS+ecTGPX+Wmz7fxQW/GE2bvYL4IT81xAi+LI4NPbdn6j22iri9oFCavrsDSr5ilvy9VVMnPRg0n738Th2+MJAnvb+MFr6P+Ca+xreKvd3ZiTxf4gu+hiUaPsGmab6GquC9H3OAPo+ihLyyNvk+q81dvp2Vcz5G95q+lFwcPBBccj1QLi+9SOM/vnCmOb6pUmw9hkzaPptY8D3R/xy+syrWvQACO72HUsQ+QekwvpAm7D1emqY9teYTPlGJUT4drqe9PZX/vqtCAT5p9808+DOSvRt7i7waqRQ+Qg03vW2nR76OuGE+ZGTVvSn5qD62XrU94opTOlw3t72z45+9TP1ZvHgXur7A3jk+UcJSvvSTfz2Alqw9V8lBvj20oT24Ha2+wG5dvph0lD1cjwm+1LsQvnVVlDuQ8yw+SuA6Pj8NHT7FEMW+/HRbvTMKnz0IpIQ9B1lbvp2gwr6Zo0E/yLMZPlQN876RxnS9mke2vSf5Lb54yhc+BagaP/Kq4r3RggO9kuqgvTg0SD1NFNq8zVnfvoLbSD2FdXO+pLEIP4d/dD7c7OU9N4LrPFBQrb4rbIS+BYe4vtc1xr1ujZ++0Tdrvumf173G+8G9ldy2vclwdD7jdQY+fHsNvZKyrr7PX2e9eAJdvlM2oD0MhxG/lY2jvXlgBr6MGLy+hB/NPpIiND1+4E69xdSNvQHjGj7v5ac+w3WhvuGVWL5zpfC+gqblvQjXFD9rwyq+","QPAQvp47eTz9Yby9DZnDuiSwoL0ccTu9MFvUvRtyNb1mKiU97GBZPtaygj2Dv4Y89byxvIrIWD3PJyI9Q+olPtAeaTtl7T49UEEivvByFr3J1La9/FENvr+H3rwBhm49PWUYPQ9+J749W+09r2cdvq9cXL35RkA+tFx5PqWdtb27FG+8XLOyvleo1r1DAqY+JpRwPJBLJT7yOY29dO45PjWamD2LlIs+31WePdI3Bb5Rau49l42nvDaKMjz81jO+BPuJvSxdTj2Czr090KZCvo7HPj6cQ04+UpefvtpyFD3dBJ68mt0ivORWJT3fek88DeNNvOqRYz7O69w9V3ErPtcsE75GVf69hDEqPp/yVLwDOra94P+JvuP6kr6+9lK+y/3Cvkc5Hz9zRPG8eKU4Pm/Mbj5yOdq+Dx/wvXcIYT0P68e9KuhYviWCD76KFo4+Zr7OPRHi8T6EwK0+ENSLvlLvaT830QA9d1iWPJEJpT7sCza+GkQZPi2/Hr0Nyoe9pAI/v2EvGb6E1Fi8CfymPj6S270O1j47r3BGvgJWHj6iPCS/dQ0wPYxHKb/3KJs9AB3pvs6C7r7oNJQ9kLwYv1Z55r1eZM294GCavgO7kDy5Yb0+U2yMvtZ4nj2Fwo++tURJPWfXKr52IgA/su8kP+DO+Tz/qES+e07xPck3LL/zauA9bwgrPleeAD7+s2M9IvEHPoE9Dz5T6A885eJ7PT+QHD5TuTE90w2AvRdzADvls/A9tsSKPW/68b02T1U/vpExvm5hxT35c2m+8nAxPmwd+D0TBmQ+IxOrPhzpXD73YSQ+/6E3vmJl6b6ffUQ+4P02vJvClj0j5CI+XZGIPo4/uD2TFRc+jxbpPpbqUr2vFdS9HchrPEsFmj7R4X8+am7UOyZXVT01bc49L6ASvvT1Jz2NPtM9UTZ7PlCesrs1Sc0+bUo7Pm8IFTwbHhg+hx42vmrMvz2DRY49SPNKPZy3lL7G3oM+tG1oPS/Mij7ouIS+KocLPxqwBr1ZaqM+","KZiSPYW2Ez7jeLY9/mcmPtZqZ7yWWOO9GnfPPv0xbbyVdNo8AO+MvO1Z6bzwrXY+TTcpvwKX3z4FZr++7HHkvdyQY75sw4C9RfUGvuWAeD0782q91zNSPOAHQLwroKE9gIkfvQeHpj7FzkK+2sb/Pcqa0b7Ie7s+CYRAvkDxvD4UMSa+9pJ2Ph6CvD0vWCE8wFh1PRxxKz3ZPeW9RuSSPl7nIr76va49asb3uwaGjjxP5W69ZIQ8PoPSr72XdgI+RhYavnAQ7L1JTME+svcwv3A0Gb8UWJs+PQ+WPXWaHL4+lxI90ZEjPLWdZj6Dk/O9KG2qvZK2hr3wBcC+7moQPZsmET5AQKQ8INn+vTbvBj4/eoe+CLtuvet6D75Z3xs+H3pdOw8ChzxP1QE98S64PSpKob/j1Zm+q1e6PKn0ML5Yloa+ezKIOvzipzxlWf8+DNQoPUGxrT7ZNTQ8RddCPu5rcb3adBE9weXTPVf/jT7TE90+imWqvlCRnT1pKHY76c6Wvjhio772+I8+JNkxPYWg970e4eY+MsejPZSNcz2WXxy/3bEIPcwS2Dx4/YM8ujAtPgNWwDxBonC+IBOSvTLMwL2PMtk9bq0/vUOryb4J6dG8jhRDPU32l7yRwjm9MPaUOulsBb6F9IG9DJyAPRSXL74jqac9A/vwPPFdxz3Fid690gTbPYDMJr5yfgC+rqFPPpZgnj3GBA8+RnfdvnnasD1E8JO+sq/evtv2aj6ywGi+jEYCP3ew974a22g+mmVDPgcN4j4w11g/RQV/vhv3Dz7tdA6/Za3KPCvAp7taaxe/FBrJPgZaVz0PQYc+YqEevXJeJj5jrJA+OKR8PuSqnz58cgI/x1G1PvATUD+wfDQ/3J5TPSrV8T3hTEO+z5rBvr8VNr0PhjG++QxEPYXzej/w8VC+ZagSvRCFJz0GqRc+nkiivNqUPz94ibK+h7PDvlEv4j0f//k9gv8dP1vSH70h31w9ENNNPqpBYr45B2a+o65DPcwHd71edYm+","vrDDPQHVpb19QF6+u5o/vRmAIb51zhm9AfJGvkCHPz7jeEa+Z455vvTUaT3LGpK9HeeIvrdtgr2jPiw9qcUEv9YLhT5VXMS9yl8WPiNvZr79Ij++yCg2vnyJm73J4Ge+7SFmvRnRp773B80+RiSIvmycer3qL4c8ARNWvisJjD2VGac8mbcQPRtjyDy5e7k9Q9z8vSdgPr7KtEU7cqYOvanGf77w/ha94veCvjJmmz0YQEe+y2eFvfyIsr1GijO+CleFPlGloz37mMG99WGYPdjJAj5uopG9zNngvFIbmz5xbfi8zMSCPSMGub4eKwU+c/XKPhZ6PL8XOIe8zDyLPhVZYj0FDKm+L8nXPtQfFjz/POY9DGKPvbU/NT5TrXW9Fu4wPbcFWD3PvL4+yxiuPgqbsD78YSi+v9GivgclGj61fWa8QJdOPjUgt723VpG8WxeePUb5WT/7JK88XHYYvZMMLj4s4ok97fEGP81HQj/Oh+8+WNNxP8n/AD7mbX2+tXKuvsuIK77G7NQ9ljMuvj5lqj4+p/C+AZwIPtivHL6zXiG9lpKCu3DUy77xmqA8fHkCvzrTET2g6i2+BiL8vmN8o70S5JW+vwLRPEXM0L5PwRM+UqSAvjC6ZL203yG+6Q0OPmGTpD7cEVs+pBgMvsL5CL8Fm0K+fZ6sPj7Prb6ggQO+F931Pad02j3NLUg+I0+YPV96fDxJ1Je+M6MWPM6wrj0Uvpc7uOwhvgdVcT5clrM+QTIJPzzW4T3OhLY+keQ3voq66z0jT1u+AVufvovP67tx/zI/dQbYvu8Oi77aLXK8uNnTPSMSLTzEFZg+6epSvAnhmb06Xom+54WivWPe/b5UuLU8/GWivbddGz41iHu96AqlvUrPbT1GF129THH0PfqJjT3BvAs7Yo9rPcg7KT7qZH89vAv3vCPqkj4rY4Q+CtDAvSHgeztJYac8Gzzcvcv8CLt91wE9TOn3vX30DL7INZy+ckCPvQmlPD6et/Y8BSHYvp0Mab3W7Gi+","NHvDu99Crr5Lz/i9b0ayPmgm6D3o/hw/u8jJvmKstr66s4i+dhqYPRi0kTwuYpo9axpZvoLKQT5JY58+sxBHvgrPvL622zw/jB5DPsMI3j4doB+/GQ2iPLB46D7pKoG+S9KVPytbOT2/Yss9xqMrPg5TRb+W4Ue+W99OvZYxxr5sGgu++Owfvzw7nb4Z+7C9IDK/vL0W2L7RAAI/sB8cPqIED793b8e+2Lm0vteSs71vK6G+bYrovhBegD6+KUQ+cRKGPqXfXDxmtk6/o9B/Pm/2yb5jcQU/xT0evfQfn77IqfC+rSZTvqhr9Dx6iSQ/2NQlPlSIiT7Rd60+nHgQv4PqAT6H5vC9UMmLvgRz8L3ZG0u9mobqvAjRKL48ZwY/xSkDvllWQ77N9Y2+2wsBviyGj767QVY9asCovWTPXL7K1I8+E0TrvJLTibz9hQe+QrfNvujUab4cy1W6fuBqvgAUkrxZhA09eKf7vu7MBr7jiz6+6eamvteRGb7W5qg8bkC2PZLIYL5o2dm9z1MKvp+ydb5eFsu9ARLBvZ+PD77IZNi9vgSCPZaGQ75wS7O9ES6yvjhuNj6EIKO+suNEva/wLL7INha+bKAEvp27oD1tT0i+cesrvrnvSz7JCCO+ocBHPohnEL7uvNG9wXzFvG/RyzvnKJU+jo0uvnBCwr29d/O9x3X+PdUizTx30aI92mG4vc8aGr4lPMw9lvKIPduTwr56gtM9soSYvgcHgT5z7Y69+Usav/HeSrwmLya/gwb/PiDuF7sbJ6k+mpUOPrT7L77VrC4+oUFmvdmRtb462jo+n+L/vLpeAL8Trwk+Hm0xvvQTjj7QZMe+kJERPrlRlzyDXaM6DYUCvmmbVrsIFLE+CuWCPiu9cT6jLJS8uhWbPNJ6n74nl3s+THx0Phz9pj3gtAE9dDpRvvMajj0Eg4q+Qw23Pml2nz3Z/QG/VePzvJJe/j2Pbmw+6WHgvsv7Kz7J2p8+hO8kPpJNhT2TbX+8Q7d7vUzQ7zzGKYW9","BT0uPqyR3D1Hyt89kBbRvXtTpT4++l49265WPQXodr7rGJW+ceGfPVBmoz2nPAI+ByX+vu6oeL7dD+Y9KHKDvn69Gr5J/VK9E06svXFpuj0cgdq8cv8Svgg65j0h7Ku+IOQBPYxiar6pFky+JjxKvdlht72ZaVu+LjZovRVx072fZiw9vDqlvMSDjT2i+Ju9IlooPuwEVD4brUm+280NvvgSLr6uxW+9AUWLu7fYVr2gwBQ+QnQAPh14ib4jlYA9qCuFvm2yDb2Y1HM9uapdPvtLOj0Es567X2YvvhLbNz9h/IO9FI86Pkeyfb2ptss9LbhpvCj54LxmYSw+zKK0PqPcPD7ESCe+1ef0vmPgTb6qEIU+lqbPvvqfpL4s/9e5UN05vlJtAr67gjK+/dISvvK8KrwFB6q+RnYwv8Evor5WROS6mN5ivyWV7r6mUjc/OiPPPih7VT/6bqo+cyxUvP0pKz9qcvG+EW/gPedcIb7liQK/jAmevEJXwD51X7W+inSxvUm/5D67wNy9/I6Ovurhm77LIUi8C6Invu1FU726W0C/Za0iPkLw/r48RpY+64Ifv5p2Qj5pntM+Gwahv1YCEj+7KHu+OIQQvomXUz6rg1497HsOPxWd2Ly570c+WuUYPvNUA7yrl4++/SL3PvldHL2jz3e+e7URPomHkjxgkK+8nfdbvR1Mhb0j8SY+tQ/WvfRWO75e5Hs8+nlKv2+KZ7yQ8kA8GZqOPl020D2U3uy9r5MmPuC5JD5P5f26LeoHPgsHgTwkeO8971XNvfc7cb45ATc+rnDCPGyBJb1uea29SDH/PFjt4rs7FQA+3QSBvFvrpT4CPmw9M52Rvbnkqzv6o5E9XW4fPoF2kzwtc2+6v0dHPuaPsD16ctA9t6koPYlBzDxqjyi92v04PdcWir5e4IC90eGUvgRQ/jw6GR8+S+22PijUgj1OxnI9GxWJPNwSA77xs5m+AlyzPtRuXL78Oee84X6jvO2Iwb2+8E+9fCTFu3FOXzyW68y8","eHFjPoIkj7wDoPU+6GC6vL5QIj7NvQy/tbu0vJaP6L6e3C8+/uZvvjihCL7ICnk8W/GVPdCRoj5niHM+RJ05vygkqb5PZ+e9uWnMvVCqiT46ciA9m/UxvbWU4rwfiq29CryaPmx22z2p7Gm8G9jMvnSM8L6NkWm9DwKtPfOGD74p/cY9lVbxO/wt773DaJG9cWEePr+BPD2kZLC+WkcmvdKLKD3dXG69Wp9ePVju2zwN1vS+juqsvcsj1r3o62O+RLCMPBolOT4+v3I94/C5PTd7Rj3ZuJ4+MFHZPjDwA77OCF29sXBVvmCRAT5HTtI9X3O9vXYCh72JWJG+LNnJPGnZED5YQw29DO2tPksouL3cYHG+tK92Ppc7m76JUHu+pJxlPVhmPL4uPJ69rWvpvUB1Ib7qlFo+w1ikPW+hEb57fBs+DieNPZdJcr4r0Bs+toihPlycWL2fPRG+F1HRvbqEZD3gUu2+LBHZPWNqEb5kGOA9F3WzvEwMQrxwRuI9Nm93vqCLR71Tj8Y907peu5IGKD26F3c+/JwnvszrALs+6cE9f2RoPmS5jDs5OaA8x1atPVJ1iT7RaiU+LpFevUFFOryjxCE972JKPv4cdL5VgP28gJo9PVuegD6k5dO90EUQvoKxED6r71i+RGjwvFK6/r1mcAa/0GCQvXtF67z1g1k+5xbZPS1bxT7XOhE+A4pAvcpVSD+dzr89Wen/vkq98Lwh5J6+9qR8PAvMmT3mP7W+mHNTPjy9M79Twj49rEc9PVnZtb1lUpk+Gty5PaE8Kr2VWUW/Hqf5vidYor6h4nQ9fp0IPwPix74AeKy911cKP8YZ173r66u+WGg/PkjNoTylQoO+u+YzPrNl+T5KYnY+St6qPQE6QD5kDJs+LU6TPqjvfj5rvu08EU0fvsCE3z6fAig/avTtPvUKwr5tjj4/s9yYPOtygr9YFy89HpK3PYIuIT7aJeM+VBEAPsQCFz+O/t295n4iv8aaMT6Ssf49Lv51Pr04Ob/3gdw+","7uJhvnAgSr6PTh2+XWMjvNPomb7SWAK9DicCPvZmBz8F6w2+97oEvg+ojL18Dwu+2HYzvkDhC7748+u+MpcUvi4CYz4iZf297fM7PuuV0L6gGIC+WM5bvZ/pSj0Q7Be+rzrCvZ30BL/bRRy90AygPlRrbD4HR8e7tmUlvj103D36hkO+es2tPZ/DeT7XiHW+62hePrmyhLyq4Cm+4fUuvR6F7LiowWK+ZWDDvU4+Mb7JZyI++EX+vT2ML7pxXaC+v8luPZdW0jufea28xDlEvJ/3ab23K+u9m9wNvhTvzT554jG9bdSgva7d/D0boy0+FSMdvbOJnr4/c00+iVq6PuJbG72ASKe+QWs4vnK8x719NiI/30uFPX6ozD4L7aY+ZIUNPnY4mT0KyNk+ByozP5rAe77Q9y0//3/Cvm995741wSM+5FlRvibudz2Nsc26V6rVPV7FcT7iqU09eq2dPlCNfT0gDTo/xwKovoY+TL4RaT8904seP+FLvj6MUkg+mm3PvluWjr5kpj8+EKFZvX5Qr74AfwG/QKiQvF6QOz5OX6e86lcCvfgbX77cPrI8V/USPj/n2Tz9S7u+Ir8BP1azwr4Hm4a9n30oPHJvP74y2Qk+Td9rvmljhz4ZUMS+jxiuPR1NdD4q/c8+4SGLvn7PC78Vi8a+JPEbvms1k74Ku8293YLCPeCj9L0UE08+qntVPnz0gr77uLk+fwGpPVQUXT4vqai9o2XoPNUh2j5vF54+XqSIviwIaD6tPK0+SaJYvVx3hz1/mrO+oYgVPuV2dT5i2/M7U5pSvnoWgr5tuyw8/Jv/vfmGkb6KnI6+A2GlPaNls77yGrQ9avyBvkIsEr4qu+y9b7iLvg4//j263cW97bWNPvW6GD16Wos9juCEvVQjED4t/8c9cvE0vZNa0TvPdn2+5YJOv4EwD709VMS+ZYTmvCqdPT1qxdm+bufePPX9dT2siv+8UTgbPeLVrTwua4++jJ/LvTTVkr5Tp929aUcyvW6DlDpJt6k9","uu27vZKNMT7cg7C+VVlEvhkML730Caa+DDCjvuLJvz7PMzq/11EePZX+tD3SIwC+irWlPXApU77h2ZK95bW0vDbBqL1Y9Pc+xN6uvq9VJj44VvS8KD2uPxRYgz7PzK65SPShvtByJ77xmtM8p7BxPc1qkr5Lb5e+AqdYPp/mkD4PUBa/Ah8kv5Ptcz4Ut6k8edOaPhZG7Dxe3Ki+T3GDvkzGYb+MnuO9Vmm3vuqZ0D3lEiC+ihabv3ZXmL7cYJy/fkF7v8MCRDzf/Gu/wwDWPZlcyD1rUAI/lugSPZVfir4Hwwu+6cWDvi/yEz9pnx8+Rwpkvth/cD1CH68+Q2Q/v4VMSL5bQge++smyvPFVZL1uKgo8T8k+vn44MrwLAqM9LgUnvkqSDL6VubS9zakyvh0GgjysE6W9U9gvviuVMb263QE/9jWKvZcEuzyM8j082EVTvC4XRL79v3S9uUyLPEBhH7xMYSe9EMHWPhYAkLu8HTy9ly2NviU10L1X76W7s/umPeNPSb5ilba8cUG7vdJQpjyDqvc8l2otvWqAKb7ljGU9A6ePvRp4iT3owBo9AM+OvUrvLbxdT6I9895bvfzI2j0c9RG+p07YuxubZL7yKSu+OLihPA6OFj1EPv69N4UJvC0ihr3oVim9QC7nvYvsHb2/tJE+jRNrvRQggL0dBYG7ukM6PTCcJb2MmQw+BMDQvfLwhrzB0Di9NDriPSk8Y7p8/gY8rEUcvPk1Hz6I2h+8VC6sPflCT76g4KI8KIaIPlTH6T0nCPG7YsEWvz8Vt736wgo+W5iTvHI5Nb2mfXc9K1gTvkbtiL2kXr69IqAGPmCnbj5AVjc+M5WjvRVe4L0Ssy+9SHvMPfgBLbzX4La9kr6mvpVej7yEiC2+R7ZyvDQhHD2ORYi9ASi9vT0VwL0hzpe8U0S5PnQInb7cb1g+M9Javsaw/bv7GiQ+BnUUvsSDf74gZcm+luW1PZeYvTsmZGm9K1+BO01FEz4KUbo+N4T5PbsdSLykdoM9","9skmPCUyeL0WU1O+nfIivYAb5L0JoTo93B3XPZN4RT0CbCU9lm7ePf8v/r09wq29aZKoPs/rxT5vfBo9GNBOvaNOvb5yDCi+JxdBPk8Ypr6Qf/y9qW+yvD6d4b3KVMu9TSKwveFguz6yWZg98TNrPgDU671G7Bu6FSGqux55tb3ethA+nY2iPWw26z0RkYe9GWYdvh85yL2xyie+RIfbvUIKBD7/6Ea9mbDgvMDwDD4VQro78wW0PR0qib7nCbs9k+YEvjgvib7f8ps8rksKvMVZ7D0jnAG+W30/PsYgt71BOrE9NVIyvUwddzurTDS8naaovk5pFL6ceUi93EqQvrToAr1MtA4/m6YpvXahGT9Trmc92vypvqTpn76fQ7K+4k5hPRlrwb4Jbee+3WWWvt7mgzwk0Gq+/9MNPzyST75LZri9gTnWPR1MLr+a4IY+z4DOvtDSGr4oRBi+dudgvSTFJT9dPZe+YX/JPD4AFj5SSS2+B3CMvlCSlj6uTGC+ZZFqvivRCT7zHXi+CPOVPl6tAb5S8ba9TjMKvpvdybyHloO+Ig94vaS1171gJQE+01EePpGhwT3APIC+FQHlvuuoLL5+nwc+U0NqvpsZlr36V5M9dsqgPux8672ug7y8oe/7PSeEuD1rR1k/5bGLPRZgab3GtKi+HjYSPgykxD6ZNJO9yZKEvbd3lr3wQL899IOAPK7CCr0wsDY9xfhfvhCgwL21Y2E9SFZIPg4XCj7WtqK8z5cjvWVpMD0/ZuM+IeypPqB89Tv5V2M96ekcvX9QST7M4Es+UbfPveuq9T0ZYiW9WyoRvWQzYT0x9uW9H462PbiNxbu1oia++zK7PCxsbr2j1fK8rOMPPZGcnT1pTfK8f8+KvThDBD0NwUQ+Lp6cu0IvET0ufea9GjD5PU5FiD6hRQW+GuybPeCcRb5/bv09Tw2qvZ1Xl70n+jg9hEy7PQ+YJL50NIs8JZo6vQLmC77zQ3i9nj/+u9TukD0fgNm9ihIfPmWnzb2lA6a9","D+8RvTf06b1vDMc9Lk90veuBVj4y9YA+qWm6PenW8b0nj5Y+Z2GRvfuO/T3eZ4W8C+vWvIQcxz60HeQ9ormlvAvUg76hk4k9dxdOPC2kLb0mEk69nP++vWRF0D55NXm99Mg1PS6+AT7Zy1Y90PDOPdgBJL5ZGy++NJQHP7aCCjxf0sS8okBWPghg8b27odk9Tb4ivefFkT0aFS47UPkvPrwwizx6U2U+7zO4PY0IeD1S0wK9J/W1PSjkhb5wDyc+/OpQvveFIT4SERk+CP/hPmfN1zwgXpU9yjkzPlmjBTllW6U9Ns1ovnT3gTwIpMe9jHvEPX1KO72iAQq+SS8NPdSXK774nIq9ZyvPPTaLkzyxqD+9wgo8vvsgGL192B28MDhpPmdIAryCfRe8tDG8PcyQKL31NUo9JHxHvVxt+D3vLbE9X6h6PrPmuD1hePc95bHhPLEwELwB+ay94/kpPcjrOT5Qa02+cyXCvd4zAL0kWYQ8T5noPHHtsT0Jvv09m1cJvj2mTr4Vh9w7Y7cIPiJrMj7zOwS8P8ZCOzoK2b1fAQk9q7LnPWuPOj0sHUi+gZYTvNi7V7wmqLO+ZLTqvADAlT1qZ/49nIiqvNHovr7eKow9Yvf6PbLjq70TMA++5eLdvZeVpj1kPFK9KZMyPGdswrtpcZ49ckrJPTVoPr1llQg9tk+kPmMxDb+HHwe9Z7HYPJY+hb2kjeE9zki/O3cp9b14fX88TlqRPU5coz49+T89tatIu6AiYL8vnis9uFYRPk9yyz7rjR69JC6kvTIYILs1LTk/LScSPr+VFD5fRCI+bzievS+2lbwngHa8u+2yvcbBv76tENs83iIGPyZfx774v549s0FXPvWnCT4TZSQ90DtrPZ4zmD48trM9YXWgPVP+EDxCaQ2+phmBvV0gRT3gijg+ww4yPtcYAb60FhA+3T5vvSw4BL8xpYE9YjhBPp0PDr4vAK09jJURPljG4DtCXrM9DjFtPu0ikT1VRxM+ilpPPm1ihL6QQKc9","p6OcvaVRZT2fdxU+O/k5PvnOqj0+TZw8S2UsPVAz6j4jA0o+hDffPTiAsb0W4749Xx2vPfHhBz/84bU9mbaXvjFJFj4Mtgo+9RCavWiSqL0lhKM+ej2FPhJ0wzuTdjI+PvkMvJ7Xwb2ebSM/Idm6PPNwWz2Dyzs+LrhVPkZJVz2GyRW892KbPgXyqTz4hiO9PnWTvPoGwTsN1H490RehPWATGj3mXra9/MmRPPP6UT55CBg9hA5gPcRbLj4+DiU9RohuPg2F8D33UAA+qMavOzAMnT1sO/w9RphEPZb37z2dlRQ9IeutPenhPrsHq8a9VQmdPV6WzDhTJ9m8m12EPg4LgD3pnKm87STUPX2cGb4JnQY9VKLCvJCoHj2RhD+9ruhrPYftjT2CTjE9I19KPAsHOD7HhYG+uZwtPn8dqj6CZpO9hu3YPYiGGjwUe/69tGTFvFVD9z2czPE90a1hvrI6mrx+/X28gidwPryKJ77Lx68+iNDmvN0iZD4D+nS+lYOlPgpd2b0+6eO7mydQPdrYODzB1Hy+Deyiu5pwrz0FRY09hinePQQkP71+mOC9e1hUPs+ivDxtLdk9wiYtvSHEDj5H8EC+YlQSPjzXSL1lUxm9byzXvOHen72M7s89VKAKPllbJb4r8o+9r//YvcwdMD7cxTo+KkfmPTAHYr0WGTS9uQmSvb6wVj15CFI9YUQIvX59JT3x+5++2pthvRlpnr3ATLK992JxPXg0Cj62LZw+bV1rPkuWlb2WyAu9DS3EvMCSQz6E6gI9Xah6vqOwt70HEPK8SZfuvVQkvj48BdQ8nCSrPmIeir7AaBs8REOpPI2A2z362pa+JlVtPCdz/b3XMjw98daQPs5q0bzck989YDWmvoWFr7zGTOQ9verQPHyhrTxpIw4+QsHRvPs13r0xX+Q8aZohPLJfZT6HyL29XLofPnSUrr2eRoI+eAdovu+Cmz3A0pQ9JTJovsx0Fb5MBqS+qGa1Pec7Fb4Evf89MVGgvUoMRT3DmYC+","HYNiPeUUAL538Qk/3YTEPnyYQL72YEw+eZenPoffmj7QBAo/rRGRPtEKcT41n2E+eyJRPtsjTz78OwU/qwWGPiAO3j4/blA+4FTjvp/wZr5qrQe+mA5Xv5tqYL50FA0+FN24vj2liT6iDWc9C4wKPvodkD5O35o+oOZHvt2L67rNJFU+o5OJPNh+zj2SLy6+WX2APYLSLL06tAk/bnQCPXiugj4fKxg8ipWPPutjXz0mw4c+xMTePvY3CT82wCk/up7cPPOUOD74QZc+ys2APp/pXD7WU6G+C1EEPnT3ID69ky8++YvNPTNwJL45NBq+3gRNPkFd/T7GrBQ+FYXDveKf272C6xS+f2tmPt3/qj0Nq4s8ZTxTvVIe5rw7hYE+VWEHPT8X/73BfZO9YY0sPt9IoT29YoG8mZGcvEulUr7Z4Au+WsScvXf7Zj68EGq93vx3Pn8HeT7tKlg+ZCesPT5WSr1ywxs+h1CJPZH2YT1um8Y9PPQVPc86WD7ThKG9TpT0PfIICTxiC0w+8k4rvNq66j0Ee2o+5pAtvjX/ej3qFQO9EmZMPZgS5T1DhKm7Asa0PYztWb1b9oU+3UPCPKmYPT3oBBo+s7RiPnwI+j2gjjI+DKBLPhy8R70qoL08I03wvKg2u729/7o8yi2XPOHN0z2lDUU8c3JWvcA5yrxWulG9a6vIPsD69r16k84+URybvb5pcT3b09O8fhSaPe+whT4hitM+4TqrPgi8rzy5C8o9oIjQPqZM1b7AKyQ/lW7pPi+Cgr2GElw9P5Zrvt5HJb6X9c69y92KPswb/T2GGSG9VLvPPTvQML5oVcW+GCUFPoFDJL6a5GU+RIULP2BIiL74zau9I9iTvpZNIz26AaS+CD3gvs/4cD2edvQ+NzTpPoe9hL2WlzS+mkXPvhixub2UOQC+20OfPmoF0r5XMFM+TeONvnAcSb0m1bI9MBePPmvQl731Tqe+Bx4GPoFMOT7V/qY+gh/Wu1vvET7ZZJy+Nj4Ovmf5lD4zCNM9","Ff4KPWUrKz7xFAI9e+Gjvsv6yr2k7wu+4M95vOhHujsXBCa9ihdevVOZQ76pqjE+Mj0MP8CLwTvDy109/HnAPgFCCz9kzyY+f/4wPcfkwr6dZv09uINzvv9+Fz1MH8895UjyPbEp4T1zWIM+wuAlPkIeDr4AtGe6XU9GPsvY+zpyVxY+2Nd5PcWw4rzHNDK7aFgZvrkMNr5BrGA/b0YgPs7L0L0rYhC+V9AuvqoNVz6CagS+6+GoO/kzJb7gPzg9bi2EPg/LwTxMaRm9s5rOPWoOsT4lROa9Q1I7Pvr3AT0W4Q4+J/IiP9utqLxB8mQ+RRH8PZWDhT4jxqk+jM7IvXtfB77d3b6+IRwKv8JWTT3Uvgq7t3SVv4nTHr6qUa2+1t0sv3Jpq75zfOq+2T7sPAptPD5WSwm/4lZ+PkRqaz5qUe08ukWVviR/hz5NQdg+o7CzvmSbaj+aAIG+ZrI7PnKnNj6IJUS+W6a/PS9FOT7nyQM+9M+mPXrlDD6GsT09mjtRPzOmzz2nrWY9ncSJPlzUeb5mVXk+ICvBPHPAML592a2+MNsiPkIxGz0c8Js+xNEVvio7m72uKKI9erdDvxa6xr6i+y+9jHWhPyDjX7w98z8+E1vAPjc4y72NnPW9YdfsvQEhCT50AiU/2vDIPkH1tb04aYa9jbXRPScvQD9XwC68gUZLPKQ7UT5aVCy9d/rlvVo4XL1Ewo68XOadPhwajz14ni6+CYeyvueoqD0BkoK9tAHxvVr0x72hp5U+OJ0HPfHhCL0iVNe8t85dPYt9xz3NNOc9m2M6Prx5XzymX6s7c144vgql5D5hKwU/IzBpvW0ssT5W5VQ+63aivQtlPb6E+gQ/jN/HukT1QT0rnKU9DAhrPj1HGL7LcuI8g7Uevlqegb3aTY495nUUvsqC5rrqmwM+OxJXPRHBjzzTxGQ+to5BvU6X+bwoXb+9MDlEvtYAcT0Q7rO9vJfhvAiZzrxNGrg+jbuTvG/KWb0dY448QZ4ZvrgpkzxX+nq+","4SUJvsjFmD4XKxa/pYs4PXp0Zb7ghq++dp7wuteuKr4Xq8C9HCeGPh9wAb19lYE+5WobPmATrb4XQKq+zNgwvd2+Ar3zWSM9ZO8YvtrfST5SBlM9e8UBvzPlmjzFdW09dvgau92rcD4+GkG+aZyvOk96gDwH2uY8s8ravg4KID9O/8W+z+ayvkh2Ij27yxo9el7PvBYS/L0p3Au+5KnRPHt2cD4ktjE++M8uPlzDnr4Mf0+8e7lsPaw9qL7bt1s/12GUvvKVqj1Xzo+9jIs3vxyIPD2geKw+G+LqPfM5l73Iy1A+tkbAPmyfcT2SdR0+9JWAvtIhwz3Blms+ZZAAvlXdhT18q+Q9AzruvTl0S73Ql1y9iUxZvetjAz7c+NK+xiA/vToGWz40OkU9c5XwPRnGt70ljtC+fXw5PZocFj1K6kQ+PMIHvpTMMj5hGwU/uyxlPkrWbj5VOQw+pa2XPcVIXr7KWDe+uWgvvl/pcj0Yvr67BBNNvtwslz6CYBy9Clm2Pf/BMj20fzS9s4zXukrTej0OMrc+WoZpPnvVgT7Fhay90YOxveoWdL0Xx2E9GwNVPg0xm755FT4+86zCPc0Yi74IxRo+9ouvPfBMrD6siak9f7XwvPNfnL2WAiA+MJHnvZ/4ET69XYo9JQQpvaCAGb1WGXE+xg1TPluBRz5hqrm+PR3UvbcIID5nh4C+Me62vkDAlzyksHI+9SpQPbxNTb6VRom+5E3lPXD2Yz7GrGa+iiEPvv24ur0oGXk+5r4sPopAGb5tf0y8hT4Zvy6Wir1BKUQ/fb5NPsrT0TwPnje/GV8CPtO7U76BzSK9vvhQvfZXbT8+JiI+q4T/Ps/O9T54dvm9g0sgP4iTtj5YvVA/Hn/vPd6yOr6mKe6+YieMvglGbb5kmbu8mTJxPnMtEb6JE/W9z+OqvhoZmzyF8/m+X+0EPxrJTT8r0lm+BFfIvciBQb7UQKO+aacEviEALL7XubE+YoYyvuneA7+8h0K+Ibx7PrwsaL57f34+","0RGtvZIptb00Key9OaqpvamjFr7dVzi+0VB4vsQHJT8cFni+AGgcvoW7q7ye+fK9cIgZvpmKir4sKxC/mSjKPkCP6D3EdZu+8qhTPpdJ5r2rsHU8qgybPb+J0jymldk9KRUFvlGMAb9IyxE+lrBZu4HY470xYmK947tovS0NUDtMXPS8j4ToPf4LED2efm89FC2IPbntab2X+s89i6EuvknqW75Vdju+pla0vQC8pb6FzZE+3R0VPu9fUj7yP329nb1Iva0FWTzXhnU8SEAPvVOQJL3jEGi9FAUGvsihRT24Ly++ty72vQZTAD5XNTC+tZEIveVHEb7epFO9ryO3vcPg5jzJ5DU+yMS+u7k3rLyuEhk+Oxm2Pci0bj4cOHw+WLrLPXcNSj6r5pE+SfWjPrGPTD49KYu9GGBgvhdpLz4V3Bg+zjwFPTbMwL7JK4E+418SPK/8xr6ApgQ9EfBlPrR4LL1hkbI+SziZvTrIMb58Ss49lumbPqVVir6k0mo+PrpyvnkW9bsyQmw9QXsAPo7ohTySI0C+GH9KvfzSHz5khjk+jjcWPpXMQj1IA069sRMPPYmrYD48bTi9+GbHPWGzkT3OerE9b1RmPK5LTL3x7989sOARPun3xb0ZwgA+Hq4WPnmGHT4rE5A+Y355veUDAjxcdnW9aPsPPtFVgj2ocLS9CXSBPi0ZDLwsS4a92RgRvvUZob61644+VwPAPf3Ovj1nKOc9mO5BvVIGrT5nYw8+GdO6vZTf9Lt6zRI+I0wxPgPQMzwpwRO+4se7Ouv/Sj6K5gk+wxqpvrrnFj56l8o9nqDhvCA23r7qZjg+bXF+PInSnr5tYAs/KmU5PvxULj4igCw9aQzZvLrszbzoMI+8t/K6PG7SJT4ymqe7rBMlvSzWZT4ou+a9fjPyvfMM5r2u2Su+8ij9vuB50L40IXs8aLVXPdrVQ7538MK9bsAGPuBGFz5x74u9y44wvvXbiL2v+yA+AaONvVH8QL7bgQO9XZAUPgYlxD062sy8","EtpLvgzDdj4HDJa+hR2lPoxHzr5nhym/YUNrPvXtvj7bnaC9KV8cvtrGcb2r4ES/I1IAvQC5yL6oow2+n0UGPtECmz5L0NO+Od6dPmL/AL+HnjM8+/SAPkMMqjtoixA+7Qghv6oP8j0QFdO9AzjEvej1qL7RoCY/jkXAPvZmEz/y8W09D9RdPq6fGz/qLh4+e/B5vG4IY750UaA+sAVWvjHh3L7jT4G+lmqxPTM5gj1FIBs+G1F5vhcFf74Fvg+/PDCcvs2k3TzoOs8+aQJwvmJDmL3pZII+EEGkPiQSkz630mC+6W8+vkbs7T3UiIC+NrEdvlroYj5dQwm9o1YfvcriaT5lyZq9wathvZuuEb4Cmnc96Ga9PfeOvLwa220/BeMwPeeUXLy1YkQ9gdYgPaPWmz0x5cs+1F55PC9Kg768osy+m6OhvL+C775Uo6g8+gZcvgnFdbwO6Oo8nbemvSR2A70NXJe+nyY9v4SbZTtKNBG+I6cXvKNh07vyQ+y94ad1vs6ReT6IA2C9Auh5PVIOlD2lBwk9ZKO0vEjbaL5Xazm93gMOvHxLJLzpcm4+wl0XvqAkWj5BcMY9euGOvP2otrwKkOE9KvqyPXzopDrovTe9rJkRPZWNAD4yiKE99+GmPdfZEDySnhg92YeBu3OopzzZ6aG+nb4Wviot/z2hi52+9l+VPkJMar2U50s+vQBhvguqHL6iagY+Cz7NvWNHbr5P7lM9a4KfvuKR8bsn72a+xeWbvjkaAz5aGB6/LIrmPAOOI7468DO+84cYP+70sD1cn6a8eD00PeCLF74yJJw+fn0/PfMwG7+0QPo9EwDsPb7hIb2o6GS+HVzFPsrFc75VXva9foQhPj5SDrowIfe8m+4jvT6LsT0EEzG8GBKKPT0DXL5keLm+5SnyvTcTGz0+ArI8j34Yv938qL36acm+qEtYvbckWz7nhzi+FhAlPlO3Rb4X5hW8jfztvo+5Nb5fFaQ9BFSTPUObOD6uDOS5E0aCPi96Oz3bqMi8","I96mPhQ1i71gmXq9V475vfr7+z0HIX49+hx3vlteNT2dEk2+Qixdvdjb3z0tGoQ9jt8SvkeU4L6qQ408CEnGvkun9D1t8C6+bmBovZ0Y5rw6P/a8u3cUPZwCFL2IzIS+ACPJPls+lb4hSwg+CfNsvUcWQj5VH+E9TP6evs9N0DumZNu6YRmOPjfv777B2wc+ZGSGvXbBHj4yh4++LqQgPq8G2T0ksfm9ltK0vVzcSD5mk/U8FS8XPg7j/r4Pr1E+0QqrvjEFab0hU1U+0NP2PVzkfT2zq0q9x5KtO2edNT/yblQ8c6kUP768y71fdr+9RISDPO48nT1ySQu8CooYvtfnD74dP3k+MkvAvNj3gL4MKCm+LxOUPsOxWD+ZumE+uQBSPuQlur2xCrA+8xvYPR3tHb4IxBg+Fmlvv1FAjL2+wrE9O0buPSbPjb5/RHC/36zNPSYTLj8cQbm81SqJvVVYFj8J4w8/uvM2PbOUZry4s909lUMav1Hmor1oB4C+uHTLPJ0nmT5jbNC+9kLPPo2Rqj5fnCw+MQK7PpkGBb48U6Q+diOwvZnbpD45BbW94X5Sv9Tmcr6ia589iGQfP8i8D74Ptz+8/G34vl0AAT47RJm+q1WEvsf1+b10nYm+SKsTvkzig76+O3I+ULEfv20xPrtoOKA9lzYLv4g8Qj7mqnw924AZPusjMj2zyI89zUMTPoBg8b2jI3Q+ZakCP60tzT1D1Zo+hhlqPZXoMj1pcoo+vniLvZ+JxD3AXkq+npXqPXDVl70XKLO+2lidvVt3a7qmS2S+Vgy5PejJGjwRDMy9vl0IPhxjmL71AjA+ZzcePvZp5b08jQm9qdlfPUwYurxFyTw+yEKpPu0EhT4z1Vg+bRn1PUdcar+3Ciw7+gcCPkzBgD0Cg4M9r/VJve1CBzx6P/09OwL6vdqqqT6Mzw8+A51KPtFjFL5kSwY+bqUlPvS3GT32qxi/mikvvT1pkz2wbwS9Ws5xvbRBaz7fd5y+kQzyvvl4jT6W+qS9","zC6ePYIlTb7r57m+whxkvieYd71jeQC+/NVLPSidaz6rTzy+0tFevnFVT75YDUY9gIOjPPdKHj497bo9cFYAPqTidT296HW9owH3PefUQ76aWo69aduFPaHGOz6jueU9ZAegvStD2Lx6FQG90YgIPbbOtD6inJQ+1PI3PrCCmL5Nvh4+WHCfPoSw3773Y1K9zoOtvHnBmLxIEiw8sWKXvnLwkb64xK498IPlPi//tD06W5W8l4qfvhmCuT4ljy2+TiHAPY+lRz6ZUWA9z05QP5d+Zb6CZIg+X5hPPlo4pz6zqio+/psDvxsAmT72c7K9AbRQPieCID4/6Cs+wQ26vr5ggTwH9Ey+Q/ntvfmFeT5t8dM9SJh4vOu/rb2bCtE+EtyFvaI4AT12GbQ8e5gQPoaMTr4z2aY+X+hmPV96J79T+Ve+mXo6vXhmGb6zywQ/ArDLvZfnCbs9dM2840QfPlOJMb7DR50+UD4TPjlI9D3pCTI8UjdJPq02LD4RNiu+47WLPuv7fj6bzha+vHC3Pr2oxL3kj6y9RsZ8vkFS9jzZUNe9misyvgA/jb1DobM9C+ypu1BoFj7TiUq+hogNvkocpL0Xl/g7KofQPEwSrL6ht5K+OBw+OrpUmD3s8Cs9PD1KvnW/vr5mN7a8bkBPvQTfuT045A++c/sFvWMthj5uaam+Q2QYv8aLYj8gxUQ8GViKPrHuo74rVnM/BSwiP5xkbz4rRx2+bd2HPmXZ4L4n3Ek/Sjkmv5rbZr8nxwo+aAYQv+1V5b7ZcKW+SlwuvwhJZb2OsFu/FlGuvq1VND4nUAW/A3cPPkqsrb7Tkg2+bP4DPzMVsL/uHk2+UybQvtdX2L6a02G/uyZVPseNXb3IJB++/1/jPkYvgj44F8i+Uw3kPltbJL7+jfY9wiDFvsWTer41ECS/pJ+YvgzYE75j3B0/Qz5zPvfBqT6Nlo8+dV5kvgPuYD6B7tm9cu/Gvki08z6YaE4+RNghP40ZfD6/xdG92QkGv32GKT49GAy/","CR/BvDx9Fz4mG0G92pcNPq3PajyHaQi9BgcMvS463L4y0fQ8gy9rPpyVcD7f+Co9BEq9u+q/VT4K1bc8HygdPyqQX77rlcG9AFKQPgMVB74JnHs9pxozPsY1Wr4Cqcs96YVUPfDqmzuZoL++QssFvgtiID0PoQ0+rmAvvicr4j29rQC+VVbYvIXxej4jDxk76N0yPdZeDz0adXk+MmyTvNHMyb3VfGq8sODvvenVDj6tbyU+YBfxO56aZj2R/Iu9Av0MPhC1QLvIfYi+Em0KPSfVFj75rNS9h840vaSajT2hYne+srilPQ3BZb1QDhc97NkCPoJWsT2xVDA8QUuuPNOHNLyATrw88VuuPdVJhL1KCzS9bAujPoUTgT44Thu+Hi3lPk3tzzzIlJe8FFAsvkrCCL5I8xO+JKdEvYJNi7368Ea+JMjSvU1A1b0CCPA6SgMDvoFbnz1bjUs+l/wOuwcmyjw24h8+WwxvvAW31j2T9g2+EnhJPvzxer6htEM8NQQYvllVvD5AQVS9X5UxPi17ur0fNDU+44dTOoZ3iT014CY+jSQIPxRAAD60OCA+LTe2PbYdWT4rU1g+EZb9PEkzqj305bW8LZS8PUv0wL2bw6s9N3hdPqdVery8XSe9K8VrPnnYfT1Oask8y9nKu8BtsT7hZIi9C+gGvtycqrsdtcq7f1nCPPulLr7Pzn88aAdbPVpp3jwlL3S+thkgvr7aMz75cwG9ajnhPb7Uzz0K5Pg9S/HmvFWBZj1fOSY9FfqbPf3E3z1nJ+68yCMuPujq/z1r+Gk9fI2gPXUo2L1TOLi9pvQnvjnmFz5UElK9hJI2PljTUb2Mg9Q9ziOyPUwqPjuucBy+awVdvAmHj706TEY9LNIAPlAr/71AQhE92zeKO6Ah2j1qgl8+m+SEvoe7ozys8qs8aOGZOnCktb7j4A88MUQqPgP1UD1mJVO9S3hbPf3+iD5EViq+8iUJPmfqgjxbdyw9EoOmvJAJWT2loIe+eUb3PJPtUzy6ICY9","qwzPPc4tYD8ZlYy+J0iNPtaDZD3mnAY+lk4dPpLenT4Tlso7gtqXPh391T2ppeW97Mr3PbQUUj2BT709tJh8vI+3kr0AoDw/GkEevm7Mrb6dISQ+8dIZP211Jz0Z2949BiIbP5iCTj01plA9pa1WvfhPML6EroY/wKr1PLzyQj+T7vg+swbnPV40+z5sn54+6Do+PrbFMD2TaTo+M+WmPpkQyb1byg89oLROPTCIHj1GTHg+ozm+ve1KK76H8RU+Sf/pPug88b11nCi+tUH2PuYQ3r0hQJ69Q+5yPlXTND7GTXa9gykXvZWsnL4EmZO9uTBWPvqmT74gnxi+UDKbPVEyEL2nIE4+7WvHvYrrT714BEw+ZJrBPXuI3L3WVWs+xWKxPUaLQr7gksG9+t19viHJBT3bO1U9OCJ9vsDYiD7sj1q+nEO4vKTyjLxFUoo+c6PQPdoLaL3WrSg+KEGNPS73bj7Sm/w9YuHSvuoopj4QJHM+xfU3PmNbKz4v/l+9XLDvPU/4wD49k2c+8PxGPvptab4eESc+RSHMvWpFsrsrDy49reaZPXRyoj2UNoO+rG4+vUr4tb203Y0+uuV3PXwZ8z2A9fi9VS4WPlzEMT1hkko9+2V8ui4jnj5riBO+IUqYvhSh4j3/Tya+w37zPY0VGbx2B9e+oqnavO7qpz6tEyi+lFDqvE1Uy76eCGE+BqosPEzHwbn6uEW713P4PN7ODr5fY3Q+IrYnPhSjqz7Q4sm9p0kfvoWATz4wogy/sFbkvbUdqL2vMUa9FqUrPl8NRr6FPwe++IfBPFt1Tb6Svpo+yby6PeiB+74YalI+zFHYPRtejDwGU6m+7E8YPnP7CL0Lz0c+88MKvIPTxzzpByo+v7cmvfkDnT20aR6+UcumPUvXVj7t84E+mHzUPJ7gjD0KNcQ9eS+QPj5RAz7u56I92LsUPbIjcL6i/IO9JpSOvQDWhT4LA/w84W19PUNn7rv42ao9SDKiPfTo27xP2909JjcVPtUWyTz29Hm9","2TaQvdlYcD5k3sW+mI/GvoKTVr26kgm+1YfCvr8ALr7w/hS+leCHvfaNdr0dEBE+MGrePfIZBr5r8Uc+jxdCve/90DyvdDy+dSBWvZeA7T5MXZi9Cx1Xvrz3DD6oqjW+4tpGPpnioz1v5vw9Bj2cPWzL1j1wGam9U38ePpl5DT5SpIg+hB7kPZqpADw2Gua8eii3vLBD4j1YWEE98oeMvqwRbr6uy1K8v1BUva41kj2cekA9I8UiPrLJn75n+Y++PQXFPvDSMr7z65c9kBBHPVwSFj6oFga9YMUevsjYJz5qXg2+GjAJPgg3IL41JpE9/GWFvfc6LT11e5S8wZNzPYkapTy04h+/cNWPPuFdqr1G8Jm9c6PlPoaePz6cAzU+j/gaPb3oOz51V6y+gp4RvnPMkD6DnYq95FgHvxlQJb7NXi69L5ncPuxerD9SQhc+RLOwPuU9Gz5bC3U+V0kcvKDlvL7H4aO9BsItPuX42z3sB289a5NyvuD3xT5uNP0+CDN2PZy0drpxgDk/qfDvPnkoLb4bCZu9O+MRPt59pT7Ds3m9Mg+5vBaZGD0Wb3M+veUoPvJ+tb4jYYu+tPd4vu8Jwj3m27i+XJU2Pt6ltz4ckba+aU+/PoslSbyYNQm+A6nDvufQDr6cLxi+wW3tPcgfhr4gaG69oXRcvpjiB7/CWv49ecQJvtdPZb5/b6q90t18vR+bgb0R7T2+HWd9PuSRk74RAaa9rXqbvV2xtL3olUy+EjQDPqffzbyOAt+9D/k9vlZvXr6LKLm9nmyFvkIrSDpfcn88b+gYvpSCBr6RI2c9DnqWvtZZjb5Se9y+1cDUvR8Rir0oqKm9tAdEvXPgqz7c9A097pHEvsNi/T0lqYO8Ny7AvhFenr20upy9JMmTvsWSI74VIxS+RdQKO3WXRD2dvh++23/gvtJqmL2tm3C+z5MZvlrTcb6jR40+yMCSvSgVDr6hAoe6YuKhvhkwAb4XbX6+d5v+vt1dhr3Yc+C9mDBKPpQQobygpaS9","6h0GPh4H4j2qYFY8L9gkPQevCT4qrog8kAVtPnagizzNQbU9s39LPkxeVT5Ey3I+YLglvU/iRDwuVd2916ucPe+Nvb7M0om8dZmhPGoGsTu554c+6PKzvhVLP72c4wG+OAvPvNmQuD4xH8A+a+ssPnIuvjw8pem9TctpvjeBazy8NBa+B7WWvTfleD1hEgA9iDOePiCtzD4dXJS9zK0APlCtJb0aNRk9dKDgPevmyD0hzM69g5rUvKkWJT++r3m+8AE1vi1+KT5f9Qa9460+vnlbAb4hG2u9+h69vXGjFz6Z3gc+3Mu5vm5h9T14t568tDMEvojhPr4ZjJq9cJL9vEouMr60RxI+FwpGvW7qOL0O6Fw9MmMDvsIWMz46OHo8oSoHPjUXFj5Z9KK+Zbb/Pf74sz2ApQK+KkiLvIwDCj18vRu90rmZvL7c5T5e11A9sdJjPe0xqz0P+GK9MtCnPmKKsb31AJQ7504ku/oBTT7QPxu8WytaPSQN3j7RpLW94ruKPIi8Vb04tie+tVzuve+noz10qPG979b4vePFHL3JZBW9fpvQPa3/sLwADyG9ftssPjDZkb2hqLg9aDWQvpwdyD3Vttu4YCfivYjUkr2XypA9fw6UvSywmbx/vU+91HyXPKVbcj3ib3a81T0GvCiqGD1tqaq9ki+aPdbKIr0TOAc+LqEqv1FPsj4rwd89ydQuva9PFT56RvO+KfaRPqO6CL5T0go+PTQAvwNmjr5Obpa+Q4BtPt/Yxz6BHlG+GCGtPL6i0j5thmM+gFo/Psw1BD9n2sU+KIKAvSfUPr72/T69qdmovq8eDr7ieqq+MW9PvYHN/D5dFTy+TLiOPiaU8D48c6o+k3+hPMaT7b3r9YG+fd10PdmDgT7YpqU9DHIOvrSmBz6vufO+DbzlvX5Y0D4ob3S9hDtmPijCM78/HVc+qsh2vTf/fjsyp7w+TvYPvpsDJT7xpTi9ve3NPZHk6j7+LVG+SvCNvvrTjLwGc5I+oE5mvoHz973eRUO+","7i1EPou79L25EVw+XBXTPVSAyz3ab2o9/SSPPS9dIT43GOG9tr/WvWp6Dz0Yqy+9EgcIPvZgp71+37w7D0Q5vuT4iT4p3Ao+icDvvZgqxD3pArU+lC7lPbYPMrvaDcU9qBJoPmn7krwi5NI+Z6lLvYYEIT6s2CQ+nfTTPVgVwjw9UIE+m1dxPgiDwb4+TIW7TL5Ove71ML2SMQK7HCFFvQXx6j2deRg+kZmOPUQXZD2PrV0+weYtPgbMtz2jmJU+5t4/PdRDwTs2zwk+/8aSPv3ldD5dhUA+cKyoPjp/B74Rnu89yoKZOrIt5r340q29ljRyvp38CL5qcQO9c6jivax3tj0hyJw9iv2OvkQ6yT5Q52e9Pc2iPZ8Xsj7jGWg9tQmSPeFBrDzz7lc+CqlHPoLlzz1mOXM87HX3PREuzD1qqOi9p9kLvN/jWD2bdTy+IjXkPNumeL6mwgu9C9fqPSw8Qz34y6O8+WanvX0rMr7b9kY+n1tYvu4crj0MDQw+KBXePKaepr0BGFQ+otlDveLRgr741iu+TK7QPQ/+NL02BVM+uck3Psd9Wr4OvR6+e/KfPaRt0TziC/Y9Koctv3uVr72/BH++YA6pPFj0sD5qyLQ+R5zovtBNEb8Z3Go/F89pPGuISL6pRAM+ygFsPU6XjL0n6FQ+Xe4ePaHeWD570jU9WA8uPt/Wxr3dEIq9CmitvWf4xL1lD+E7hl3yPsLEBb0mUwY9sRwMPJOHVDwKMw0+SHxLP41Igb1YCYS+K2Zzvgj2tz2+xoc/elWIvtp3SbyKD588RDl+Pf23173nQhs+JSf8PSC/SDw4uOY9O+B5vnVZm7241tk9xMXMPbMMQj2nG+k9Dxe+OidFRr2WkMm+j64cv9IdljyCPj89m4wRPisUfT3tKTu+CW/pPIaSED6H3As+42X4vVNt/z3td/Q9gvPtvnbXKL6RLkC+EpYAPurNfr5StK4+4eYsPryrlj0Rr8A9pAvyPVEOIL1ReUc+rPaPvYSThr0Fl0i+","8y9fvMqkXL/4d+29fnkKP3hvz73zQS6+npdqvsDAmr5L3KI+SAlrPfHhur4OlbE9JSOcvLZd2D1lGqQ+tGPyPSAVUL1HZNs+178CPr2JMr5Hns2+hqQTv6ykmLztPDK+GlcdPZVWTj6/Fiq+G/nyPOaVhL54MDG9Z0aQPguHWjyoS48+EFtXPsDLHjyC+qW7QtQ1voXBfL0NJIo+dsNwvvIs2T1mU3M+nWmNvrUS9j16fO4+UJ61vQRMjb4AKAu+HD1GvhPQdj7vqTm+2hoiPktSvT5brMS+u5EpvK0p3T3Gs7k+Lo1WPke4ZT4RKmG+lyNcvq+UBj5PR8G+mL1QPnWNhT6fn7g9JvcWO6/327xz0Fg+8NQAPssXrryXJJK++7/vPXOFGD6VrJ289ycrvUguST7sgho+eXmYvuElnT6HfKq+Z04SPZWyFb3NnFG+224YvTyJFL4jIIK89GZoPTKXoT2bDpM+TjrivjjZvj2eYjo+Cg0LvlTQAD7Pl4u9XcqgvjBhK7633vC9ld7cPb37JT5UVyo9/31XvDGnJr7zZi0+ScI+PirBhD2Ge4O9rkEoPeN3+D14Zbe9sgKIPXwkHr32CLK9SDgXvnUFIz7bI5O8k+lYu+a0Fr0/7nE+Z49SPWieI77FK2Y+ldZlvZh3HDwqGsw9E23WPQMOxD6tGXW9NBnzvT3For6ADBa+KJiYvd/0/Luwj5S+/YGOPojK5r3fnqC+kF2cvg1zT76/wVG+J4UIvrzIRj7QMEi93xtqPupYiDxBmJs+X20yvZuV+b0jVIQ+pu0ZPvsyqb2Ui2486SxZvROjYb2xH1y+FSWgvWHDxDy/xmu+fqUUv2tlfT3GbZw+vsXoPXYpp7x9QwC+/VmdPizIAD5JL5u+4OYfvhDQU73eFoA+vDSDPqywzD0jjuS9l/KsPj/0jj25czY+YewlPiyFlr6lWRM+UH6XvsD4Tj4uXB8+LWt3PbP6qzzu7FK+6zn2vWElHb2FnJs+D8dyPccJoL0bDqQ9","DBUQPnU0Sb4vnQ2+8mvXPYOlrDywgNQ9yvgrvhU/Cr5Nuce9xvDGPWBOLD68dXO+cFVlvuL/hL7Rhxo+pZ4RvduMcL5lBa49GHoMvrmLIj70c0e+thBEvk7I0D1MZCC+PrgDPsnvKD3v+34+ptEUPSiseT7vZHg+yTmlPkUlcD0yRc89PAUPvifWp76ajVm9vvlJPtIze7wgFpO+auDOvYlNZ74RbMi9iM2RPau9Az01Qoa9bocFvue9Yj5BZoq+EISIPsPx1T2pp6a7au/iPYq3270pfB8+X8VZvnB9mzw24Gy+uM9AvoCeAr5KlhM++A9IvUyuUL7QSCW+k2+qPB4B1byV85A+WDioPp7CB77GP1Q+QLVivvzEl73NkLo+MAs4PiU/Qz68RTG+9QClvuP0qT52DGU+0mbQvhO/AL55EX2+GJdSPOC/Bb8xo7Q91dNkPipCCL+7ujg+EauxOyk76z0D6Lm7Q3FYPcitTT6BrO6997mKvk4kTj6VMfY+nAqivkHNbL4w6cI+IW2KO5FA5L7vpPi7vDjoPsqhMj7wM6M+yLR0Pnl23TxnuaQ7ReyLPndrTT6JO6y+vWXSPaVU0j3SuSW+3kH6PkukzT4Wl7I9z2NdPN1vrT2qhL0+VkeSPr8unz0fjgW+PHF+PoiGgb0q9Eu/CQU0PR6GO7u2X5e9TFayuy2UkDrKg2g8CHBzPhD9/70TBFI9M6jUPmBv+z1/sCw+wtJQvOxtu73uBzA+CW/PPWl6g70Q36s9QC+nvZQNhz2p9t2+DxK+PSEkhL1+dI2+ifdUPTSl/D2jA689jisDPj3lC74aU9i8FQpHPs2pOj4J9NY9jT8KPm8GSDy+JxM+LSkVvuakVz5+o/o8/SQ4vv8zYr0YGxy+6XhoPqrytD3eZ0w9OIJHvIPhHb0eGau8a7UtvcehJz4RzgQ+Y8H1PdV1aD6rzZc9YdVjvQp+Qz5rfZI96pMEu+SfDT5//NE8xsxjPsEgab2Jemo7dYYTPje8yz3qMCe+","mSEMvqMHUz3jCLC+FSIEvgMyP706VP+9mjzNvYpHXT6Sd8E8zolXvlA40j3r3XW+yiRdvlJufT2y+5o+ilacPFZzjL5EKQW+W55vO8zpgj08X8G9Z132vnF+Kj7ziQy+IvaFPjoHqTw7W0K+G+4UPeW/Tr2zzW6+Uj+lvQcgu7woeLC9XfMSva+5hz5rH0O9seHrPXi2rD4KNR6+05lPvCBj4L0tGLu91pW6Pg/tSL1kasa60/nIPf+vPT4YdBs+ndZnPvJsqDxN8J08JtvHPEPAsDv2atc9uBBpPV9ktT7LW9U9lyTNvgqMZL7AjoW81ktwPp68pj1xec2965jOPc5FRD43zTm+teEOvd4guD0HoUA5bYFWPiN5lLtbhE++4HlcvTFpWTxeEwa+bYgAvln/Wb5eyaO+vpE/ve4UGb7jOgu+7yXQvO7KG77J9vE9KY6jvUJfbr6cF/o90nDYvWnwKD5mboi9L3Q+vn93RbuzZwA+LIZMPmdO4T2L3R+7pEuMPkLEOz7WByw+lbn2POacQ7yGmng9YZLPvb3mAr7ILt69bDayvWIX/z1AZoS9l8btutwqQj71GR29VWxivU9Bgz2cWXo80TMcPsx0kD0Kukg9zlWPuoubOjwBiDY+lt0CvgL6Vb3c9dK9YWnivN/+Zb3E8zg9QPMcvmycyD1jxac9agEeP0obqT6rQTk8mNpsvb839jwLlg0/CWhcPvvr7j6YhgA+mMYlPhDsn748o0O7+d5ZvVJm/L3j5L89Fj38vU3NeT6FsCg+RwgTvwe/UD7uKRK+X5wwvmXqNDvnTMW+e3CXPl7ZI74lpak5RrU3PlyBDrw77Cc+eeawPjDl6LyQ3SM+xmBBPumGcj6J0x8+mekvPfaWYb7bC2O+nZFfPg5IWr5w6PA9fqvdvapPmT34C6K+Jkv/vlRBJj8I8k09POuqPbv0GT5RQRk+yAYPvUcLxr1Fjai+RuyKPol5hzxmrbi9o+Ecvj0CR78ZEee9I4m9vqCoo76/2T4+","TDy9PVyqBT0QRFe+skW3vIJy0z2/qnU9UO/GPaWbZ71pNMY7J8HFPRZnfT2aI5U9AtECPZRebj2CyUE+b1wTv/HoYb0E7gU+hk80v5ST1b1QF1C+xtEAvp4FDDzU66A8UXwLPS1ngz5MQu69pVeKvjmtU7xbGGg+h4BYvTIalzxAxWC94ZkZPl7QkL3s+4g9Xu6wPZ5Smbu+tVy9PfPdu+yKbz7cNzk8J2dvvAJDljx94fe9qRbLPUvuI76WDi88f9rOvSOz3r35I829bGFOPXnntT1fEFy9SzojPkzgmT0Rspw+B94fvDw9TD6FYXy8sWCeuxbxZj5pQSY+/H+jvVYrnT3JMJy+3+/RPnDpcb4oEiU+HdU1vv41Ob4OfDm+IVCovo7Fn72MQ5S+fOy4vrC8Fb6KXPe+o1DoPg6ZAL7ZFEe+sNhwPlhghzwbj0C+lmAgvjLKmr6W+6I7pwVMvrO3LD3n2MC+P/GpPlxYDT5xrZ89a4dvvk26jjyQWMq+RwqjPtiD0Dw3K7U9rllLOli9PD2RizU9vv3fvS6pjb7aom2+2fHSvVHoV70YgqA+04y6PUW5Br6WPbs8jG8VvsvbTz46lpg9uTDhPec/Gz5hP5++Fcylvf/bwzxzFLW9ORiUPDGgcz1KMUe+uO4YvhqCLj4Sfsy9buDtvTB06r1obBE+PVhHviGBcj44TIc+qbX9PpkgCz6hWKW9Yf7EPAH/+zy8pxG68YIAPsSswL3eJOg76Ms3Of7V8TyOmkq+OteIvstIJDwHcuK9tDIYPTORjr1BDY2+tzs/PTMTUb14n5i8/mcdPnp61L6Z7bC+5xh6PXhoRD4sZiy+RHg3vV1oqr5h0Vs+cLmXvUFZKryUVrk9TTFhvsFVvr1UsOk8tWAjvNi76rzIDAe+JfWxPAm7pr3klEg+5XWCPo5BOD4+2o69foVQPRNuXz2WDVs9WSWTvYgQq71mKM281HmzvCrUzDwczkG+Ks9pPrillL20zA+8k/8RveiyoL31BJ08","mpofPmA+DL/3IpQ+jMzlPnWsGD44e8A+dnVUPrxjVz/NARM++lUQP3KNDz9X/og+gwoevhacrj48Wx8+VDpavvjeGD6VjxW+27tZvsyEyj0Q39e+TVfkPu+mTL1p9TS+NLSjPh3NuT2nEei9CMpbvPBZuD42Kd6+IOchvhfBRb4R2P++6nKxPYvBkr5X3Wy+pmT2PAAnoT0BlbW995QgPnq5Vj6dCpK9pJgBPWBNRDxfJLC+KLD/PrqKXT6AOy4/X9gBv4vMUr4Bu8y9mRWwPhIGRD7Wd6G+ETtGPUAdqb2pXV8+DhOhPfsBeb1acCm+Z+PKve1daz3qKzG+Kb2APjk4CbxI3KA9pMWTvsysSrxBZiQ9G3eJvFFiWj1rxZK+WRM+O8n4fT4NCQG81BMFPfF/+b3dxCc+0Au2vSiQXTrlBwe+JpuDPfyaAL4b38+9T336vWGc8L0E9mA8GOwivpSxmTyZ+s8+q/j7vMtxgLx+XxS+i2VjvsWc972VQmU+Ar9evjRjF77DUgQ97w5MPndznT0LA8i9jEoEPjVtDT3IT7s9wcIkPeBlm76GoH++ae1nvoKHu73pXVe+GnY/vpKYkL63boC9HRKSvXcy570IFZ69919yvj9n2r32oLg9ezoYPt0oHz2qX5o+pwqxvaq+Er3O1/k9AiChPW/COb5ykfS9XJEevrr7yj1nyum91a9TPQCpcb00toS+meolvLwRKL6uH4W+uLmovl4pxr32VNS69267vjju3z22fEa9z5nhvAYXIj46ZOs+xQiMvhM7kL1gUOa+F+GCu2bUgL60CrS9qz5Jvd86iL2z7hW+QCm2vp1Qazt0gIO9S9RNvk8tED639JE9KC2APNlDSjxDEt89N1yUPb/vpL3MD3G+2dSGvi9KCL7ddHQ+u2nJPo+5TD0UAoW9cOFTPYKaqL0z7ec9tm7BPgTIOz5bFdI8IChPvogC4D725xq8Qkn9vR296L1lmVw+PsKlvQE5zjzZ/iQ+0FQGvpz0A77VY+i8","2SK0PVSC7Twcnog+sdWzPWhZhT4cU+A9I+QxPhzhH772a/Q7RakbvSvTLj7MURC+Df9cv9a+Jz+/4qI9HRvWO/U7a73pmxq8WhtZu/pTNT5u77+8ReAAvymMIj4FpvS98mZ4vmKdaTugayq++R9BvnK+vz2I50w9n9UpvnkytL3Sfvu8d1xlvpu/Hb6PPAY+0O47PgLCCD0r9sG8GJRsven8Kr1GONe97sK5PQvIJL1Grbm9GFtbPehQCj6ecmm99U98vokRNj5U5wQ+vgaOPYbfPb6YgDQ+OxCVu576lT2a6X08t/6YvRAHLjxz7s69uoOOvQtGzz17o/29/EoDPnpMXD3vZqU+liWEvjGP77yGbxc+I6DsvooykD3mWFw+LtnEvY2voT5AMh8+Ur8XPbKXCT4vkWE+xaE7viHXpL51lbI96Qk7v1qt3b5CLks+NmUkP4O6bL9IMTA+/NCTvQi2ar7hrgc+xW3gPP4LKzs8RrC9NsOMPhoDIj2dyli+XPafvvY+UL1W2TW+XfnBPTZfC775kZK9569OPH+pLT4pQ4o9h+eMPiuseD5psg8+b3iJvf3n2z4AfRG+Ytl/vjkwYj8LUgy/+oYWvuyiHD6NDeA+Ft6Pvlhvfzx4CbQ+zbyjPXz4gb3wLuq+0y/iPqxNGj1fzAo+vnopvQlE3T7UT5K+1gbJPSW1V75YzoQ8B9+WvgWIGL56geE9jSdmvttQTL6yvgw94WApPvyf4zxaGEa+XE2FvjqNkj31Xie8fYbdPUEugbscXLw9e1qDvs4pc72OuJu97rZOvceYir1lYdo8DXh1PY1YlD2323w76l5SvfT9Bz4x5d69PLFOPg+ZRb4CjRc+6mKiPs7ZTzzypiQ+mjLyveKFbj7ZK4y9JzOwvZlTub2lx+u88aUuvaw3sbyUa7K96RotvgyYyb5fNx8+KOi/PbBNAj0WdVC++gwxPfrORr6DnCC/4kkwPrVv8r3ark89TPqpO4RY8rufFZu8QltLvsdlkb6a0is9","DUYzuli6JT6buA0+uNWEvsJOhT5dS2i9dACfvtqETr53JC29GZwPvl/v+DxoW1K+krWvPY1OAj7t/zi+eAmUOdTKoT5KFW8+Vo+5vWthW74N9M28f3ISvlCjeD4K/Mm86SOTPCvsSL67miO92yyWPbZJ1b3YSLS8tJssPiiuUb6htUm+NkK5O9slub7SiZ09ymnIvWRflL4HnI68FXKZPVXvXz5eav672nIvPnmwWTqthI6+w3QpvW9d6L4/Nr69EGLmvl084b39Wzs9KHCWvjCVcz7o1/w9hKgxPpbNhr7GxeM99i6GPcOSKL6Vbhc+1P/1vZLrNb5nw689mG44vv8V7r3gGQO9bKo+PSSPrDwKrTi+RPU4vXKy6j1myKQ9pSQNu6fNK74b09y64Yb7PRLKo71mb2U+yBnmvRHDfD7ZHMw+vILWPWD2u749auO9BVcLPo5PF75vtgi/uU5KvtvBwL1yiBu+2QSevaYWFT0m3ti9QpkYPf70bD7DJV6+OIPEvMxMGb5MRTC9Vs9Bu3HB7j1Iaq0+rR0NPxqfJDyYbwQ8ojYLPgMnuj2z5Ww9XUJBvfoCuT6aK9C7sNV4vboXKL1pa1u9LN2evUVgjL5NKvw9jOJ/PQ5e6T2bC3a9ifGFvqsF27zs3Sc+FGnaPfCD4zyCBjS+dLYAPhWBHbwdyB6+KPyNPvgtib57/0w9qMWMPniLGD4be/u9XSymvuzjBL89GoK9E67zPfhzHTwpYGs8VBfkvVJ7Mj72upi9pqzjvZoL+L5pT/m+3m8Yv9UE/b4OnVq/NgmEOyucbb4zsY0+BMUfP9tqUD3KpJE9yqRyvv2HPb1C70S+f9xMvG2ER71Di6C+SUwmPjHIwz0mqCo+Wr8PPdALHD2Pqp28vh4kvpmqj7zpLIG+hrD2vLW72jx0co++rdYuPi38l74W5lw+wKYfvfOxcD7+eGA9vmmfvSiYHb4+FJo+/cYxvfRzxb4L/6W+PmunPhUiHryI+Vu7U7H/O/aqK77Up5s+","DRk4PtMRrD2Z5Io9pOYCvgJFAD4Q9CK8WW5avgqbwT4DWEA+B58UvvVnM77G4Lm9sumVPmjKpr0uIXy8gTTRvBeQkb4PFwA+IJWXvUgSXT7PU70+uPRIvqo+gz6XnRQ+RSKDPWg19jxDc7K82HgcPXixcD6w+tK+pi4pucYunrywjEg+wunBvaNeMr44FYk9ogB5vX6Mgj23Oa+92fGevQ2ezj0T920+tQy/PP2sEr7sA9G9akV+PVz3ST1Wmw09GqVCvg6beT3+yYO7s6OmPaxghb4uCHY+kYuyvOZtkz0amjk+WFrFvfIWCr6SA6+91d4pO7viNT2BfQ8+M4UxPmRyOb7fWbu9VOWsvcE1zr1Ufqu9fYxQvvYqeD5FBaQ+ehWmvS4x3b716pI+5m8ivpMXxj30nOQ8ez0TvESF6r0gvQk+en9XvqK8+jtwUAS+P2TXPYpf1T6iO9o9sGwDPdkQKr5pB5m8pDx6vZwnPbxhz8G85CYpPosuBz6erGw9stBaPmMqDL0z0rO8O9eKvXeEJz4+BxI5EscHPgwzq70FawS9p7mPvr2jVb6jRc+9TLUNPuRPS7wLi1Q+wSbtvMpV+z1bSpM9YAlGvjiHMT4Ag1m9VxoxvebFzjwCh869bMClPdL62j1zFPI+5gk8vg9FmDw/jUM+7kANPs5u7b1Lnyg8L3kEvRfoc72Jl2s+ckSpvS00RD1Du0K+SnL3vaPhYL45Mok9l2sKPenHF75awzC+TfyGvcUcHb541qW79mTXu9IuQLxDozQ8iqspvTr75z2f1Xm+93fQPfkasL0epgK+5aRMPhYnkr3XmtE8lNKzPGPlLD1JSyG+gypQPQ3Mvb3Fwvw9y9pXvkFlMb2oXJm9b6eBvQswZ76DiDm+sJcuvOWCL74Thru+zU9PPRBoIj7gzRk+btYZvYkXCD6q9eq9KVRTvZuykD6d7+E90+MIvfs6Bb7RGb49P9ShvGKYubopiry+i5cMPrqw/L2u8QE+JY+TvQp3Q7zVK8U9","kA9LPfZPEr62ScG9nTi9vvz8pT01y3O+VCxRvqgPtb7w59E9E/gDvv1Xj72qyXM+SpCHPgtHgL7G5Qg9qADKPZ8qX7w+nNI+eFJAP83jAT+Al/89D9uMvaTKJj9qna0+hiKMveaAEr5Uyms9wAdRPnJIj708DLI91tphPVmwmb5Lg40+PQoHvHCzjL5bvou+oIemvnnSbj4v4j2+7j5Qvq+UQDvtjhA+mZQmPr1UMb3Xb8G+2TRHvKfyt72wv4i+bbCbvlbaXT5O9FK+hzdEviueWT6nMBI/hV1kvIkXGb7jwxq+LBjnPXKNZL4MRQI+tVDEvWTRlTpW+fY+qncOvhc4lD0ZaJe8JMYrvbUZk70x6Oo9/fF/vai5nzwe8a++1dS0u7GeDj0cf528u+0vvkid/TxnuoU+MG4TPhonHr32Kpu+qk4HPvZCd76CZoM+5M64vR+AI74k08U9+Q/PvTV6+Tu0BNw9lTvtvRzGf71OY4E91kt5PP+Q7z0N6wO+n8AVvWO6sz1Ptrk8sZu9veW9Bj5ANBC+Zy2bPYChczxsEuU81Hr0PJMIFL1sRZU+Ygdgvt/wUT2el5E+mo8APdtDsj2sGXw9yL8FPnYsMr5Il5s+hAx9O3oLcD5aoRM+qONvvcT4i70uyjE+EE8AvQ02JTz8Tta901q8uzpcnz5ix/S98N0YPtE1Kj3iarq9Gh2gvhoEnb2Malg+7QquvaKMSDsDbog7/fmrPqp/HL5HfgC/SeqHvG412z01cwe+pD2MO5tCFz4fsLO9ADSKvtg6mjnF5Jo+OL6gOaFxnL63kA0+9ZiXPck5P73js4U+7lUUv5aKirzf5Fc+giAHvTdkwD0+Oia9k8Q5PTPnCz0zjaY7+o3aO3GvMD4gwps9GpQoPVnBFT2x3cW+ep4BvjLaND2VGO89OIKtvtsmeT0el0A9cz8ovRL2mb6Iie696n4Nv6Rfxb5AUg09I35OvsJEZjyg8I0+Ve9PPRq2NT3lIAi+I2VyPWozwL6whMy9","2G0tPlzLGr5RBwM+qaPcPf8jDj3ot6g9gze1Po76kT1vfrm96sSYvnMvij3aPjA9u8TUPqTVnL6RLsu+EW+ZvX+SO73LQ2E9ormsvj/3AT55JJK9ZeO4PoZ41Dx3fF2+k0nIPqRbTbwHfCE9oQbDvTayej4sCxu+pBOuvjJXlb0Gic48cYQ8PVur2bkzc5i9xVzIPYDPDb50MXc+EpzSPPeljz2dSea9RtaNvZHhnru/9Ye9TSdTPjw/9T19HLE+q7e1PnEsG725v4w+cHaDPVI41L2kH+o8Y7zcPL8nZD6Y5Zm8DiarPpcaO77cL5a9TGPJvTPmfr463i+9PPR2vgkuqD5QnfA+Rg3cvh/BAb6S4m89ZPu8Ph/AvD77SaK9InivPodTlrzkra4+W8hsPlHdIr7INH0+jI2zvY+4Nb4SAUE+fKGNPvlEUz5FGIa+nL5VvvxGpb6q6JC9KmbvPbIkV790na4++36+PIK79TwvBS0+s10Uv25Lbj3Pp5q+JyUjPzbKCb015rm+JD94Pmu2Kz4TcjS+hwGEPu5I3LzReCY+MDo4vsWprj4iT3k+myb2vY1asbyiiaG95N+GP002Ab+wQRw8XpP7PUQOnby5TeQ9AZq2Ppxc3b3va7e+/I+mvjqyH7/7wc88ip/Vvo34+ruOm7s+2gDcPWqEbb5eXoO7NMxpPcQF97zzEh89RvKZvELJAT7/2zu9PEKBvgh2Rj57lOs9VO0zvtNpyTy1EUy8u6yHPDCqwT1ZGu4+Zsc7vhCT1DsGM68+mFw7PbOO2z0lQhA+Pt3NvQbohj2KHAu+NbSEPKzb974Ry/S9zuHcPXD4lz4dlw69QNMJPukv9705qXM+2mbGPCmCzzzEQWg+WghTPVgVHD52PfE9DxsAPVXMTD18xzC8x0IfvZcSKr705OE8z9MNPWG8Ib7PMDU+S8/FvD+PQL4fK1w95IpEPIMlk7xkDkO+mwDQvDM9B75LpdE7shocPf8UJD2a2Sw++0ZkPZ9xgb1bGEW9","DgrTvLkKKb3r8hK9x3SZvNa38zwmtly+XzKCvD2rjr6PrbS9Iy1IvTke/r0tnKG8Sd1HvTgGXr7/bCo9cmRVPLbL6r7R/6E8typrvUXJTD0DILI9uVkBvdjpSD22OJ28qalyvRU8Oz3L8y0916ThPcSbLL4QRdO9o4v1vZC/97zp0Kg9eqmYPtnYDr2GG+C9jkl9Pa6fmD4pdry9msvHPOq4Vr5vYZW9w6KCPtpmQD1liBY+vuCfPfsw0j1UjgE/mZ+DPbva7D2waxM+FqSDvVS4lb7Jzpg+a5jJPtyaJT43jzo92Li7PJMhJr6tvPa9wZTmPnlCgb2eCqQ9UQwLvkAorz0iQde9WpOnvPuj1zxg0W0+8gpIPK+Vrr7uX7K+ru9QPgq08LtRv+a9AgOYvPelcr2wfnu+lBoCvfCLmD15Qgc+roxzPbD1nr3XjRU+heDPPRt5xT0+HWQ9J8ExvQbrjz31eAG+QbwJvvnu7b3OReY9k+WTPUyThb1t5ZM9hL5AvZ6Syr6qNyC+HGgjvXyDab3AanM+Z2nIPTvFhb33tqk9CRRZPljbWT4xjW6+7dGlvWIQW70ioV0+H4cvvb+SnT2iLS0+1CF9vOu5wL0r4kS9ZWgWPo+/m72oKYe9LO5nvrfIjLzAgtu9/m0+PT/W6b52y2O+BDt/vcbhNz1ymME9/pQhviiUgD4M/Dy+SR4fvjkD6T65Sck++wzfPWldK73G4i0+/c7rPq2PmD5jjIq+0c9rOr7Y777i5jA+xRS9PSHXCr7T9dw+6LYavQ9t7j3NApW+pdF1vaSrtD0YT/++VriuPvsN/judILI6b8U1PikjzL6ngws++mCjvF97wL0oxKe9F0l2PviFJD80AyI+Oj2FvK3B7z5kGvA92D+dPtS0pb2q7Sk+h+OdvgJKjb4Sq7y9fdWTPkgksT7HpT4+ECiWvXWzDz+3ch8+FduJvlJETL5R/mk+fa2vPgCDsTqbcRM9KwLxPBUZLz7HN1s+JZiLPusKkby5esi8","E7qFPZERAz5ZLY69AbHDPvyfgj7K0SU92+MUPnHJoL1PNZw9s3aUPNyVCD5k/4o+N2bwPObSND54Nas+KrONvhFcEj6HBJo8gRxdPuzy1L0BB00+3OhZPnublj2N7i8+oINePt5KqD7yYE09FuXyvZT4kT0XtFA98SKQvIXMHbz9TRk9kYrWPJGcrb30DQo+1z3qPVYfPT3yPhC+wDCjPsn/Jr1hsik+Ocu1Pd5u+ryfMce8bOOevcYHDT52GQw+5PmjPQZhRj3Q5K+9tv1bPjGP2rxieI87X0TzPo0Kbr2wRy09winEPTgYOz3dIZo9uJ1APnoJXz45CQU+KG2OPWqWrL3ytd294C2wPmc5v72vxx8+mthiPGakWr6LCYQ6BhjjvVpyQD5OYHu8WGSSvc+b+z1DoJ6+BEg3PrmC5713FK278GDBPBGU4T0V0ma9tWfivcOIl74Czhi8yAx9vB/N1jx4REG+wCsBO1cWBD61AzU8aReuvSzvKT4nHTY9aXfXPjXS+72Mxk2+7QwEPMDjBb7XnaI9KEyrvSBB+ry3YxY97j+0uyYAm7uk1tc+2zDpvSXp870xBUA98wl+vZGYiD3H7zM9dGkhPsytKT4YbIe6pjsCvtHX+zzf7nC+1+TTPRHTcD4vPRU8SQocPqrZAb0sjPC99sgrvaaqlr19Eta85+9fvJv2cD6FvyC+CMj1PgGkqrxXO7O8hGepvQSV5bt5b9G9UmeLu47/ur1Top0+no5TPibdnj7Ni8A8pEdDPoqNJ718KJU+wAosPQ0L0r2stQq/BpEkvEBObjkgOb69NitbvQzLIL+5zOu+jT3Svb38jj1UI3S+7Be9vZMaBr+CNwg+WHotvSxSXz4O7vM9tLmOvauAYj5FfFw9OsMWvMaIqjslppE9uj/+vZD/mj0hD869c/3fPdpUFj5tHeA6r4UEPkra9r1zBFm9A5MhvBfODj6ShOK7Awv7PNMKkrz+p4q9KOg/vic3mr3a6GU8KdfnPTo7eT2LFfw9","k3cWPr7wCr83j4S9nzoCP28NMD4GTtC9cUsZvRv0Dj92ZLI9CYKcPuPH8T7g8M8+pIoWP/iZQL6tFrg+jAonPa9Trz5HBJu+45jSvo8ylT5LhdG+7CUgP2W7hD2gsi49Js11PlPo8T1FEou9VtwgPlSRzT2eDXu+f1G8vQikiL5mHwy+r0gTPl13gL7KACa/Jx6FvXC05T10ya69DtR5PtuhJj0JOmg+8NO3OwJqUT4UVqG+OoFgP279qD7NrxS/+cohvu4Zlz3s+Bg8muXHPvxoMD+pFSa8/9eoPsKURL1Ns5c+gGBXPr3mab4XVSU/oGsnPt+xPz6uI6c95TcxPz7g+70tVZM8hUsWvAC0Pr5F8gu9wttRvrUnnL0wEqQ9KKYDvpJIAL4H6iI8PHhBvYUzBr4B4qa9gu0jvh26Gb6OqKK8u4YPPTkW9b2KrAm+ojrOvS3ZQ73nmgG+7tNLvZu3hL08YgW+CbvNvcR0ML6XXU+8wWwyvYgkwL3EoLK8Wz4Gva21JT3Num+91u7vvdSOnT03s3++b7P3vgXEbb59G5W9Q75IvHXjqLwJQnG9B4KKPR67T70drAK+JcJAuy6VDrzEJb89Dp/YPAem3L0/f5K9Qlc/vn8vA79pC7+85icePFRMBL3O/he9HYP7vZKkTTx+Hk2/820svVxme75VA4s9Jbgmvl7+pL6jWY89JKfAu2tR0L1UGCi9zzSlvsHAPr2GHLI8JhhOPORwMz7qZEG++VRiPOTYlr2UqMI9klQtvXx5hL2+hvm9QFN5vLCMxj2AkAm+QYCLvWL2Tb3BRFE9w/l3PQKxUroiSBQ93RKWvd6fTz7KrL49xoyRPfpyHr+ZPKC9x1diPp1Aer6OreQ88rOgve34xbyFOii+blTGvRC2d7xq59Q9HAIFOxk1t77z8Fk9bz1Evk7c47xIk3O+/koBvtdEDL5ZE3K9HQG+PSeczT1PouU9BvI6vUk/Fr643gG+jI5APhk9Vj3asvs9RP2KPKyDTT2YdrI9","aAK7vcnRpz0U0gG+KaEGvDr6F73EP1K98ZS5PWC7CT5SSDa7busaPtzeIr4wtrY9Ex6YPnoPwr2IEww9iqG3PTr/Zb7kWVW91nEkO3DRcr3xBK69RUFGPjhZA75vKyK91QjbPaoJab1ejiM/T4Y1Pd0avDw6ZKS9+bvTvS2ufr0RDKM+SRK1PdPUgb3kg7s9VF9DvsB1GL5CPGM+kbQrvuX8+z2606e59h9nOjEDwL01Ux49x0PNvXZz+b2zolq+Mm5kPipN+bmlb2A+plvgveJ/Gj3Cq3S7cVgiPXilGr0Fbek9/lgCPsDcT73YeKK95kfjPKuP2b3lIl68P0yNPOIM2r7cjAq9izrLPZNrnLsCyqs6y0a+PpZ7Kj6HQvQ99iUJvgUyCjw/e6A+SHLpvvf8272uUcW9w6mLvcqc971efaS+wYnoPScdkz6l3k+/2nYHPWYLlz727+89nLRQvpaYob64cxo+2aYavikGTj1q5d08a2NoPkdNrr1cj6i9m0nxPQCNyL6tPtA+YpoLPh6fBz25oHA+/JHAvpoG370NcHy+dCgGvtTPpL43lCC+/aZQu8FfL7+8bKe+VhEJvnEEuL0Rshu+Xc+cPRVvMT77+vq9uIj0PZb1mL5SOXa9ziwqvvwua76T4YY+tthQvgCj1b7qDe++0h8ZvkeLr740G5O9/PHcvefdi75LcYq9EauCvYE+cb2iTvY8aAOOPp+0DT3C3w2+fGZyvpFPTL5NoY29UXhsvh3sjDxsw/W9ue0ZvhUcmr0nyoa9INcLvWhiC78Sfie+6pYhvvVcUL6ZiAu+jusHPiyZET/Oplq+uVojvuslmLwG95i+nIaVvRzWKj5PAcC9iAIOvmRq+rzPSNQ8UlmHvTWL+b2KWxK+hqA4vmaTmDscQLK9pKJvPH/qiz5XmJu8wEd2vsxsLT36da+9cgIKPrQHLj0u1oS+EG0vvfdZJ75jThI+kT/yPB+jGz53XJW+zHg5PFz1i74Y3v4985cIvjoF3rw938S+","5qYVPS3yeL0OFH0+KWBCPbhEHr7fTj2+N7c3vtGcwbzwf6K+Ws0yPJVPOT7LsSS9cJ+UPju2Dr+rqxA+Tk4JPtL/FL27xL49+j7gvZsYlj31VQM+zV6/vh2q270nawG+OUfWvVydr70+1MI9zTMDvirVFT156CE+PUA8vngcOD1omxk+/Cmgvi0FWL6WSGK9XuMVvjedgr7hbaK9nIFUvskDdz2zdDu+jcNSvsQ5Bj0qm/m8gw1ivSPETL43EhC/EClxvVLXCr0Hhhu+vRBivuwfaj6Vu5G+V4IsPh2TyL2LbU6+y59GPrNi/Lx/4AE+KD98Pef5z7zqN489NcWGveh4iz0f6Sw+xeaBPVfYNL6ja7c8gzsOPbzs1Lwuh8W+9WHLO7tGwDwH9EK+hxNOPO/ZEj4VCz8+Juk7PjB5Hz5cW8O9B1u+PWOo8L3Rav48YCUuPbSy4jxv+yu+aYlZPeMIUr6PS7I6fh0ePsgSjr0em8Y9rpUpPYMzDr6Mujk8f/QHvhKd8z1/BAO+A/1ZPjprib17JKa9HuCZPYL4Nz0LOsk9UrUSvSsnuD265NO7TEpDvkHaSz0qKPg9a0R0PEK6gr0QxxO7SXkqPalFdz43hF89ixTAvS6EMT2+PvI+UBdePAJJWT7b6yg+I16yvf5s0T5qtB+9l6+HPoDlDL7qfoe9hJ4KO2kB+D10HIW9CaV7vlTCpD4kyjY+An0kvS6KwD3/Y829MKAIPklFczyl+rY87P8ZvpEhwz0OyMy+Vu1avphnn76Owf09ogdXvZ2TRr3n4e2+RgROvWt1Ab6KVKG+j5oVPk81l74872W979HVPdbZlr1xHsu9QBsUv3mAJD0GpOC+xp6PvoDHmjpGZIm9Qs7CvX/UBr6HRBk9JbZWPY5KHr7FDbq8jRwAvIDIU76gaLQ9BL2KvAbJmb1jvkI+IbUTvQ368L2WTJm+rjDVvbWSXbxDYwk+p/CsvpmIwjuBoHm+htxFvpSq8j0Rzj69JVwLPkQy3byRCzc+","EXhoPbjuzrtphUQ+XrjTvTPc8bzpSCg9wvjMu8lErz60hcE9YREXvuy3Gj7NDPK9XDr/PQIaE72lMkW832b0PgWsJzxAopi6sTtrvCLSvT2A2D29knL5vQ1U1T2rAxc+oDIOvGx+zrlVJEg/SUCePekkS727Yi++Mg1GPQ5Oq71NByM+viS1vQ0UoT3D2yc9XDyAPrMPdT6PiDa9vMeLvZEADj7J2Q09FXldPZ5Y4L3f3Uq8lOUfvemCMz5xOXo+U/HePW7ZLj1flIc+u0+svpA6Qb6I89g9lPeYvdX9o7z8QAc9r64svliLab1xdYs9MWfkPIcNf7+LZjS9dUlFPcMyjD031QY+rkNfvUgkjj3aWTi9v8snPqAcfj6qbJ0+Amkavie1qT3vsbe8OQPlO2Zr8j2MZyS+p+6VvuAGMb5LRa4+uO1CPUwfqD3j1y6+a0bEvfXX1z5eykS+el3DPTK0fj2mo4C9wRGAvDcdib2CX8k9KxMAP0OAAj4NnFI+QhVyvTqhKL4XS0g+2HQvvXbl1r3x5cG+YF62PfoWAL4gQ4E+VrO5vaPuW70dApG+4aU2vff5nL0sSi2+HluZvgChkzzKpYK+j1FXvYzcCb79c+k9jQcDvNOHor56tr48Gb5MvjnTVz6O+JU+PUfjPVkTRb4S9cU9/JLivJ11sz11sdk81o7VPDrFXb6OHpg8JDLJvZokDD0Sw6g9bLuMPnZoZ70MRBW8HF/cPIVvLz2iTRc+G8uSPpS4iDz41LA9A/7OPZ8+6r0aPIa+3TMpvs7zC77/ID8+xViKPZvhw72UKv89YeKIPRLtqz0br749mjJWvGsoXDuKTSa8gm8yPh/kDj2F1mY++UdEPI25kLySJxS+OHJgvqWyWb0CvKQ9SSJtPhurxL0LLjC+YYkmvXS3pL1tUbQ8a7ABvuJ5FD5SfgG+gBiJvX9hvjsYk10+UGt1u4KZML4NZMc80TwYPsQh9j4x7Rk+VbPvPAciH77EroA+B3a7PP8eTjzsvXC9","UFV2vLaBpb4XMkQ/1dHHvVKqCD1YZPG+defEPXPB4D2ipY09JQiFvtP8Jr3E78g8MMn6Pi4/7b61KhE+QaMSvi2TkL29VJg8/4//Pl9n3L2vVsW9Xs+MPnnwyz2U1f49C9Ycv0y54DtzAjY+dX+CPl6vCr5tWtS+lgQDPsYwi74OFNQ9P4GRveBKiL57gBW+mJgSvrPSED4F6V++bUl6O2FMiD30OBM+J9IiPsKliD3ZKD2/Zo6MvUumpL4Akvm+3uEMv+BZcbudsTs/Ajhwvh3MWj7qKqc+yAmIPL+bm74cE008WJ0Avu9XHr3drPs9Knv6vQgsB719d2s+IdMeP0lxbD4XTJM97m1hPsROhzzjGJC9N2UkPGoubb16ra+9IZngPfnJ3L1gXV0+t9QuPODhBT6XTsS9NTKXPsYxn73yKbI+omTyPZd4iD0OaCg+43s7Pt5eAz5TsKU9NNMLPluPkT342Mq+6jDFPvdU1b39/kI+0tQPPl7cIz6Bxl28jn6rPZMMSz2XJCi+0+gLvLV5IL6tZQo+gdqhPfDdgD1hghY9TRR0PaUW5j0Y9I29FecDPrGMLj6vXCM9ApBrPpjK9D0kU8A9a9HbPDHTXD7acSc9STw/PN9ZWr6SOOq9RyWPvcXBuT6O3oy+4fR9Pce1Hr0ZLdm94OcdPvWmRj5Aiyo+AeaDvmAk3z3IQ76+zAilPmlPIj1S/94+Tcw/vgQNfz481bC+LFwuviSHXD5qtZU6X796PlzjsjtWW5S+eddPv02yKD1Rpq6+P2/FvtPKXj6kPyQ/6RPrvecoHj60vC49suOHPilDHT+Glgc/HsmGPoOlwz3Drqg+NAU+v9Ckxr5cdU++3niBPl+rCb4PRwQ+B/a6vamiA75GmDy+fANDvY1vdL6nGKa7BACRvoeEFz0Bgoo8qy44uiFiTz412JI8M/Rzvakdor2315Q+DaQRPT75FDy85vu9DvgEPWHzeL3UMey95SDqPnJbnr2BO7C89osGvb50wzxJoQS9","ejTUvs6HFD22miI8BH98PjGxu7uiIgs9rUR8vlCdLj7pENg99RxxPAPleb2edAI+DcENP01EtL5ZGsq9LEGAPJyzkL0czpq92E2Vvc5KdryhAXc+VbWPPg9EbbxVaU8+y1gdO9+MJD6ZFiU+1EjKPWBPhLufjQG+JGw8voq9hjz/mNG9JMDMvXr7gj3Y1rW8shUgvqlo1D08hJ6+0ZXnPDE2G7v4IJO71eGLPRYKIzvHoZA+PbnXvEKFYL4czD++NzW7PR36ND5d5gc+KPfLvvP9sb6hPQ0+lM+VvQPQwr0Xxis9gnPtviikALuLlQi+Yha7vmR+4r7IBEe+25SUvZ173bwGHqo9LWibPqKFOr7RbzW+sQ5zP8RKBL4pslo+m8wBPy02uT0H6YE90wfAPfQesr76SC0/HhzWPiYBVj76Gzy+3owEP9QGp76nAgS/H1qUvuyCJr+jrX+7dEaYPg5Wzz6R0iS+bg7kOvl1i732dK69DnyGvh4r7L1j3UM8OimmvaEbX751Jg6+mBFfvQViTD7T/Ks+OpRXvpmtur2Ohdg+jK3UvYzeIr7zeB+/N6d2PcjUD7+aAW4+xf8tP8LpO77q9tU+EQXlvtnx2L6cNRG/YBZrPuWWBL7H5GW+oCn3Pi+1gz5vleC+6+SQvnbQmzw0odi+EEuDPjOs676Y4Ia7HNFAvZ7LCr5WnEO+T1s7PB8OZ7784rK9g2M+veyp5L2Dy+29Hm4VvYfj9LzLuiI9uuwCvmNocr5XE3i+dleGPdk8oT23rgK/kAnnvW2Jmr2GPbW9ZSY3vfNLF75DkJu9ESXDvZ0+6bykTU89Bh/eO00XDD19cLC92grSvP82xL6bm7+7XFSzvSU3D75s8zo+vZAvPf+RxL1yrry95i3WPdptGr7C6qa70ozXveB2Gb6PgVI9+50Svk79pzwYTbK73wAEPqDqAT4jleO98+i4vBBD3rxlYvm94Z28PTrJgr3msYu9laX+vGaOKr6tFpO9TCJnvthelD0Hvsa9","qF7qPKfX+bwXRbS8iPAjPsSqCL4DASE9XjuIvSf+kL3tGRQ+oxajPSNcP71OrUW+Q7NSvrUFgT56pkg8SvANPtX+Eb1m7IK8zvG9vg7KLz7w4wY845eDPtTqmT1qDgM+syH4PGKJ8r1uQ129zHkTvutqOj7fO1a+jRQ1PmHB9T1t6B8+ThCyPSbpbzwK4s29nl+6vrm+XT2YfMy8y3U7u+Ah1T2mTS29e79VvmcWlL7EeFK+RUfNvc2OKL4zhSc+k4nsPbDA5L6nHJe7vv9kPlQ52T2Hyga/bHIWuzqMtj0SzsQ8QP6IvSrzH76i76O7G7Inux4Szz2cLUW7D98BPjXq5Lx7qp88GyBgvjedKL4mJw++PgWTPfGW8b1RK2w+8x9UvZjb4T3td9q9USoLvmKJEz+Soaa+KpfevcrI5b1B3YA+/xrwu8qGlb54bfe9pkJZPVc03T4DMXK9VjYtvQ6bZL2EqqI98XqAPsjkNT7lVIY9x3QrvpCuQ77sey4+HHtXPptoCj0pzau+CNaOvKo2vr6DkDa+a/N1PkYtX77hRAA98TtPvGHbmr2F6r49hZLUvYugDb6Nb729ZSyxPn6UjD64qkC+hrerPeQTmL18l6g9Eep8vl+MSD1zSV6848w4PaSBvD667cI8UPDpPXp/yj3q/Cs9dyfHPQP3SbySGT+9JACoPrYebz4LeI49hQ5Ivqq+HT5f87s+djZrvs1sXT4+3We8EDYZPlQNQr61xRu+QoeMPfzUCz2iGzq+jc9ivn6GKT/P9Jo+zAs+v8CwSr9svkM/jdxtvqulcz2dvyO+WpHcPniAl70gFwU9TioDPr0W/r4KiYU8fM4VPcDXCj6KlmC+9MrJvUFKrj709eI9JeSDu0OKsjwcLXK+1uIFP7bLlb5F+4E+ZEEPvh7b6Dy5VbW+LRIfvGrJxj8FVEC/gQ5UvWZidD1DGZa+7AL1vqVwob7ohCi9kxKXvoED772gcK69HS6APo6JHb+NjhO+gOiEvf5Wt75+UEO+","a5lfvulzIz7O06a+CSpXPQbGu73Xapm9pWSIvRdQDb8xvcm9EQLQvUmjgj6YRWw9CmBJvtBIqjyoG4m+84C7Piukgr436aS8S6MdPcJxQb68yiC+x5cWvjfTi75jw1O+oZ8qPpdtSTyVy6k88TiUvcsqjL3dljG+1+uCvmRd8Txadgq+kJvXvVHMXj6Ou24+3eKjPrs2472vHDC977ckuuZyYj7bfZK9INiMvi30Gz3mGvO9vmhBvu6zhL0rvau+7oKhvmauAL755xO+AVyDvnQKzz0jhJ++A4aHvlbrCz4BGJ++G+Jevot9sz4c9MY920gzvBNxqL3BlqQ9FrravhW8jz3X90K+b2qcPRzhVr6ooZ6+hJdjPu+8Y76V31++IodSPqi6br4IV92+r56WvXrjBTyc+Vw+iD/8vYlsdT7M0Ro+FGzLviyGjr1s1ig+U4MWvsXYor7EhTW+M3QjP2cgVj1kPJy9IIlKvAyKBL5qERs+CjqFPQm0Yzw4PX++PhWFPejMUz6eAdq9xjifveszrL7ZPKq+BxkxvUGcPr5FLtg9xHUfPnheZr2saX69YJuRPd6KHrzGn9U8Si0gvjQw5j0dOG4+zCMNvSB/ZT5yFIY+WDXDu1G6Kj7w0jG9bhWIvfdGvL0eH0i+aXV+PmK+aj2e88o8fjduPaY36Tw/2fa9HesBvsftaD2B5WA880JJvm3kvL6agrM8x+pBvhFLOT3kKcI9ktaKvMa9Gb1lYDG9FgUNvtqOzrt5pl0+79IHPozLFr4uLJC+tdKNvBrC7j354zC++j5nPkAdcj0kqCU9I5x5vstM9j50veO8EK8qPbcucb0oDy8+PPUrvfHtwD1peiC91fMdPqMYAL2WSGu8TDjwPTj+HbyByZO8PsyGPe5+vj2oFh8+/bOuPG4Ntz2f2Bi9uiJIPLCbVL5F6OA9nsivvS3pvLxUEwu+O6mQvZaWjj1SwH096vlYvoPlfTx3eNQ9hIEMvh2ZvT58cvy9HIafvWmgQD1sMeY8","Xc6Wvfq9QT/OXtu+ODAuvkg4+rt4D/e9W8PYPsyt1b4TVp6+0DOpvsBSnj4cVY690a+LvNaGvb5o2dS95RbYvkNwBb8dQgi/p+J4vpYYir6G/hK+YNB2vnZxxr0N7oU8CbqSvkkXdz5c5DE+QsIcPkDUlT4Ev7O+oSJXPncAyrw5mz29mmRavR0gir0dctY+GokPPvcG1r5iGHY+jeqvPqdghT7thW2+GBj5PqjvaT7syrs+03CjPgTid7wjQI2+jbUJP+XKgr6LOu++/yOAvlxN976SjMg9kMwEPsVyFL2TKwi+XK9/vlSiRD4Ebec+u5ETPfns7776vTC9S/+RPsex3jzvQI4+xU2qvjreLj7SnxI+Jo1dPRk5k7xUnV2+2+SrPRO7iD5Byaw9VN/fPXey2T1wTO094lEwPoWPGL8pbaK9suGBPujI1zyyg4S9gQRyvvyLcr1TXU2826Y/PdpxWj7iW4g+5rXEPFoVcT1Q8DG8QKTxPdqql71RcxA+EzTFvsMqGj0IzC8+MKiRPFImwr6LLui9rEIiPV4gATwH4iE+T1d/vMXolDvVs429HfmcvRPCJD6nZ+W9YYu/vcu/Lr6Z3Si+Ac3lvsSpkr4Kr3Y9IfbnvTkftL3kOys+yAMkPaMa3D0lzxo+xL0IPqm3VD0RNe89GaBjPZPnhr4WxS49IF5HvvxknT5U3sm+Cb/8PXywdL2H26++d5WUvFwFLb7FsHW9mWGkvjCejb5vd6y9uy+UvqV5tD5Mw42+W6HOvU8SBD4wozU+4vwTviNVzjzbF1e+H7+nPfhOQ77M3py9Kc02vm163j5qIyy+SbSbvs+1wDu7cdK72TcWvs4tgj7NAE4+ZmgEvjv2KT5XIAs9gywFPjUfuj2Fqs49YZqJvhvOzr2swdS8MmbwPtMQOz4yyUi+wM8+vSIEJr5wvY070+HGPDtCKD55YN09+SgAvvQ1Vr5+qgq+7CKnvXxxXj0Wx1Y+OYocv0dmer3i5go++F2vOg4MNL5Sgos9","Aq2GPdg4Jb5UwWI+RqbcPuqUCz8Pc6I9GnGEvdgS1L3blyU9GoJFvogynT4Ciqa+mKKKPlRgWj13YII+WsGcvG+ewL0c8qS8lX2RvpIzh71YlpA9/86Ivu57Kb76jyG9uYu1va2hmj0GbR+/Mm6+vqFVTDqqIAU+J7igvrnZS72BIIy++3pWPALgJL5nfaI9KONPPU0FnLxpJwq+HRYrvsHMAj6YPoa9Ytv0PWapgb6GiEc8xHeEvDwcpT04AA0+ZZ0+vLTLVj2U0Jk9HHzcPVI7k73LNas9BYxjvlKV4LybESA9wV8VPj251z0r/p2+u1Q4O9YevTw7/R++BWa1PVGxFj7W/pK+cmmNvi/JCj9iVse8DHTPvoK5gD4fm2Y/dxsGvv8R5z6Pp84+JXg6PtBb3T7Ew1O9YH0kPiBIdb7V9Ws+Z4KtvvcssL4NVaA+6WXTvovJFD8Etu68sjlfPmiFiz65GDY+LLedPv4maz4i1ZS9tPsEv3Xz8bs0Nk++kycGv6XztzzXyjG+Crm1vn6EOb2b3Ic94mQkvRjKqT4ug4Q+XYj6vVWcej7wn2Y+2a+7vnF3zj4A0To+c6t/PbupEb9Cz6e+En+bvHM+4j6WVvc+QwQVvus8nj58Jjg9E7s+PnKsKL1bWrE9sj7JPr+pfT1IHFI+LTMXvo+d+z77WKu8o0Uuvd1vRL5rWMC9ecavvRh70b1DH2a+tEYiPI5fn70x4a27uUYrvGVodb5yh5a7dTjCvZvULb6sb/2+WquLvgLc0T1AmL88VEmCvS779L17O1++bovcvSHVP74O5nk9TJgpvBWEmr4sNBy+7qa3vd4Ik77/h1q+gggtOu/WAb4EHIq+0rwNvTSaLzyYqyq95SZavcZUhbyhPSi+L4izvZkY7728EYO+nx3uvTqzpb5c7w6+ByWmvgS68L2esTG9CDr1vfsgmb0ITtq9a10OvT4Ie70S1Vk8MBeZPezXhjyDVHe+5ucVvZKfrb0ziB6+ht/uvOGhob1mJSY9","IIOjPYtFP70/WZo9H2wXvcBxA70HUx09Bx9/PU98Sr1ZlH49F0zfPfNW1L3/x6I7GHAivnFdXbvZuTA91rAKvLwHSr1EG4m9KQIEPsFiDL7K8tY8Zw4rvDsV1bzRPbs9gh7nu2DDXD3KABU+VaTsPWo5r74OIDg+zthQPZurtLzE9NI9sokNPvXmxLyI5sK8ztZqPagxzL1x0GM99409vCEfzDx8zs0807eCvtQHyr2ABIy9YiA0PWO8bz0CDym8lvICusxc4DzpLsC9nfRMPmJRNL6iIGi+hvmAvaiFhD1vRdC9M3iePCXGmj3hFj+9FuSsPQ8XmLzJ1HG+UPodvH/5Kb1u4+U9wOqUvWqKP70V/t89U+bhvO3pLb4XNP89LaPOPXwCsz2qH8i9zj3APSxOgL3Z0oY+rBYRviAswTwo9ao7a274vf0y2bx3Ejw875G2PB77zjzyazW8OJ+ePmuWwr04q0U+a8JBPuWDAb4oXlo9CVITvugBmb4c0he9pqYnPqpsK72wGhk8eGWlvVaLmzxu4Gq+gG9RvDfWzLou2ck9FRWHvHxRqzyUClG9Xjp+PaD2Bb2HrIe9wdJBuz8M771SYye+ddGCPV00ML0N1JK9sCA9PSeXwryhm5o8lHPNvS8Qhzz+ocU9uhLcPX65j71/MJM6kkSpvXOrRb60kYs9nVDtPWoUWL2aSAK+kEmyvYnTwL0jlkG9lQKnvpkZE76GKpA8vzzEvVtWwz0kD+y9/mavvWKpMD5vlde+upq9ve+4gz4WGQ2/1HQfPntnFL5+/A6/4gy5PK5kILzJmD+9zMh4PM9gHb0+fmC++JQfvW/9cLvzaKA8orkFPqEwgT77Pfk8JnuCvnipaL4KVO69/avNvd3AIb4p2S6+MKkEvilpnr07THC8tGeEPD+8RT0Ht1g6HIJgvgHFe74APEa/rGoUvqhCEL7lUpO+TcACvAnCmbs9Rv+8ifgrvh0fGr6iozE8+nqZvYFYmbxkt5m+WtoFv6OVUjwkw8u+","DilFPhkpIr2agcI96XvovR7dGD6mf849lSv3PFeiBj+8jCQ+qnPdvaYxJb77CeU9TUejPQi/VL26v6w+yAzjvnLZ5D4C7JK89A6nvdoObT3ErFI8EaVuPXy2xz2EmCS8+URDPpdPHL5f5U8+4KhuvhvXED7tufw9SCdqPd/8hL1yfJs+RVWmPHAqrL4Fc1o9ruWBvr+Q471kTOm9g4Z2PZUEyr2ySSM+SU6ePWkVhT3ZlGM8SZGLPcaGDL5ZQCs+DaIzvpCAib1Yxx4+GF2UPkDkbT2y2hs+zt5VPvavg75P2oE+YLsOPlCziL6SzQ6+rX7EPWeRBD08GUw+VV0pPog68Dmlwgk+v23UPaMfqj1y30c+ri0Kv14RkD15B8E9HgpGv+8VWD1gVNg9HGRoPnHEfD7TM949Q83gveOUPL71C6S8lujFPUK4Lj0WFLI9SGOJvX7car7dCq2+bNHuPRavrL0I9Xw9fsowPuX0cj1ba1Y+wxXIvRwffD5ESxM+rJnJvZwWcr52rc4+zPEpPheHUj4ZGNI8cTuQPc9k8zzVp3c+ucs6v9xT2L6p30e9R6XvPRewlr37VRw+EtRUvjmIIT1QwnA9GMeaO/hHnrwfU7k+cZjJvfMG0b28v9o+Tj06vtNQvL35ens+0G2qPbWq9L4GKzA8toOsPhjWOz1T6rk9IkNKPp/2QD5BEao9FbazvddhAb1aX4c+Wz8mPmHKdL5YVvM97iZpvTJqjL7xQyC+A8twvCr17bzIdQ29Rn1PPfW8jr1PNBM/E22DvsIt9b112bI9I8lNPeCg9zzTATU+KE70PQE7qb4obsg9uYUEO12c/D1FttY9tBDhPfszXr2qwdk+h+8Avk6+mTyp6By+e93uvbS9JL5sSis+NO1IvPRF/r0/zXK+pEURPo2cFr3Fifu9QiCDOj0iYT5Nen0+ebjWvZxtIL2e+5a8m4PcPbH6i77a9CU+FqpdPWRxqj23b8g91/IRvhNTw7xD3Qo+6k+ou4KS6TxAPYS+","FYi+PZ5jj7/7ryk/bQkEvkQ7iD7tZKm+IAPzvskZw77bw4c9W1aQvt8OJL8zTDc+TxvbvFoSJD6GTaE93UaKPnYffT4gtxy/fQ+yPlK7Nj8gSeo9rZULv4YwLT7GW2K+ZTaPvj8Lo75q1bU9faq7vdyLyT16lYO/lxCmPuPhVb8Y8fy9IgBBPppyOr/KGU6+ycaCvXtxgT01VgE9GXKBPRiDhr1Pl8Q+QoeXvvAOVr0Lb3A+Ku6VPr7lHj5ucBC/sXSCPYyolD7ZcoA+dR7GvV5UrD5sJI09bPgrPpUC3D2pq7M+0oHBPsdumj4JOso9VfX2Picg3z5HocM9r2JivkuZAr5kEag97cSmPLc0OT7k/iq+fumKvJZNjDztNmM9yzctvsULgD2h6/Y9MEhMPorq+bznaDs9DXOyvMNpoD1VtlG9MrymPC8suL2H8ee+denTPbgAKT5UURq+EG2Zvb0i070UXYE9/sgaP381LL46vgw9twQlPVx4AL6taos9lIqEvbkLOTyBLyy9XSLyPasBBD4x8dg9nvmNvBLTwrsaFk282xKMvrLAH73Wh8c8/djuPRkOXr5WX8c9l52QvVEyCr5y32a91voJvlJLrr3yL/Y9siOHvdGVlr70ki88+dQmvmt8wz2o55o9ONxTvPYnnT2P02k+6qU8PZX+T75NV5k9lcqUvkpXBbxqXva9a1P3PkX1MT7UNYK+77lBvQ5qhj5UXzO+yTksvQdLlb4nm0Y+pve4PiN7Mz7DEEA/dIxTvsmW+T1tQ749+lTcO5s9Wj13/aW+rlOzPXwM6T4nCb+9k81Xvo080j52h7+9mc5kPhGIML5cNp0+mJZ1vkWuiDzALYm8lt6cvlJE0boax9a95MMzPjCBsb3eiu28mJw1vRa9Az7I9qA9/OHDPWDSOT4xFt275+MiPvYt+bwpI+M+BDoCPg+OKT4AV84+/HcgvuRkMj6IW4c9VkspPtfDFz4sFt299hy2vmY1F76nNhs+qYhvvYeQGD4x1ZQ8","RNkevvl9Ob6/4Be8V8gTPSQZ3b1jQti8ldvYvke7QD0BLY0+mrt5PSyJWr7NmAy+27ggvRWyNT5yLTE+3ivOPVQyoz02PHu9foYzvfOwsj05UOg8fJJjPK63mL2Ri4w/lGjrPXLYgD5qAtK8aA2cOJhxrr1u5hA+fuwhPnJQDj3eKDe9qZtzOr32GLxIpUE9e4OhPQL3kj22G4I96e8/PEc6+D3Z7QQ+k2JQPd+vjj3Uidm9kxglPPHoej7B11m+++4pPhVtNz0Mbh69J5YPviXTjD0XzHw9HD2VPezoi7+aZnm+zaDiPSJ9kTzod+E8IAElvbHGCD2F3Mc9DRBRPfwQhL0EXM89QIXtPh6Xpz4jg4O8T/ybvefwRz69GKw+E/DWPUrm0r3ooAk/A7rfPT+KhL4RySa+2MNMPzxXxryIZs09w6k8vuBkkz6N/nu8t51zu/OCMb991Ye+2AzgPTZ2N78t9p29ChT6u5YVrr4Ive8+sGeDPknPrLxY4t49wYMhPuC8jb2q1Wo+okoMPlO3WrxqMrG9IfsSPrlX4z2nuEs+ZQSXvnLc/T0+wA69KeawPrhbIj4oPFE+gvTzPpvBuD1qIk+9/haMPxptyz2b4pU9ZMatvt7ZAz137wY++qj3vfXzSD7qWQO+dVZvvjBitDv0c6s8BT+RvqlDkz6yQLY9XkeQPoEHZjzM8AY+wZyMO/IMDz4/C24+sWgiP9K5QD1cpFA9aAJCvNmcsT3Lvsy8RdmhPlJ25z1G2uK+Dc+fvRnzI73vo9y9lBl5PtDhUT3sxm0+DBmDPezPrj3r9SA9GAGGPEFgcLxGjv+8ozA0PJb4rT60GFo92hOfvW8Cdr6b8NA9mV8kPhZwzj0WPxs+0PiwPaz+oj5hj5g+worZvfaw7j2G+uo99TYrvX+50L1Mn0c+eY7oPRgnaj5E7YM8KEPLPeAQOr3u/0E+1Zgmvc2hmTtCfZY9YGPYPZs6Jj1w8p89daMDPj5ziz2wzay9oFmsuWAaob21n4o8","otp1vdCskj6r/K8+IZH9PWHduz6pUHa9HcPRvMTsDD5ZFYi+36npPXab2T3JTAU+TDXdvbmCzL6m/do+GXaXvsBcYj1NbTM9SE6YvthWK76dARu+5sFaPhZArDyP+Be+gaB7vSwpSj7jN2c+tk+TPnpmub5v5+89r5CiPH9E1z75+Yk+0GE7vKG+gD37Ckk8ICrHvXipRr5Y3tA9pH0qPoXiNT7omOW94HaqO0hr0T7DcWQ+wauJPZeY8L5tBr49AUECvTJh471bMhQ+N/FZveYJoL2RjHA7wnI+vpjg2L2Gktq9fjOsPmyS1z0GaLU9StFSvgslor1g+b++SeCJvORR0z3prqs9hh8CPv5rKT2oBQU/NPYgvmHXhD7Bknw93YTmPf0yIz3uJcc8oyyFPQXmlj5yz449TnMCPucVCT10ArA+hUuTPQoeNb3hya+80a1pu8eQWTzoDmO4GMWqvTv5UD3lPk68WzXgvh1zj76H86G9+b+CvXSn6b1URys9rGiTviDQlrzrZ0S9zleOO+SroT3DtTQ97KEaPpWxrj0gUQa9EtszvZ8JzD1Vhgi+rTLJvR6m9z1lRIm+OSzNPd4zID32FrA9CY+lvURjBD5uv4o9CY6DvJEGZL2VVzY+eHWXvQxnS76rXTu+5lM+PVrMEjzzTBM9U5LUPRq327ypcHO8gzW2vifjxL68/Z8+biT+OiI3ybxrl6o9F7JyP7s8ST0x+IM+SX1APx4axD2bMR8+u1nrvYv6Cz47PiI952AmPmL5jb6g24a+mDtIvm74/r44/jg/sP7BPmweKj79rzE+kRjpvC/npz6WvRk+It1pvt4EoL3aUa+9pDhcvoR+Aj6EmUI+zEUyvpNjQ787naC9tmusPb4Wjb1OYRo+uocdu4PdpL1WcPA+8Rk4O3u78r7TYxI+7w/KPsghFT5UhPG+02cUPhu/2D0Pj/s+HmQCPobwnj6MU5I+jZuLvj5dwL3He6c9iRj0u/K0lL1J2RY+GUu+Pp6MQ74Oaqk+","i96BPTufWz2TdqE9eQc0vd2kNb4UbJA9/BQHPvkTCj+NI2q9IZ6DveNIhT0wUaw9P6SXvfh6072ABQm+flxhPEsud72I0kA+2aBLvo0N6L0g21M+y04rPdXdc710z2I9CyQfPRivrb5vgo29mK/XPfemUj42mgA+XZmgu247j7wIXbQ8bn7EPZeaXT3l9no8nfR7vYWFej7l2F2+R1vVPEqbaz0bUyq+ltcHPhJ0DD4izso9nOo0PRTvQz42IKM9Nj3qvOf6Cz3qI+Y+FeB+PAWaQzrDzoA+degOv35K9j3ZPZu7v8itvryAp75Q52w9L1L4vVQQt7yWB6i9YBi2vpp5ez2sQC09t1OMvdHYmDyEEaS83FdZvioOQz4HHgi+wbVZPmrOpDkcb6e9hdwTPvqjr73BF+M+pPEvvv5DqT74yqW+yTa/PdYQC76nqVk+BjxCPms+AL9pGjg93zekvcLEPzz6kC49jGNoPewXlb4ZKBc+JoWbPlwGFD5+mY4+hgjwvuJtNb5AllI+5cMivEYojb5GChS+0JDtPZrSVL55KEk9bEgqPtLkf75If7W+FS1/vRcsUr4VRIW8E4qAPq8dkD7qWMu+4+OHPD5/Tr6weIq9gWy3vn5gZb5cn3q+t0MqPZ4tEL7jAhQ+CDZoPTyjCr3xTNi9XM6fvAKjkj70Z9s94BLnPQkMU73poCi9OCQxvrPARD03iMO9PPyHPv9EgD1ygEQ+Kxx8vU7jWD1jry8/b8L9vuB5l700KtS91yYNvkKwkL00OSM+Rz1vvWW4Rz4kIk89WbzUveKTLb4/Mkw9x4eVPvd8Zj42TBg+Dac6PUF2F7+Ld789rprQPlHhFT3IiAM+DwIMvXAFwz2FkjW/uIh6vow6cT6fQzo+HSobPo7cMzwtLKK9+XsvPu7fC733fMU90UT7vd0hTL3Axx8+OS1tvs5CF7h10ji+6Zp/PdEbEr4SoLs9yUHLvmkBYD1ry7s9ELxkvQfpST2bKc29SQgOvxe7N7tpVtG9","aYtBvFoTuL6oYKs+J5z7vYg/lz14hJ4+Lt+CvNDd6r5ZkP0+LiR8vlNf/b5fm30+t4sVvxkZRT1/+88+yocjPjFiC741Iz4/74GZPsLHrL4OHzO/9LI8PyQtmr6YDLi+USiavmQWOr3VQza+xUgePhmT5z7z8+G+5lw1vnmIiD7qHaU+WOxAv7IAoT6avWQ+HSyHPqOeOD4mGnE94veEvhsUqT6VZeI+IptNvvreqz2kEoU6lg85vqxQ/T2xoj2+Kn0ov0XYlT73mSk+CSSBvtdrDb41txw/ezZEvQKSkL1/6VI/4mQRPpbW3T6jqty9RR8MvjgdeDoF4/2+lCkjvz9qRL3v/sa9ofXvPYKNFT3Rnjw9A4aLPu2tOT2LZZA/ZNxOPuFJ7r0OlGq+acMSPYYVQjws6cA9il+4PYVUUb614ky9rUdVPnMGBD6WIzu9bjEwPi4DgD61PHM+XFJuPh7Z5rwoVc692s/vPu55wz1s3Lw96T1TPoQu7z01sWU9ELRBPqfNiD7Z92c9YiUvvZDuDL0R8ZQ+YrwEvvKwSj5MKfu9cEqUPtpxWD4qSL+9tIO6PfoZij4aGDU+tD1iPoINVD7ogC49dmI9vbxgJj5QyFi+PBHrPQ/wDL4YEO09lqg0PsYEtz6a7R29ps2IPTo7OT5AxT++oAm5veTVIb6Sqqe9KbjnPi60Ir6u/2U+5A82Pr971b7YTfO8pkCLvaFD877ioZA+0l0cvQpsXj4+p0M+zGpyvw73Tb6++Ko+N1bMvZIuf73t3LC+GwrjPuRnHr4/iB6+qIm3vu3AEr5c1ae9T4gKvlxZcz2c5ZI9j/KXvNOhaD5lJum+NEIjPwqOor44M/6+O1pVPa6aNL6CB3u+CrLHvToynr3FzDK9JYqNPm9Gj77e7sA6qtVYvfxVCrwhSPs9NSKuvg2lKL1H9vC88ZR/vi1w7Lq/Bh+/2rMNP/qZUL3wziG+LcrlvUKaNruGUWo+vPciPSOlOT4i0Nu+jBFWPcFCjD6Tf469","+qUcPoTQKT5UWNC9EyQ4vnaYdL7P/gs+WWLnvbYJCL7z7US+JCtdPlK767yk0yw+dI7uPWPdPb2EZKu9iaiaPQNwlT7NXQG9YVobPtTV6b1fTJG+ZaOiPr9HOL6+tIE+7rcZvdy0Ez5cuAG+YPmhPe7gUL1tYEi967zdvWKK5DtcRya90bMvvdMTCL6hyku95VL1Pecy2rqGORG9+TyLPmbtUr6OAqi+QF2KPMmS1DvxAyE+9qgzvuOJqb6kE149xJUVvCNaOb3GqRw+MykoPzS0az5wjTe+XPoiPkLGuD2mde49womNPXuIDLzDpx4+jp2TPVudKjy0kvc9A627PBrKCL51jp++4VFgPyfOhT46kJo9LnlkPvfz+j3It+I9c22lPX5gcb4f1Q6+MPEhPsgLFL6fdN89zFUDP+7ioT4Syd68iiLPPoQe1j7Rc8m+SleSvnSstD0PXeM+k1SkvHc09b6IwH09560SvkAORz29V5m+niRhP9wQMz0b1f29FKMRP9u65T1JVvW7l/unPt2ezz7dGaA9pe+APf7oob69+ZY9QP7kvjyra76f9rq+I+00PTTm7b7jj1G9Qg61Pp+o7z1I5Lg+P/AePw98170+8/K+KDmsPgVGZTxJ69E94+UcPKSWnj456Ty/LSpqvq1gsj09Q/i9D5Qav3PlHz6JG6K91M0mvln3rDwZfgO+mPCGPYunF74h7dM8ZK6yvl1BYL3M5iG9YjhfPlwdZb4ywHY9pYgIvj3yfb4zZ0K9YB7bO1W0wj1IFeS9VWRROxOpdr0Zz1S+wAINPl+nIb5jL/e8P7D/PVsEmT5frlM9p9sqvraPnb3bwtu9Sq3YvaZu8Dwc7iy8xesvvlUD9Dwnm4o9j8c5PdbDAb6wMTi++JnKvem2hr0mLnM9kWxTvf+JBj7mW5K9s54cPl1EGDy/9qg9UcvrPTcqeD5CZCm+LxJOvNGXtjtt6ci9poMDO1piCz5mMgO+LvK4PRC8Vb6n3c+9VOEVvswe+71PW00+","5iZsPHb8Mz18rTm+evs9Pd8Ylb0GybG9fCcEvZQ3ZTwbLow9HRMxPhPUQTvoBJe8p5FFvtREkT0lgpi9rhgxPsNBnj7b9NM7L1hYPYu8Br7k5Qy9oP7SvgIblb1NyRw9zJ8NPv5Grb4Jcua8Qqihvi82Vj7XrwS+vt/YPegNqL0cChi+zOYlvKMo5b2wC4W9dLRWvvpBTL2S1Gs+arrevEMWID3aqfS8UkctvoSU572vCDQ9tL9LvPPPlrzBAIm+IA0YPkKwYL7ZT7e9kgGHPiS0WT6Qfli+EaFjvjFZEz6czS49wMAOu0kSAL4cymA9mFgMPgu62btA/VA8VoKfPMoUlDuSn7M9dy8evtjiKT2U89u8RWBlPXILMz74SAY+fp1ivQqDOz219Sg9pnmMvVuHzT5B1yQ+mmxsOZ8jjLw7F/09QIsgviwMhL1V8KC+pQ3XvcsK7L00fJy9iq7lvboPsT3mNdQ9s2sZPpXKDj5km1m9JXR8veXKUD0ECjS9jf4lPuUeJj7tM7E9kM2LPe5zMr58THy9TWBAvYxnwT1yUxE94c1PvV/td76b9qO9sGscPn28o70rLia+DNTYPRvmsr220QS+CS/qO+o0gT6qGjY92xlzvtTotj3lR9u8U0wbPhCejj4o1yQ+tThZPY3hjL1Eusg9GWehPTf1pL1T7Ii+6zIiPqr2DD5F34Q+KNIlvrak+L64fOA7CMk2vph5x74uQAu+KvgCvo88iL46FiQ927dDvbQAwT6qzB2+ENK0OzkuBT26Rkm+4DHmvX6qwL4sWHS+86KavoKnIb6Q14w+N2uiPbIk/r3YuIU9tH5bPoFSiD39d4I9Ov6pvBuoWz5RKCQ+juukvbR5YD6tozU92bUTvfAhIb5HEo+9bQAKvonHLz6VtjQ+OH12PrAPhz4xVBY9uVLovpP8Br6Pacm+fhGPPFCeIb63l9i8FWN9vi+Eib4TMd2+dl04vdEdib7uwSa+sG0bP2jXRb4QxEO+IJSnvXsK7719DfA+","ZkoAv5/FFj7Hrzm+qJMPvs8c8b3xc46+t5sNPl+O9L6h0ju+iTmSvagPFjzYi4M9SdTrvpTCbL3pG5i9jTGKPeb0m702Awi+9UBWPktMKL7E6CS+qgGevaAwjL2opDK+xPePvaILVLyHpnk+VSFcPUIdur6hGjw8DHVgvrBcQD4REy++Y+tBPVaU4z10cH+8eT16Psh7Lrz3IvO8U5yxPcaqlr0o+P2+D57zvf6deTx2OZa+n34jvghGgT1V9+y+A1ymPQzLF75rR4m+v8ixvmcGlr0sBCe+oMS/vq8z+j6BSwO+36Jrvj/ycT6aOWs+Wi7RvT2rfb200hm/OZcyvjMDpb1j6oA+B/j3vWen4TwMqk49QB/mvY3d7b328YW+w4ABPnGjLj5Ntti9JCiGvlxshz5Ad6s91PfZvvLG+T7ZINY+G6vrPfl1XD3ZrJM+YeK9vTmpyDqFxoK9sDT7PVFxoT02jIC+PqWzPSgTH74AtGW9b2MPvshG2r0IX5C+g6LMvgK5LD2FaVe+ZPeIvF8EwDxrH+u+iLWBvWOmKr6Fzik+Tkd+PqgMsL2EiNS9KoB7vtBmfD7Ycce+P9jYvFSjDr6a+rK9VzQZPphXt77wb9s+55OMvFFhoDzsPfO+AWK2vfUcRb1txMa+g6SlPvYbRb6zFJi+kPQ+Pq5L3j14zBe9q/3fPXAOnT0b9769hArkvHhGCT4uYX4+1t/cPb+sSj4u2PA+rqZCPnoSdL7zecO9nY6NPZhujDxuE58+RhvdPtU2+jvvn4G/Xum1vcCf6jzc43W+Dwx/veQfAj41qgw9tj5Kvj/+1b51lA89TUZkvfXyFL0sht4+A8OoPs75lL1Eyca9RD2ivSnXQD5XUEW+oC5XPpTaibwNBAw+/WZ/vROBWT6VVa49I/CWvdSVAz37hWW+mlcOvrGldb7+ArY96ukIu3zjIL69Bc29DmKFPliaBT7Yz3C+cjsTvraR1L04HB29NZ6UPWIlmj6/cAs9Q0BSPk2pAj5nfZk7","eZinvMgE1D6Znoo+2Dl3vcWNCT3NGZu+naJWPB00bb58bAK/E2cBvwrgmz6nSym+If+Yvl79UD4qEJ2+72FvvqHPnD1DLHW+KH4/vwWzdT6d9Iu+dw46v8lXor0XQcK+EWgNP9wCpb4cxkY+uLrVPUwPJr+ofrA+8dp1vmGELD8hShq+pxRBv+Me7T7t6MY+V+o/Pr0jpL74DbU+vsIYv5Il+b7Iava9RCwMPCBLR75cZhG+w9eqvtlHkr4oxIe9Agb5vbBdQrt+Ly8/gqmzvrKiqr7DJpW+bbNkPryydr7V3/u+TM7KvjJNuj6+9DI+mkcuPBxqJD6w2qa+WYhsPthwrb31XXO+UTD/vS3GCb43JhG+9coEvRyO873gGk8+neQnvq50j71fXuY908civvBXSb6/jVs9LGpevRlRWr7gh+k8rlCRvqyHrj6Z6oS+nCSFvsvulj3JK0e85dbsvTmthLzR/46+YyxpPgEyTjwhkA2+bZ+OPt/SOb4A9dW90Ga4vnDNXD7BnjM+utkru866mD26rwQ+0VeVPsjI2TwCMYW+t2gPvMXE3L3TxMK9KTAaPTdfEb2UxxG+A6KLvqtDFb4QC+u6K6+dPVnyqj16vyi9OteIvUzcLj0cfQ+8cnDVvRejbT1aAU+9Pp/+PRGw1j1LnJi+mZJdvqrz4T2Mx8c9TSWZPvlZ5D1lE4I+jQ6UPsYP6LyyrJe9lXOevuDJab77lqc+9TOfPl9oRz4c3K8+Li4XvrVxKr5iFi8+5MgqPrguzb1BYd++G6C1Plq5FL7fP6C+yKuxvgkqpr3tQkS+lb1zvbIoDz4xUJu9DoxlPkzPD70Kr749zIHfPlvCtb1BajW+HJ8wPWZS6bvL0yk9SbAvvnZZZD74n+U+GbcuPj9O2j25YoW9ETKBvT0LuT04BZO88867vqvOJr5ccgY+gGHsvNVqyrxeYLY7YXFCPpS5xj2wdhy+gdXwu/MAsjxALwM/RuMJvm4+BT7AksK+NGEHPoncJD79jU2+","Mt7EvfNuHD6sJ5g8t3X/vXdtqz3qWmW+UP1DPlavcD01ANa8KRztPaBlbL6ZX589DxQJPk+01T1ezFC9adAbPhn2rD62p/O9p3mPvXw+lTziiYO+oRUgvPN9lj3DIAA9pF1EvqYKQDwcVSK+46kqvuVc2r2MStS+7yiYPQCdUL1ouEI8NjMovtY6jz1sTbW9tOhHPR5ETb0/Cvi7130EPYODhT1YtWC8Y/yiPXLNUDzYukc+T0wtvrkPeL4gU1A82sjHPqHCOT6kwTA95ZNFveCRRz73NXE9vAwlvkucgz2EwF47il2gPoVQWL64HsI909B+vtfQyj5wsVM+x/ChvUZBgr0znA2/TFgevgGjJb7NC32+JD/fvkAkO72fBK8+emipvjoPqz48RuQ9C53bPUg0rL4FeQ+/8jtNvd7BDb6P2Jq9NUxTv7Tf6D6mKS+9qA3/vBNJjD40VeQ8sRhTPQ2nk772b2K+Fx+APouDdL4s3gG+BXqHvgU0yLxFP8a+GCGrvdf5ML7TqYy+kQtnvbtXVz6QzUe9D9hCPKfYhTz/ow499LqhvTD9tT7u25O9gkZRvkaVnr7o358+gi8evs1o8D24pKy9QKn4vff/cD2Va4i+1rHovVHFOj5HbYK+sq6MvjU5q75vxxE9JyV0vYSzqD3VFO08+p1zvmPyzD2bPxq+Qo5jPDSRh72Fg5i8/rKAvkFszzz+cc28ODvAPJGnNr79Y9S9J54qPqP3Bbyt/aK+6AT9vZqBO72l+E09eZ9CPkGg6DugLi49fqH+vujBkr0Pd/M9pIkLvo+yx70JvBu+RNEfvsYAdD4A7qm+zZU4vkCiXj5Y/+K9LcI/PYf1675xFmc+O1BbvVMfgDy8426+WaJ+vlV7aT44VRE+3yrOvcYjBL7EDpe82bYXvJTyyb3jMRS+poDDvD4Bab65SJG7yWBVvfyo8L4e9Oy9mmzouzrfs701fZS+wwgVPlcsjb0IP4q7IyVXvgUQQ7wB9NE9wMiHP9jtML5gpok9","4r+iPII3jb0u1bC5R7KXvc3RwL0dktk8xcHWvdR/nz11P1Y+k9v1PLqzPL5PO388CvmoPkSMST5qNaS9UlHXPrBUjr4VK6m9fNkzvlGtvz7KrlI+qGdrvr6M4D2h6e4+0XEcvGFDxLx+7UC87BuNvVtFkD7RWaO+vG2oPQtNTL1y9J++lihDvpzoa76SU+s7mfSaPq2yLb15zdG+tJFMPZlMPT7v1do9TIY8PVaRozy4cPM9C3ZLvZVtrT3KlL49Y+FRPkfBKL48rT89sTPRPmZCYz7bAKI9dkxtPUg+ZT3mmio+fRkRPIFe7z3n+IC9RVSXPS3FrjzBBWI+7RP1PR98Qr0OUIG9wQ8kPts3VL3rV7+8SqAMvnqSQb5o5wM+4AZOPjgwCbySg4O8doqCPYcDjj185UE+34jFPiVkib29r2I+xkPtO3+Xlb5W5Ba+/Ke/PSqNCT4qaNm8aCGBPgL1Oj3foH09vXVvveXaFb0DxgO+PDg+vhpeBD4wrD89pbLDPeeql7yFsRm+mzwfPtLcBL+iI+m9y9OlvXxhYj2Abdg9fWJjPowDMz3kgzS+tdllvRR/fL0SAQI+UuHfvWP7g718Azc+iE++vfoOFL6o4t+7790HPjnOUL02RCW+xME8vsl8Qz3ArOA8DzNgPv8H8L3Sr9Y8mrVAvXDh1Dv1F4C9PLyQPk6FQT4fJoS7kGtqvVuWtL2e39A+mX+iPtXEf768ygg+GrpRPup/i74matS+D3DkPeNRDz5Mt4q92uR8vmgSODwDHEu9T7qNvwrStL7EnhQ/TQFnvIvvp72pYcA9phs+Pn9lhDwvpVi+89mjvnUOpT4iHko9x/xfPtIkLL5c4VO+ozxDPmxjVz4q+Tg+sFTxvjZPkz2g6LG+vIFDvgLicL7273Q+N92AvOJ7Uz7kwHu+4rygvZmVvj4eG8u8jEcQvWrWTb4RBkI+3890v3o2Gr+3kKY98gMvPqJ4SL7PZZS+fzsgPrRn+b6bQFA976YevkeREL8N3yO+","2e1GPsKMJb6cY1w+Us8VPX0cuzs7uXE9r5gdvsRP6T75g6+9Xs69vP+niz2xeaM8g09GPcfcF74GSJ49nNXavfNkRD6IDJE8Np8cvny1lT1Ff5c+PuioPVW0gb3Tu948dEIFvvYvh75dLSY+9HBdvY/LK73JAEU75MytPb/ucb6QFks+TofZPJ2rUr7oPPQ9MCudvrchoTyY9jG+jl8wPaYpxL2a23G8FaxFPcf3ATsvgI4+cdkdvXJ3Gb4K7gY+0FJGPV9MCr4BqbU+2RuAPraF8DzM4/g9RsYxvqCcpL6IXU+9FxERPt/xhb0S7v88/f2NvZjhPr5Yih+932pDvaohczsiCTQ+WgByvE9YMj3tX0U9ZWWgPfydFT4iUUG8rmazPYv5rj4LP34++PavPmpUZj6MsBw9tI6cvdIpsL2c7Ry+uUbsvaoa+D3wWry9jtmLvaK41b2yeA+9rDRaPbv5Br1PbZG9Ps83Pul21j5b3F4+aHnBvRcqgD4NlC0+yeK0vvGarrx0lLc+FUCxvkH/Z76pvE2+cQp3vfeakD2RFnA+uTRePQV/2b4gOF6+MMo8PkzUOT0I0wU/hA6Jvg5dBD7kfKO+1dPGvZjFOD7KOIY+H4mhvk/vpL7Geo0+JYtQvuVTZ774HUE+2wwkPdHavb3jPXY9i/0CPnWyE73pwF29eQQWPmDc8b000q+9hxLZvYWqBr5ay5i+txyMPkRJDz5X9rs9RdANvux43j17I7E+ims7Px52Sr1utMG9OjyKPdXAzjzTIZ4+7SqsvgqF2z2/gIW9oBtUvjrT1T3TIAm+WLSKPcXJoD2TMYw+pF9bvviezb1J/SQ99lsaPAVDHj2kgrc9RyiXvoPgkb0TFMG+n47AvlNwwb2A5JI9cdOdvfsrcD2hAFm+14ZMPiFl0r2FzIw9b7QwvXiGjD1lvuk9RxOlvvunGr49UqS9LhjCPIBJu72kwo8+Rp6MPAcztT3dO1c+WS4jPtEppT7Stts9iTwUPiugGT7syQm+","bBz5va1YIr88lO0+KSyyPouWOD7Z9Z49qh6ovlYebr7dlys9O7VEvio8tL6Dj+K9t9YWvzgMvD6zKPE9XpSFPg9rFr5GrCM/HFqDPgXmaT4gqUW/H4YUPEYDqj7JLB2+LMLiPkV4/b0PF1+9sGLGvazfg7vRsyW/FKvYPhWBIT6Qfrs+SplYPmXu/bo3V1C9Kv6SvDXgzL0jdzA+Y3ZEvWMBjb68WAK+1SC9volEqr2VbsA+v3eYvjwASbxiurA8bmgFvyzbnz4zDh29dIkuPGmtqb6VaI09ZWquvfHcyr6zUbC8JwWfPVcLGD6z1XC+P5gFvmYBaD3AHXe+ZHgPvU6sar4PFMi98ooIvr6dHbwXv4u9yKPGvbx+wbzvEPS+tEokvlqWNry1kNE9zJqnvTVlA75EbCU5rlmkPduKVb4RUYU+av24PbTw/D1b4h2/sTAGvkfhw73Kr3q+dn5DvpvKtD5Flgc8BPBZPpy+oL1ymRG9+ZWtPLj32b3Eiro8EeLRvjcFy70I8IY+m3kcPapFEz7up8y9SGXcPP/CKr6HckU+zPlEviTYIbwy6H693jlKvLG3eD0khS6+FMODvt30or5KX3o9nOi/vWY07b6KSQ48v22EvoFqtL5lQmq9WfG+vX7uBL4l8/E+IVR/PWgl671TsUE+WB8XvvHqDr5BAKe+kma2vJ7/kjrEDFW+/w2gPbs7L7z87fC+U1Vou0VAOz5/SM+9Xe0Nvocosb60/aA+zCIAvf5Wu70tirE9owNRPhAiQD3SCjW+VhGEPiUAAz6Hqii+qC55Pjz39r0nwbU9Jqs8vnWBdD618Vq+NQOcPtmoqz6gZzo99SyUvh2nurzHWwA9b6c+vhQMeLwCTue8MJYyvLNncr2CvuK+JvzivD4oBT33DLC930LHOnWzkLuBLBI8NMHyvdPgg7ydSYA9rU8uvt3KqD3m/4g+lVvzO3yoQb1Vvkg+jraePvooED2jQxI9S4DJve872z13GFq9ZFDsvFuUvz4n/vi9","Bh8bvUA5ob2JAlC8EPXNvQoxKL6VqQm+45+kvaaukb0ZZok9RZlVPXgaer5gtpY8bxHvvazusj2nMVY9QtRoPuYxSj5PIQy9Jib0vktppbzyw8w9ZQ5Hvv3jnb5zHeo831+0PT/Aab5LQDc9p54gu0nzj7wkXS4+EVo7PkdDkr7QtwS9BVgLvZM5Yj3NHB8+/YuXvTO7Bj3TRYk9zCLPvWRvQT0V67U9gcZUPbxutzzuJ8e+W1+Kvo2kTj59XnC9cOeTvksrmzwc3ga8JgFWvcBrVD6hka29ulTkvaaOrb6vrzK9sbRRvmuP4jzI7RE+VDcJvablJD7DRUM9wuksvOPp+j1eHBY/HseXPfEhfj5fE3++9DO9vi1Bcj5zb7m5qp7dvdVSiT4wdxk/WNcivH6tuz34loK+CdoHPbvLkb5qASO+eNrAvrBSM78PDxC/FVqOPW3GDL8GWF6+EJP3tkz+nj2d2WI+1YdLvHREUb75UXO9X6vQvXsuub6kP3u+0M/MvgSXDr5YOcc8JHCRvuegHrtB0Ui9GDUQvlmg8rxgRHo+GwSDvdFfxzwPJKS9nt9kvmEAq74defK9v4qLPtDdNr4ho568OZyyPqf+LL1Ju6a9VDY+vgCG9bzEcR87MMTdvNajgb6JWI89jzQCPnlXlr6N6g2+7KWsPUWAAj6fAxu+b7UCPY55ob3DWDU+Oua1vcgyA72ujCw+efwYv7lOzrvh26E8SKCLPb1GqT0orKk77RgOuvtgCz2IxY09hzDLPUgW8T1Cn7S9q0A8vqQKhTzlrRo+ng4VvhvwOr6Bu5a9psEqvYO6Pj9QZly+kfAmPob1vT0TVRe+wggsPCW+rrx5RBm+KzaGPaxG1z1g3Mk95kHhu40YUT35KiM+GvyBvRMMg77x0Aq+ip+pPTGKyT2RI069XGvWPdmMi76A+B49iVWOvHKyC77MSQK+xjIUPm57Q761+Sm+feOmvXywg710gry8j3/bvUVe3r0Fef280wgmP3qLAT5QGAI9","v1a0PZBjHL6acYI+lv5ivR6daj4ClxI+pyG+vZii+7YILyM+uG0Nvr+jnj55Wfm9+MDkvOIHlD6ABwA+LQlgP3ErBr3ww/c9bcKXvUB1AL8p5Uk8qvsGvhOycD6oBfo9vmv5PPZ9xrw16bk+XjjPvf4Gfb2uFpi9uNnFPkVBHr4r3VY8wmVOPtVQar24pwe93cSFO3dqGj7EFwi+ELL7PZ4dHb1JMoE9SvANPhdWnz2nzU29QUopPRKyqz6jJ2U+WyCmPktCiz143so8seaWPi+XIb23ikc+SCUKPoVmnz684nQ++h/XvWJ80b1N6uW9LswyPM+dvT2a+cm95I6qPLyiVb4XBjU9tKaKvPgmiju4Tf29QAmtvcQeRT5WZ1M7Nl5vPnq/Gr5U2L87M1vMvWZAYD0UrQ4+BW7GPDN74T7xyhw+tQQJPoYKnb30gJS8vlc+PdX2Nz1+wSQ9es3wPkRufz0rmrw9v+nYu6kX1bxkQJ88KgMWPow6pz1HK4c95Ql2vQ8eor2XWzI+C6WTO8/0KD6Rd5k9JFlbPvZ5mz3yA/A9iZ4VPpZh6D3r/oa+APAavl+iI76vuaA+Mce1vXpijT5pP0k+o1EQvphAf75JuBu7xEeUPfIGw70c9xC/j2RxvovZSz2t0Om9fdgrvbhvP76kL5c+CHqsvK3dxTzbrr89rh2APeGl0L3o8uo+sNIEvVMSsb6Qq/o9zUmiPo4On7zYDYQ9fsWXPs2GIz4y//u8g/WEvZrreT+TTl+9UO1gPos2GrxFx9c+ocxNvE/ti73lOC6/TZpXvnOB1T0Ca12/1vOwvaAkl71tgiC+Lm0/vPH6bD+wsRC+QL48vUDYeD6GrUm+zwgQPMRKL76WvAO96zJgPoEuPT6bRCS8DaJOOzQxsL5P3Bk+OdM0PZdmLD98fXs++n83Pd2mHz7usHI9RJ/mPNV6uz6Vh8I92sgSvLrIeDzhwLC9SL85Pq5vLLzVLLS9EiwvPCj0rb7PDp08ESSQPqRTZ71aT7C9","Fwg5PVYiWb5qSH6+0u8SvlzlIr5FMtS9puKfvlgN8T6teg2+BlYdvmmTwr0FUi6+bx3cvb1VLb79kiy8dLiBvXr1bb5+lAK+ESUgPjtVQj77dOu9yoOYvrpX8rzn4jG+kYsdPA4IcLtYLMG++7ByvYTKuLxbSIa+Q9hzvujSEL447h6+JeJUvv5CeT1ZK5G9epSQvE4RjL1XNag8hCBovprjjb3SDu892JmyvTaPrr1oWws+9MF0vf8Sb76kzgC9d8qivgydE75z1NI8KdhCvpSedr7Cnka9MNYMvvmTer2lyT89rWKivmEcdj0l6z28B9ffvZvsEb7P2z69scoXviQfuT3fMZI9sXHhvKqbYz2PDF89YeshvtbLFz4ihYk9kum5vZP+4D3vaou9JNG0Pnsis7yHQR++NssDvsp6fb2qtQe9s9MYPd9uDL5/QA2/skV7vMCSRD587QC9tcx5vYiE5L3dCn0+HHKlvU6cGb6DxtK+4sJKPlMm1b4tEvy8iXVUPime2rt9/LA+JLldPTgpkzu+zQq7v7lcPeR0Bb79SsI69LmZvQCNPL4S2C2+4TpWvYBqCzwxgoe+E5tmPszWj7tQ3RA9eqyOvpbuX77mMT2+tNRLPaaGzz2wJre9dD8BPefkKz7Aej4+QhqnPGLpdr0+9co91ZOKvrFftr1Iq9Y8zWBiPmsc4r1vFXo8q/kEPndLnTzpJWI6pwsuv8Uwdz3gDuw9exvtPHbTUz2KxSw+hED4vR0xoDxnmdQ9AJaEu2Bj6T1vq6O9Lgxgvlasaj5A+Fo+9ZFUuhYLCT43HyK+1oC+Pv2olD3ijII8HZ2ZviQLoL4D3M0+hO3NPeDImL6fzUW+azTavqL0ib0O4Ri+5k6OPk7LAz78wCI+uz4NvnOeMb0VqSy+9RwIPm/TgLwYGiS9fWZJvhiqi762+BM9P6HSPWHWcz1o2hw8okmsvTZ6oLxEgq29ET+9vXLVFT0ljRo+z5z1PB/n/DyU4Bo9J8SEPF+Inr0JZD4+","/wPhvQlg1T2Yacy+oEHbvvObQb5p4RS+M9MHPaGqmbz4W/e9F8pLvf6MoT5ek4M9LTWBveI3pL6fBVW+wcBuvijI0z31Tu08kimrPXHPYL5cCj6+2VaauhDJRT2IG+Q5OpoVv1E3JD2h85k9FP+0vaxNa74ix+294VglPnJW3jzMYoO+7fQ3vkin7T31Bq+9LUT4vfBtm764XEO+juZGvumaPr5CWP69BV0lPXJsK71mA0u+mWJ1vj1M9L3L3Hi+gwtXvgihoL3ACsy9W6uVvsvsn77o3Z89Un76Pehe971Voze+YXKJvoug+Dw3M929GYx/vox8ADy9GAC+oekfvw=="],"bias":["UBmWPMzjxT2Eq4k+Gy4NPs2uMD7Emhw+r0UGPjNnNz6NegQ+KV+FPemKlzrbNtM9/HQDPrEyFjsuXOw96ioVPsy94z0w7ZM98vnTPKux4D2rIkc+rRhWPqqX2z0TxmQ+c6EWPVt2Dz7U+wM+Tnc9PsjRYz2Iupo9JqGHPm+bvz30m5495uTVPcjCwD1w5iA9hd20PeMnBz70MRc9x6lKPhDRDD71xYA9GlBKPkyNUT0DLZo7cAaLPWWMRT5OCtQ9ANsLPoN/2D0s/aI8q6sEPq4qQD1Ch0o+ssc8PbxoBz0vq7A8BJFePqWU+Ds24zs907WbPcESeD3ko1o97rOTPeGRfT/0iYQ/p5GEP5HwhT8ojHw/YXyCPx8HeT9CxHc/GSuHP8YLgz8BqIU/admBP3YPhT/qWYc/VIR5P58dbT+p2oE/0AuDP7SYgj9STYE/K0B3P2RgfT8ksoQ/QzOEP6xGgj8N33Y/vBR0P4+yaD/Xe4s/jcJ+P5odfj+nM44/1U52P+O7dT+CUIM/1UqEPx5XbD/tYnI/YKN8PwORij+9YYE/IOyEP9g3hz+ItoA/KaqJP+Mchj+Sn3E/l0WIP0zCfD9r1Xg/T5uEPyFdfj/q14M/xi+EP/zybT+qlHg/8ISGP3pqaz9tA4U/055+P94Fbz/GAG8/Yx2KP0QJiD8+A1A9logfvQzjCTzI/ZQ8+SZJvfROrDzlvCM92znePE9OXzwqT/C8pPiKPLKsNr2A/lC8VN6auz/JyTwqums8oEfouxYmDzxDl1W9M5QJvQcE4rzk3GC9GfwPPdBz0Ly+KXM8133Bu0cFEr12Kma8qE1XO4+/GT1DPSS7OtIZPDTL3Lxf1ZA8h3Z1O2HiejzZNz88GE5Mu3olyDmAHHI98VMbvPHrBb2/PJk7xtpJPLlU2LxKIuo8XA6CvFPsi7wDhs48GKXSPDFqR7stDDO8nIEQva2WDTrb0Hc8zBgVPYzaJjpn3gC916M6vOhtVr2nS968Jm44PL/dA70NYF+8","+4xtPPLimLtgsyw98o+gPfge0j3Wbkg8uuTxPXy1xz1A7dc9j6+DPcvfJj3EdjI9sQoUPnUOBz0lXyQ+kUDAPug13D1Qe548YXKdPJW7Oj34ehU9qLmPPXSEoj3kw7M9idyZPdpHxj1OyjM9Pg1EPkgK5T1klz09GFHIPNKKSD7PHAE+2tUOPZm2DT6gTIs8hNTkPZvQFj7x5Zc9WmoKPGNpuj3yg4w93z+HPRTagz2jpP26nXx0u7Cf3j1J7F8+GUtGPuGdQz7ho/w9Lc2YPVOuhz3V2bY97RrDPJUQdj2+r4k9ptqEPnpBpj1ArlM9p3D+PaW/jz37h7E94gJQPg=="]},"dense_1":{"weights":["suG/Prxbjr6k38W+ZGVJvmZxS70lIDS+AHSZvYYmgb45g9C+CPofvvPRlTwhRGO+pObOPDSekb7GwRg9NU9nvsYwC71zI1s8zH+Jvrv7sz0pI7o++zG+PRnR/j18kJO+r9V/vXtxEz2ITvy9H9SWO36N4D0lZWQ+5Rwzv5T6Ub+TNsU/zkK8P9sQbj9MOQe/MZFSPfh6uT7K0lI/DMFXP9AeeT8tVY2/aJ9lPjrO8D/q8oq/RwBAP4vcqD4Gphg+bxR+PvKuEj8G3CM/U8sUP2/9HD8QTxs/IAKsPidOBz9rFcg+D4lBPwCap74iomY+W+iZPvb75j0wJG8+aXcoP/TxQL28xog9tnlevl5E8r5iqIS/VtfXv09mi7+SUre/kl3Sv+HO27/kl7e+t/XwvpNMJD9yp6Q+aosKPwZVvj4lsoE+PYaqvLRJsj4F9Cc+324hvRCh3D5k1Sw/tJBFPvTdej7pfrM+WTw4P0FBxb+vRIG/Wd48v7tUGrsKLN6+cm+QvkYgcL6voi49d+TTvWnYcb4GeaC++40Dv/Pt6r6mfWa/GGsdv07F/b5VPuQ/IEMrQPQqAkC+Hks/wSJOvzgxa7/eb6w/R0PdP6NE/j5CI4Y/O63kPliqwL6Rb6S+n14HvnIwBb9wFBe/8a8PPmwjaj453UU+Hdr8PaVd0L4yaSC+fTPVPrHvRL9GmQs/KXtSPug3CD/oyVA/MLdFP+fNPL+iWQO/302Fv3Qs1z6emd89BhUgv9t3P7/0hBW/2/cavd4bbj2Z+Am+Dr06v76hwb+ueWW/kvGuvp+vT779hLC+Y0fwvfqSaD+Py3g9i1qJP8at2L9M4mS+VRYIv2ach78XTlY/VQ9Uv8Y16b/XLQTA+B6Nv+5YOr/EQb+/zpAEvfctCD9ANwO+qtnkPqu2tz4TdnA/c48KvlbK9D3Vb4A+WmPGPapKBL4X9ew9sAKMPm4R/j6EzbA+PUFjP8FURD8YEzW/Jt4jv5CQS79rkgy/cIjMvsdf/74BzKW/","hUisPk5+Mr6Xl5O+HGAKvRjhOr0XFVq+XCB8vkXqw73kuMm+PSOYvc0h1z3incO+nhwYPbPjUT5QAea8qboqvzSvNb78ADi+K1nHvpvbfb699Gq95xWcvrdnS7+wS4E9uM1KvyCDFL44t6I/ztUnv2Raiz+AmeU/W0TKPjZ6Wz5nDM8+d6SMPqdDmT4uwx8/THAtPzXxlL+bbL4+jEV8v2qtBz87ROw9MhmVPsauJL9Lsty8fTBtP/efhb4MPc6+F0MJP58UOT+Hn4c/JF8PPSjPYz9OfRY/YeXHPmcjhz7kc/O9C0LFPXgMVL7jjMe+B4BYPqUuAr7c3OA9fOnOPzO9Az9KOgS80Q6EvuYfdzqe4y0+VTlGvmHunr63Co+9N9jIvprijb6wkFm993zkvfjt7LzYoYA+UHBZvtk9D78ZlUQ+1GoyPtSkAz50J06+5OMQP3lFMb4YTZO+PbLTvUn4hL7iis08Sd55uxdMSr6uHpw+bBrFvr1ALj5zlgm/ZHogv77+E790QaO+ra3Qv3MeY0DACOe/8g+Tv89q5j/bqiw/DOEfP22ITD6gPRE/wx6EPyFnVz9CrA0/rjSHPmor7j7a+TE/hHsFP+cYJD/9JnM/2TYFQG3ZPb9KQue+7z5vvgAGB72E4Ka+84UEv4x7brovvNW+Yh18vis3Lj7YFx4+Okv8P/ussD+VN72+lD9IvlWfI7+NyAS/Ey7GvrbZLb5FKDW+KC4PP0mxqzxoU+2+5yZHv+DU9L6fahA/gFtLvjVcmTwMrQJAnSEUQMG0KkBknIJAxZgAP+CblD6sBSA/ys68PpigwbxZmz2/WbMKvi92TD4cKIK/lA7LvopmyT8VsTE/evWpP6lDYD6FZRa/BG3qPgEJDT/EG1s/Cj3cP23il79GKIG/eUhNPyoLvT+h9so/iIvoPneqpz5AsNM+3tuKPiQ+6T6J7QA/SYXxPlqxQ74HVxQ/0H4NP5Akpj5IPha/PCGjvonLqj75iIW+C2W5vo3bxL6e4hk/","tVEDPxjYVj8r9Is9Pyn2PXoiKL7xQDC/QPk6Pig6Lb8jqpo/HS1nP+NOuT//t88/tsr5vvwzH79XbQK/5wdZv4ctp76ygye+DofZvz/0dT8rDMG/7ok1P6KhCT+CfGw+Ar8lP5+PFz9JLSI/jh0nP9UFDD1Tbdi/Ink/QGucs78xo30/OMOAvohVYj7HrNA/rz+pvvfxvb/SprG+0Ez5vmblM78csLy+vmeOP1WUl79Rvtq/KSX9P5Ez4T5H4I0/u4cSP40f7b8UIm8+VCkmv7Q7rD0krZA9BaYWQPnfjr51avi+2W8NQHel1r7ZUOS+Bpq9vjsBrb6E2iu/EJ/FP+cBXr5oVRK/RWn+vsbYfT/feMa8EZ9nvubrxjxn2kQ/oQE9P7wVUL8iTxM+iXiwPpLnkj6RAVc+J+UfPiMtHD5FlxQ+WogSvmkJCj9Dhy8/8Fufv4IZHj/LSMi+OkC+vuD3EL9uDoW/on5AvzhI+b9FwGE/pc+uv1OeCsDxIia9vTqJv1UOqz+NklG+SJ2UP6ttQr9oAm2/eDxPv16nPz5OHMy/GKwiPgOnSj/c6ArA0lcIP6L2Ij8XPLq/WjZdP0u7oT6uHSTATF5BP+ZrDb0Bjow/UeoYP/p1nb+xY1W+jNs4vrAkGL4RuzO/DdSfvW0cgL+vFgO/WuIVv3NYzT/Hwv07GNZ6PmpB0T4Mz+M+wMRWPgUWDD9ZRaS/27VYvYLBU74kfgi+yYslPg97jj5CVwq/fGK7vqzrLb+OyMw+tyk4P6CvTb/0eoA/te6Ov1E8EUC+X72+kObvvfJF7r5xqy++s0HQvJMTBb96Gi4/JOc4P8N9rj6xwQo+L126Pm/lPz6Mm7Y+ZLPZPkGBUj44gyQ+Ey3XPu3efD54Yxs/fMtSP1icUj/hNWA/szCwv/0vVb9GA+m/Re5DvonUw74gE+W9yesuvjRh17w/+iS8tWR1Pr/6cj2g7AI9ekecvgf6h77tyxe+HcyMvsv+w752fd2+JVULv5lDzL5p/WG+","fgp7vvGOQz7uOIi9BvrZvlRRQz40TVo+NNxlvfvAG78vN0M/ul+xPuou5T1szMc97dD6PvC6zz3F2wg+aiQqvgGflD3pHaA+bJmCvgUdpT4+lQg/CMK9PvqkFz+ndb49lMorP6JHDz8sla2/nkJTP1nphL+lRAbAKXO1vnKg1r1xpja/LG2cvcrYNb7V2Wq95RIJvtHle74gE+2+2/vuvhXhOrwuqTm9LgeevhPVzb6JLO+9db3KPY2r5b55K4C+GOKOPLyMgb5ed8O+o7Bpv4IQlb9yQ5G+m9ABPxwsGj5rFY8+VHUYPx0v370wMpe+c6mAvyg8CL9YNdi+fi3kvjg7nz3ynuw7Yo3pPlMh/70KaoQ9j+bSPiDTnj4dv4W+ow4QP3g5Uj+lOTu/Hm88Pphmo74PLwS/1ZfovsVX976AaLG+jilUvZiD3b7j3LG9igvyPUtMxr6QPRm/rD/4vkpj5L4FPYI9Y4GlP+LIQr5dm2c/DzHTvUFdvr/ataY/zHqRv2ihKj+S5To/j16xPsXLHT+8V/4+h4GKvGfxRD+Xu6k+ziYwPp+Z1zy14XK9ddOnPvee5z7+8a89JED3PYVlCD5z5v0+VvJPP8Fh+z4DO4I/f7BtPvSvGr4Ur52+N4H0vkTOqr4DVk09D84LP4YcHEBnY4g/nGSpP9gIgD+Uo3m9b6x5PjsEyT6KB10+dFgyPqYabT7V2VY9XXyjPqPF0T5y/bc+u/lUPlBwoT7VSPk8F62XPTecgT6oBMQ+EUiHvbkxgD5xy5k/U6eTP4+PnD9W1xs/LJ4yv9jyUb89ASm/0mscv326Vr62WMK+22D7PuNRG786mza+4buDPEFF4DxWSKc9n54EvbQz7r0sygG/ZJPFvq0CUr5noem+YxE7vSrdgj3MQmA+AptTv5V10rxis7S9oFmmvm0ZIj7lVwm+e6ZFvqPuor5rrba+BAj0vvLghr5FJKy+GXDWvsfVn77jIk++uovmvvQWGL8wX0g+VLsxvfAn/T7+jdY/","S1mkvmaEVT5Xds4+bEo5PV4aFr9kJpm98hXjPnp7VT90f6K/OY9Jvq+wxb4q6qu/+BoTvXDgdr7wtRA+/g8NP0HJUD40gnk+Rb4APrq6oT6f0I++ecWnPlyVUTxhx5C9XgKDPmnhnj0uOIg+CKydPo678z9ONwVA+hqIvyCoW7+hu5y/kG6Mv8o9or/jTBO/DL0lv8kGIL/MGO6+IT8zvxfPlTxLzLq9avhavp0Eqb6nz0a/+uKZviUu7r6xwyC9l2NRvjRFEb/qTL6/5bwlv3n8YT9ffaC/eW+1vzQ0BD9PKhY/tW1+v51ykL/psNq7C1k8P/aQiD8VkmU/QW4xP+kcWL6QcxY/9NEUvxjJ3z5QsUU97iCGPam8Mj2a+hQ/Lo8aP7nlt77Lyhy/0tjtOhFPJj9onDK97pA7vkt/uL/v3Te+4GeFvytqY75cCpe/hD58PzOHA8CyGoW9lKZKPK+OPb3Joh8+NLSePthzOj9lfra+UaiTvUiaIT537+g+Di8nPw9/uT4qGkk/DuihP0hdoz9eh7w/MXrUv6FmWL9E8D2/q99wv3VHar8aNQ6/7J2zvunBnr508ou/2A8Zv/2eGL+//2O/5aMoQAKEv74749M+Y6Giv0tF1z8IgKe+ewkGvxcmez9RcPu+ANIhv+/bQD4K/q4+7beOPq05S7/98rS+ZONkvv4HwL523I0+BZZUvxrCMr/RoD0/EjnQPkO1wD6EGeQ+zvHbPsLs6j4PqjU+C+1WPuZXMT7kkw4+EYWtPkxEXj7jmfG8H87dvdyYmL2ecSQ+OdMdu2C3br3ETa09NMo/PqN3kD6xMVg+VeUNP7zsDj9IoUw/SuIIP3sxiD51Zxg/R+gRP9dzK8AUcQrAfLDvv11srL90Qfi/vv+QvibU4L7WhEY9GLjsvzYHCcCJdAvAwlA5P+n1CT/iNAE/LwiJv8RHIz9BkOS+vKDIP1PjG8CY67U/Riwcvznxyb5Q5GG/OlCUvXuWx74I24E91DyZvpeZRT6TYYk/","xwvrvVGkmb6tCYq+E+F+PVKa0r4sKxC/BB8zP0AKpb9xrXE/39hmv0oKvT6P2wQ/u3oIP9rsZD7bo6U+lmxKvoWKUb4rnQM/CeMbPDQgTj5wVDG9GRfEPhNqbj7m01M+iQZsvVHaAz6tP6U+PX6qPjLPlD85o6o+yRTYv9pMKEDM/W4/Nw6yv6t5lT/XLCi/p6qYv14+UL/Nt82/UA3CP0GhKr/En2i+NNBTvlqgwz/ci76/jnV2vzhVTT90TIe+UVe+vpZYrT/b+jU//y8wPlEsnD+fs+s+o4/ivpZYBL7bF2S+S2v8vj6usL4fFKc7zs7LvoPFNL7r022+LPsWvlzNRT3cyZ2/raNhPrQaFj+DsAc/BYHDPhZfpj78AAA+32aUPSV9jj6ZWjg+CIYZPjEzpz4JdX+9td5SPjGOiT32JHq8XiVuPqDhyT71GAE/V3kCvlYnxT3tqVE+81IEPxGNzT5uDkI+iZpfP3nWlr97pDHA3qzkv4Uec79cv8G+NDdMv2QN5T6u5UO+W4ldv794j70JUn4/KF+UPuqfuj5kj5++C1IVPsMslz4HHxa/wyd1PyqMjr/LZZW+UTvfvLV8Vb425Z++Uo8Vv6BMdr4EonC/XQ4ZPzCaM7//cwk/Pa+gvmeSg78e4/G+WqgtPyxc0LzdhYK9SPmOPXUCI70E3yi+O3agPqC9tL5DJhw+haXLO1i9ZD57Xm8+ftbfPrK6Ij9mZmA+i3VPvpomFz7wrTk/ZkslPolX8T4jEQTAdzqFvvil0b/qWom/UOKYv9Ej/7/P7ia/54sFv+DVOr+FOe+9FonOv3UGSMAwv0DAwe/mPwiW+L/BmCc+AlYtPnc/uL/OngY/b/JNPsTC3j7mce++m1zpPj4Jtb91xL+/gWVMPmVjSD82zkk/Olk0wE4yEsD4cAzA1n0ov0KWib9LpfW/WF+7vq50ab9luvg9E3awP7Rb/7+SD30+tOKVvdBBQT9fMYk+CIlmPy5JM77H+z8/yEsfv2egAr83d0+/","+ZRLPtYIUr8RGoe+tMtmP5YrZT+psBO/FB43P5Aauj67h58+rG+mP6xFjD46/hc/SNt/v92gJr97KNu+jT46v5DVNL88R4W+mgxDv/+WEr5CxvS+Cj69PpKz8L7D374+zwoUv8uwTz/4FYM/lF9aPyG91L8L+x9Aws++vuCssb6LlIA/HFaovntaSL+Az+K+rZGePxRgkz7fkpo/KchNPwgfNT+GUka/emwQvxf5FUBrDNy/79nRv9TSgT/KHwo/bESuP7eCnz/ggtw+LnA5PyN6bD/gKws9uTVBv4eTiL0Qj+q+c42UP0ZkEb/Xi0A+ILuzvaLL0T7BGl4+3640vo4/qD5dBfq+jTXjvtsNxD9k8uO96gYUQM/dJr9meiI/9nomv5P8xr5hblO+tAC6vjIMCkDOlIg/j4J0vD6sWT+M/yU/d7/SP0q8Er/KPZe9i0ZOvpIxiD95J6u+nxSHvnjmvr6iW7Q9xr3IvoPI8r3kDBm/jRVjv64qJkBUuvk/zkP6voBTTcDGnje/PimBP12uSz/UfGo/xg4Jv3WLar9/h6o+5svBPSJNkT7rWXU/DhJdP0akkD+BGaO/7xPFP5EUkL+Lwa8/QUFjv8gY/75hAPa+iaPXPvnAlb4Qmti+li5Zvn6heb/DUV4/f6KaPsbQbbykFii/bI/yPfq+/j9TQNy9ef6RvrBXMT/wxTe/PclMP/3SM77ZQPS+wVvYPoMAET6VTXo+hKmIPggIIz9NUb2+HbsrP03BLb/iIi++SiK9PviHHT7cFs2+4kMHP9P1pj5awsc89IIEPoPEWj7eYVM9XLoVvihKe7xeWi8+cA2APghbwj5vf2A/YtQFv9VmAsAGulU/xNd/PpzLYD46kBY/52U/P6Tggr8cED+/Wj0kP57onz7m7YS/T6ONvoauYT5tuvS+SL8nv7R/Yr+62CK/QtIaQCc3Ir9KWei+oKlzPyXAtj8Pm5m++4oZvVpqX77KpQW+DxQWv6w82L4IK4W+L6O9PjsZV75Zz/e+","op2GPvtOFr9gwxu/bdCrPnErhT05MS4/DvmuPkB8jD5mkDM9BnbMPpc9bT4uPhc/UyQbP80Z6j7kcVU/bThzP2sQGT/9Vjk+9HePv6aJZb/lerO/su88vwg7q76XHAS/cgkOv1fXIb7Nl4NAt7F7QImCrr89QwxAfkFCQN2WEz/bOIg/nmf1Pvjo3j/gasW/ml+hvy7+jD8FPMk/I/6mP3WpQr4VcyW/4U0OvxA3MkCpQhBAwcsrPzounr5P5zy/CKGcvkr2kr5c5jxAi7WSPxr957//TuK+AyAbP03zRTxUquI+BcwNP4HKqL1R9l8+VrtfvqOd370Cl6i+4g7nv6h7h731dSO9E+tXvvU7JT7RJ5k+iyyBPqU0Qr5L6bA+A3kEv64JAb9n1pW+XY+NvtlEhj7lNO4+fVrSvv/1zD7VCUW+VL15P2x5Dj/Al5e+5f5jvwZsFL8xh8Y8fZaXPRAbCj6vKwU/qPzAPl/qAT7Y1qo+MjIDv5gAIT5nfTq+BrhbvjT9p75Y5a29v5uwvtw2i70SgC++wzP8vQFVrr5Txsa+KeMmv8Hzh74dRmm/ORRxP9BA7z1CeoC/vaHYPm9bXz+eYhY/mwaXPrHONj/DhjY/EF5GP9fuPD67ORO+EsibO8Agwz4z1aI+HLkSPxrxFD+xmOY+xyUZP9Cl0r5tc4G9OzGAPsdD1T4K3gY+pst1PcKk1z1VXFu9yt4QvdKzmT4qEJG9vKgOPtbZxj2DSrg8UAsrPiI8gj4EBB0+oj0tPjTH4jzkmsY8AftLPksnlr5iHdE9naXavGxffD5ih7E9WxEnPftRG72rBly+riGNPnoQRT7vYMM+5NUvP6T0YT9+EuM/qbzqv6oo6b9nUC9A/gk2wITMEcCJn6M/CVscv07YO78bPNe+3GPfvpNzEL+dw4e+WKkxPEYEY74ttBg9y7KTvq70HDw1dkS+ZsYovtlWo75lyYY8VU9SvrbamL6tKOi9Xa2ovq740r0JnR2/XD8nv/xpCL9l1sG+","ClSEvsny0j4Y/AQ/IQnfv2kWYb94roa/TFyLPupGBr82rwo/Ss8sPs+p170/FLI+5uOxPuO91b/LF1E+M98cv/Fcqr78lMW+CApFv0wBVL+lxQU+BakVv3Ertj7aOAI/1dvEPusGdT7q2go/uLcyP1+T5T7eqjjAyNDuvwPfOsCN+l6/PHErwP3KH8DZY7U/JyStPstbir1qSYO+1g/NvknXPT92bpi//6seP15wyb4+8TjA2VgXwPD82T7W0Q8/b9wCP2C/JD8S1xM/vLKqvzEsJz/DzZm/W/jaPnphRr9VihA9kXrJvxeXi7/fHSG/C/iJvKpl1L9qTlM/WvM4v2p9LL8+Knq/T+HQvtyIyr78FaU+jf8DP2if+b7lI+o+H9kBv7cJPj+Pa7O/HXDQvwDy8j4dTKM+kUb5Pmt7ST2HuCQ++vCCvtu0hD+8nps/Hu3EP4/I/D7SWii/RFc5vxyFWr+LMzi/m+m/vs3PtL4peJg+vFxtv4XIyr9tMMk/aH0Tv4MgiL/Yxha/cL37vkgPLL5siUq/UVBkvyh4/r43n1W/auIyv6FjsD5ebOe9fCaOv20fFj+p+YI/40akv+8ePD/os7S/B8sJP51elb0Cp6I9lQG5PayAqL4fwpI+fhdmPmO8dz4MXiQ+RlylPoM08j4GkOw+yWLIPuB54r1E3Yy+rolxPpFGzj4NfB4+IKtsPjULtr3E25I9vIq0PurH2T6BW9o+1/2bvNDrxT4oG3U+fErPPkkVAz9aYOg+ysy3Ps7boD0nSTU/EHmfvv/GxL+kOou+UYxIv5vi2z5nc0C/LaEJP1kwRL8Aduu+kSNJv7mxSr+EM5K/AOCRv2fT0758ppq+ia4Ov/ekiD8ne6I/LlfDPl7UQb+ZAVU9hIejPamikr9uXrk/b+qnP5QUmT/a076/9nElv3auHb+fGRC/vnM7v09RhL8rOCG/Dve8v8TBsr+tp3c/Q0MyPzw7Hj9u2RrAztgDPxU/BT+JqKQ8oIdGvqAZ071hZPU+","dxFdPRiOWT5D/uQ+4GiOvpKoA755Srs++xrhPseplT48zYI8OC3oPuds1T39wyg+ibUAP81wuL3CHZo/08BQP5myPz8ucJQ+pC9ovzbdZ7/wjHG/9L8Vv5QVXL3CtJe+T7dYvvzJKL4MdbC9kzxsvktDyD56MNK+WhyUvmgylLyQu7y97beAvpWcwr3E8Mm+GCGEvt5N8L7RmdS+n1gtv5mJcb5V+ze+SpYSPvqXsb641We+pd7gPcmqKb6zn+K94oI1vnRqfb5E2Rq/u0DevhyUJ7/4oQK/zReEP1z4PL8gRom+UWUCP5sZfT6XacS/EJNGPy9SgT80zoo/CHDLPoAPYD7BtRa+BOiPP4j49755yQi+Dlsqv9EfrT0IGOC+MT62P5vzmj6FauE9eln4P0bGPD9Owza/M8ybPl7zhT6qDMc+qiBVvuc+Pr9P/Q8/V4RzPmXBJj8scrC+Gt9ivUsTzb49T4q61yyPvr74175VO3K/CkfmvzIUcz+NqGI/bbyIP7KMJj8RWxE/yRe4Pq4opD4Vmss+XIAlP/kmGj+Xm3k+YmFpvSL4jD06NJg+qFIbPRm+ej44iZM+GCm3Pa+o9j3w7YI+ZvaIvZEQRj/NFR8+cVYavyC+/T2z2Ma+Ao00PxK8Iz/HVsM+Zo9HP+1OUb/uUUq/+e4svw2Mcz++NG+/ereKv5NgEj/PUQO/2Uu4Pt7rob6Bwga/yhZTPxNeRr+HxMo8NLrKP6pfsr4nLpa/1/MtPoav7b5Xbms/6BEbP1QiWD8N/hO/4usmPwLbI8B9Igg/qOiPPgrh6j6j8r4+6z+KPcmjxT628xM/KT+BP5nmKsCLMA28QseNP7bICD/6HCA+UFMfPwV2M8DUhBjAGPU9wHlYOcBtUr+/896YPz9Nyz5wGEk/AoEzwPCGSsCcRynAAbCPP8+0cb2ohxG/HVuyP4Ssy79MafK/QZTcvwbErL7CFeu+vdyWPv2f27sAk8K9i+HBPsc+CT/N6Q4/ksAsP3rNIT+Cc8g9","gbUHvuI2Yb3XuPQ9V+CEvbL1OT2lGdk90RyTPRfPRD5P+ls+omKrPqsViT7/RwS+RwRzvotAib2dPFw+D2AHP6i4ur5mi8O+WMSOPtQ2eD2etws+OxCEPh3hIT0vsrO9Jp7jvefXW72NZDI9lfkdvtruYz7m2hS9bDr5Pu/O4z6Wmos+/ID9PZqVij6oGs8/U6N7P1O22rzUwYI/exv7PrEhXL8AeF6/cHNxvxawZL/Zmru+bFc6vncePb7A5yA8n2sSvsL4r7725qm+D0SSvpIVsr5hypK+PBI8v/jAk77fp+a+X+fjPx63P78DDUq/07+2P/89b7+R7hK/vMGCP9Mr+D3RXPK7u2jHvh2o87zwml6+e7VVvnSBh74Uz9+6xVxmviDh0r78UaM+8jZzve6zt70rAUO+8jd/vTy2ob4nrlY+buv4vo2Itj4f2hm+M+WUPqnQwb21MHI/IabMv0eGfz+HB9S/T3ZsP1EBYT+4emY+zEt3vzbhPL3k/Io8JaEFvu+ygz7Wmw8+NlgHP3WAij6GVOg++izcv60RRj8Vyw0/htU3P+QuNT+/CtW+guhRv49zA8DMJa+91PXrvq9RmL9i5eK9eGIQvKcxWL590NK+B+rKvgTL57yVAls+51qGvsKZXz/Rwi8+2UxSv/FYMD7F6/i9wmjcPURCAD+Lu0M+cnWLPv7etj4ogAY/2RoMP4bUDj+bQF+/g7onv4XzBb+HFQO/qQ8VP70bT77bLo29+v1ovrivkr7y7oG+egHivihz5T/TLxu+joirvmKgjT60yhM+khzDvp6+rr5hjrK+Id2TvLQIuj+4lZo/rODJP31SiT+TrqM/6ax1v6yytL4jKr4+IJYPv5+Vqz9IjRm/6VxwvpL9qj8Cn0U+I1e4vqw8pL5977i+AfIIP49tTz+9+a69dAx2vW0kir6Klvs9eXITPz5VsTzgiSI/e1MJvyJHIj7d0De+d1tDv/H1Tb9Pmpu/PVKDP9OaXj+AiUY+ymgov+RThDzLR7w/","8g+OvjQT3z45HPM+xveAP+Q0gL8rjLw81kLYO9D8Kb97Uhy/y74dv+7iGT/+7ow//dFrv4GqV70hVUm8yVKtvoc18T7Ag6u+9loMv+XAU723PU2+F+URP7WBdb6HvUS9EImivSPGBz64TQi+/vFHvvSGE7+8Gbw+etenvufmozxugEO+knuJvlTpnL4h+xe+VjrQvq4xCL/0oQq/zyWDPo/LF79aT2i+yE3Qvgndrr8a6e+/+YaSv++ybj8xsjw/y+tAP83iWz+mx78+cHPdPppktr2nYYm8gF5OPZX9bD0n3sA8nCkkPm0+CT5PlaA+cHcGPzNkL75nwag+d+mFPnn6eT0yCQa+iOKIvk7Dgr4ekJm+0ognvsK5rr7FDu2+6cOOvnMIBL/npF2+pyEKvwhQar/gqmO/CR+pvks9/z8Bgr++PwHnvq3Y0r+RsJk/OQVuQF4smr8rnuI+f4SxPr4dWz5739g+aXvdPRSu3T6affS8ox67Pnp5Vz4ahCg+yZePPr5LXD4YwJc9XuThPtn/ST9Wj2o+36j/vlH02L5t71U/8af8PtaC6z7hAzO/fyIhvzn4yz+PjLW+cP16v8kVQr8cid6+ng2APxM8A7rBgVK/YDAVv0mn2T62W8y+Ff9Av92hcj+c/vk+rtDOvr4xCT+ndKg+VRwKP3tClT/nhxm+hJ2iPd3NTz+5H7q+pSRyvnfVAr/aZhm/aI4Svykq7L7Ehro94QMUPzf3MT/qO7I9LgfcvuW22b5f/aa++I9pPyTuAb+HICo/UslGPmsh0r9R3cQ9YID2PkeT4r419d8++n8cv+e/3D5Mj8w+Mi4MPjaVDT8Hq+A9NdJYPjnbTr2uUVY+7w7MPaUl/z20c/Y+WZoQPko8AL+FoQu/VVb7PsbesTvD2uK9agtwv6BcpD9nJVQ/okAmv0G0Rb+B6TQ/sz4UvzmYKr//71i+/TWDv+kGuz54wgA/siQnPkKtnz6ZnyA+vLyPPlvXhz6BVqS+iwC6vnTxI7+BLfQ+","zQW+PY5sn75+p0k/y/6Xvj3OkL63kmK+4hXEvlP6ibwWn7e+G220vtfylr4qrQq/xulqv/EGj74DHnA+L/yMv51SGD96BAm/NV+vP7ULej9LoSjAWws7P7tjVT6OwAQ/RjYRPqrpIT/p7Ow+a2V1Po80qz6ecxY/+Z6zPrIkzT7Jaf09TpccPsP/Tr3zrh89i6QkPsgV8z554cY+3irrPugl8L2RoiQ+wzpAPsvfFD8cTxw/W5mOPRX6eT6S5Xs+Oj+EPhs55T6LBpU/J4EBP+eBgr/DNTa+gRPIvnwVur6jef67+DvVPJ0t4r6MFKq+VYOZvpK7sr3Pn9S+Cru8vo+ziT42jJ2+ddRPvuqilL4pAqO+X+yTvpoOg7+qSxu/kQ/pvvauI77Obom+nS6FvrZXgr8oRqo/MxZxP40kkT80aoG/LPk1v2QprTyo+pQ+oPUvPzvcnj5oe5o+/opuPj8qlT6vl58+yg/ePmtiqD4OisY+s6+XPhyPIT2xOQW9Eg31Pf+IA7wt+Q0+DflLPT/xJz8t++Q+e/YyP9FS/D4JmmY9TcE6PZgElT4Opjk/Wc9TPyzm2D7fHGM/rzQ7Pz9/Sz9dpZ4/VV/cv8yoiz8sD7a+JHNovgpPH72np8o8hnyGvmsyOz5pfau+HLJGvtZOQ77JiwO/tZWEvmFNsTuQaSa+P4mJvag3V7+TEpk/m4SpP8OVyL/uIVy/poX5Pub3tD4Un4w/5OQjPGg5Bz9hpfI+G+QQP9X+GL62Zaw81fh/PlTOiTxWmpI9DgcbPmR2fD5Mvpc+tb7APvr1uj5tLdE+Qbb1PnwTqr8pWJY8QECkvhdsCT+BcpI/Yo1sv5EABz/d6Ae/5qZdvK/1Gr5NFAq/JanVvrvJyb4+kKe+IOEXPXBxUL4cOSu+zOBJvwchDL+vNQy+Gd0IPvEZhb3gqBi+ZaO6P1PCNr/axnO90xOKvSppVT5CIu08EXokPd420b7Apoi/bSAQPgwpdT95oVQ+pBVOvsffR77mGhy/","tP7FPqOxXr6lU4S+DdAEPk+7OL4IBoK+K75zvsUULj5jw3K+HdzQvTrIWb1bo4W+Zd6hvRUIbb3rnoG+pBTVvdThJL6Ff9C9BRStvpNV074afVW/ioy7PVMmRz6lq9o+BTQ4Pud55L8alYs+/r8yPMssDD+DTtC+GK8/v8/0gj+vp8S/Q4DsPsVAKD9NDQI/hE3tudewOT8TLuq+8PQRP18hl77q0TU+/H6fPYDY1j7G/zE/coxvPXaksT6+40U+qvsjPl6QOL0vnNw9d5XrPstYxj5TW5M+HewLPy4JID+sarA+Zx1rvyctaL/4ypC/L2ELvpXggb+v19c8Wm6mPtlpwT5zGzI/xMgOvxVa6r4+RhW/doUdvhd5gD+zBg49/l0kP8Z/QD9rJ4w9uTgHPzgOGz+r4S6/4JeTPCs5xr4srJa+K0ghvlubZz8Cc36+Vx1AQLV/w76k+iW/QUu+vkvIL79jkgG/ca4WvxYO4T9U4NY/AdeWP5BRuj7KtUA/XHM8P+fq+j7Wvls/8tQiPhR+UT/H24Y/MvAhv+lIrb6Ipli+2pAxP8BfKb6wjvw6UG2OP87Qib9MuYe+AhavPq2OAz6En3i+I+FJPygX2b6qZQ4/3AwYvXJ0ZT9d5Gu+LeEVPx/z+D6E9jS/Qw/8PlKN777bjK6+E643vmJwTD4Wp46+lKfWvTMC3j4dOW++rpsKvh3IjT1uDh4+PimvPhaEj702jTA/Ji49vjHwKb+0ji6/9wxSP3VPH7+/76y+uo8tv8AnyL7Zrye+l24Lvrf2gz5bkPA+chEkPjH5Wj0BLUY9tmUPPG1+jz6OuKm9tDaAvpYTF76TI14+99M7vv41bz57cR07fsDDPsUxhz6qWV4+5nIQP0EiLT/Q+b8+X9N4P27IFT+yLjI+tbcEwE7iyr/j4hlA8dDVvrgH6r719Be/sqaivtSCWr7/fci+cAgyvPWU2b2+Pzm+WKGQvmFxUr4vAAK/gl6BvsuXxb5a/gS/7ncAvwVxn74Y9yy/","D1KYvg228b7tn3o/cfm2Peo0+T7CfeO+fcPtv21bIb+3oIa/Mfmbv5lCsb6ivFC/Wdc9vL/5ij5XgbU+WiYtP20MGj5cJ5k9YF1/PlW4PT4gZm8+jKJ8PqBrLTyuM6S9cNGEPuFRCz7PmFC8aB8HvjfF+b7zgAQ9bQu2PFznH75y4ny+BtGcu7D1Lr7VTM28BxqNvnTtHj6CQ8Q+n+IAPvRahT2qvQW9r7lDPVSGlT7cTKU+OYYyvcqVwT5QBl4+elzWPHjAiD6JVJg/ZHUJP2gy+78RD2NAK94VvzfkEr52r169hzlBvTChob7dStu+KQk6v6srM7/5BMy+c/k1vxtWyz74y2a/g11IP4J3rb0cpEc/f+83vniM275Pt5w/bQODPwNyv762BAC+bolpP0IEIj+MLwm9lttfvoU7uD8wDA2+fdK3PwP5375jn3y+RxvrvqVXRT8Lemi9Vqa+vUXjH75/oBK+r4kdviE10j6KP7y+KISIP7pIO74iONU70vryuxaelr0Vlo29zoPWvHUP9z0mi748M3lrPxFOnD+dQQG+atonvh7oKT7J8Is+D0xuvq3nNL0iEg2+rFOMvlD8hb0TM4++cPmGvpH2MDqBeVW/V9a7v8Asw79Mzeo+sDsqPyBKwT6MQ2E+XwzPPovgU755xIC8k7A3voqwiT/mMr0+QGY/v9O9cj9Na7S+GBmLvh78k75XigO/yG34PxU4k75cPye/Cr2zP8jntz/Vuy4/gNA6vjulA76D4s8/sE69vF0m4D8rJb29+8MUP1Ij5buTGcw/zbI6PGtcIr4btJg9/S14vQU/Lr5fNgE/pnxFvhPUez82c5G/MjYpvw3g+r7eQEe/9CdFQKwc1j9fktI/Jg0JP80dwz5fspo/Go1lP8b6vD90vw8/3jn4P/xUI0Djj3I/dACrP2hnyT9AIBZAATXVP2v1cT94xhU/P8F5Pxr6Mj+1ymO+kVerPnMCB79JesG9CwcTvx9dMLxJ8Ce/HlZVPlKfCz9AeuW9","9bX6PMw5wr6JDKi+kQIbv8erej7gAEk+8HpCv2M+zD9yo3M/sjiBvtNCUj+mHqW+elKFPw5frDzEzXe+xTdJQCxGlb41iStAeKYhvzL/8T5dCba+PHFdP8xkU748a0++c4uEvB1Pp74eNqW+2OYTvzPfzT2JH7q9LK61Pna4t74Rvq28zU6Evw8OxL0+9Sq9FwWKPnNm9D467vm+9VKjvgrKo7/LlLQ/7VVcv23Upb9EipI/60qWPqOtCz/MZ5M+2N1nPuRqPT4luYE/lEunv0p1HD/IBmA+WxtpPrCH/z4UB28+v0aRP0p0LL5hNqO+/84Nv0l5Er90Wg6//dTUvhuY0L5zopE+Zw9XPgD/Nz62Ugo+ZCDUPkEILD9kwQM/oJYfP6od9T42AtI+4kYjPzDkab81xkW/S+dAv5w5zL4IMV6/lwc5v37ul7/9G4q/ovGkv5zeVb9YaYQ+2VNIP0WTfT6RTu8+yk0jPuCkqj7uSco9T1gdPpmYBUA+FMG+RBmJPaP5C8DgkWG9uWJbwAIAPz+nWDNAmB87vl1Iez56Ma4+HtsZP7nMpb/cqeA+magdP6Ksar9HjhY+tHCFP1G/rL8pBOo+kx4APhxQxD5iITq/7hZAPWcyCsDUW9c+R4I1Pz+KKL4KdAnAzawZwMNhEsCNaj3Amdv8v+zRLDxcwVo+nQLyP+bYjj8FmlS+iVkHv0UKGL+hTH9AD4dkQMIeHUAolDJAJC6MP1CzDUCYUkW/4LgkPUpnHL8/57A9aNikvi6B0T0H5H1A3YldQDxFVkDteDM/qeN2vb81t728C5K+PkVvvhcNir6mdES8e0IoP/c2QT75ixe+OUSOPuXy4T14WMQ9EtBtPhh3+j20sfC9PPb9PV46Qb5kspC+nkj/vaMvhT0rTfc9CRfTviqwl74rv0e+ReRvv9NTXL+B4xe/EAc9QJY/0T8qnYU/1VM7P4K5bj8RNLE+LvkXPjlKyz5tACo/AsUbP1PsQT8aMx+/kN9DPzCCBD/4UxW/","9b2zPplTc77nIby+Fbi+vlF2uL4EpMi+ZCUKP310wj9aZHW/eRYzP8XEzr7I0US/hpkJP2FMDz/9u1Q/IsEcP9kCvL82PNC+WoYMP0MP6T7hNoq/f5wfP4dsKT8fJJS+060QP94fg76H+ew9u20kP/Kp373YdnM/GUotP0hNFr9JjQw/TjPXPg9yuT9wKbY8Jr/Fvu4sVL/Xj9U/irNHP8PgrL7i9B89S6dTvaNmMb+8Dpu+YKbKPBIvnL7aWMW+NgpyvhUBrr4IT8W//DDmP9bKDkBM8gRAU7cBv1xAR77kL5+9RzfGP5b4BUD9Yt0/WLSUP01pbT+XUjw/+jkEvxv9fj6m/4y+iDHSvit1VL66XhW+4Z1ivj8eI7+MFB2/9UcHvyuMI79wlqm/Kt1DP8d6iT4cPok9seaxvm19qj8iXak/H1NfP+RL8z7s/Xc+ugk+PhPmBT/rlq088Bh0PjM1+b159bM+zQhVPly1gz6rLrc++TncPv6Y4zomVn0+14NmPAQYcD6D5A+8H1naPqV3dT5Agxk/MwPYPi+RKD+UpFM91WaivTzqjj4ajys/8rbyPnfMRLx+Vp4+gS3XPu+ugT7U6ME+/2GWP9r4Yz/F7Ya/zWySv2YwBL++Tnq+npX0Ol+pWr4ynk6++MH2vrvPcL57B9K+iHkev8ED4by4vCI/jy8nvp/VXb6AqQ6+asx4vV88aL7VK0u9XJGSvHTztr26W1u+dSs5PkTparw/fkS9h/Gbvkb07rsR5SK+I2KzPSkAyDx36J29fCN/vlZXPz7cVqW9NKqovlEMFb521em8rdo8PpsUBr9NpzE+ed9VQL2bP7/N5x5AgUWtv+irwj86Sy1AYCpSP0iuCkDfNAI/4NZ3QBgY6T/pT8c/5fdmPxrU/b5t6hu/s9qpP1KntL+1WGm/RDvCP8fFCz/Cnm++Bu0BQCVUsD9qHGU/68CJP/ClYz/E/ac/OENDP40kkz9NxlS/JSQwv82V8zyPeiw/NEIXP3wfKT/1Nzq/","ZedFPYO0dj6d9wA/yO1CP5y4Nj/fQhM/O2YCvyBS875AGAO/5dlHv6sIYz4RqBm/Yf8KvuDkkb2mAOm8bhhBvj8T374bfCk7EnauPdcLyD1IFJs+Dt7nvke1kb6B+mi+yRuSPSpuIr7Hw56+n1f9PPEwVL7/oq+9QyeBvj6n7L6wKnW9IJOsvq08d74ypK+/IVi8v0ucp7/Zqwi/6HYdv985ij+uuIQ/syqIPyaHCD/NlgA/H1+ZPknvrD4HRTM+iBd8PtUUK7567Fw/nOEhu5qdUD8I6ce+IviKP7w2or7OiQG/53TEvbfRRb5EwKI+F47PP3yhwj9oKqk/5bfEP8uuh7wmNQg/DOf0Pp0owL5Pb7e+6F2RPVGUgD573Gc+kZkJvzyFwT5Qgaq/dlDav98ldb6SBg8/d0zBvj5+kL9pLoA9cQq8vbdHFT+/JRA/vJrLPttHZz62Hfs9ThiPPg8HrD5Io2w+e9y6PkQ2ez9d+gtAr7o8wMxuH78Ea4C/UEbpvkOPab9Y2HG/1jaavr9zG79ZNZ++I/URv7yqLL/bGpS9b5r7vSpiZL00uli+uCAyPg30lr5zdbq+IYWfvqaewj0rvBs9Il8SvvKFZz10tiK/WQczv6IovL6YfzY9z6htvm8qH7/l/pQ8AjAEv6CqtD63NyA/AH8xPbPfQT9h9Wy+2egJPw5G873NtSg9bZcEv2pwC7+WbxA/iFWYPjAg1D5QScM+aA53PsPYkj4vtlq+IyjePpyLmD7dHy0/MbGVvgzNOT4nOAc9ErkyP63ONL/0rfC9BbmevjHZRL/S3pW+Ytqjvytclb9TJcC/7UfLPkTigz/393k+5Zgdu+IOsb0YMI08DcVfvk/jwLw7D5c9Nw5KvET7gD4mln09q0ZHPS2yID4p8p695SbfPS2Vhz6AEnY9Ns4KPpzPyT1dldQ+x8uNPnjwhj/Lbhw/9q2uv1NlsL96v8W+y3GIvgTLrL5CjeS+EKCXvj8sRL7IoPW+1M/rvsC/nb412CO/","AAGPvog+Wj7Aw8Q8KNKgvqWAsT42oxA/wxkZvmFJ3z7umbK+bflhP81Unb9EvXG+Rtkqv7EXgz8b8ZK/Z3MkP0urKr8umGW/QQQsv0KTEz6sRvW9PYQgPxVBgL5YYZe+GxxAvjjCNr5AJpK+MZxxPtesWL/GkPM+kFmCvqyHOz7UtGG/tLBJPuipG78NE+u9SeXTvoK+h74vli2/mq8Lv8M4mb71oJS+8gODvTsrmr5JBSG//UCWvql0W7/y4VG/UMM7v2Vnjr7nQ8E/guSPPxdIzz6Y4II+v9yyPoRjAD+XB88+cGU6Py0Qib9vi5i/dfq7v8Yuob8gE0y/tqeLvo0eP78j/nI+z6sgPXCOiT65sIE+GefJPUc9oz0KL50+B5UIP7hKdD7vLjs9cpEIP6aSqT9op42/eACzvxfLgj/F2tc94jWWPkEI2L1PJx8/31rZvyZPdz7rA/i+QTicvsw17b6wxV6+E3+gvixThL7LDjm/NTj6voNmlr/01KS/qCAyP8SU278RT/w/L1ZxP+t+Yz+Hul4+3yUIPr97AT/M7FY+Ze9LPvGsWD6Wb9y+7k2OOFQYYr1NIu09lv+Dui6htD3vOKC+xPUdvxuW276zxlu/EFmmvyCHiTyex30+lheZPtfkV72LWtY+iwVCvhGrxb7HPTy/BlU5v+TUGr6TEJ0+F2akvupTh77i++C9N7aKvqXahr6IXsa+JduLvlENC75P7Ai/T/JwviiEBr+bt4w/8X8cvwYfiD/5ApS+8MV/PocpIr5icQs/WFYsviYiDEClA9268NsOPns5Yr2fSBA+WI03vlFcDb+ndZa+Y2sRP+HyHj61ZKu++X47vgjHN77EiHc9dqVPPXkJ+b0AG568GiRsPgKglL6UgeG9WdbvvutqjD7rpr6+5A/Tv3BSoT8nZCi/q/+Cv0ELQj+kZCA/VA4iPxaWOz75lJM/YOHiPrvkVb7cLaA+SaitPgumsD5yJYE+M9p+PrHJmD4WG7I+ZS8gPxIXHT9d4ik/","pDVPPSySCb9hB8e+KCPqvo1Hn71+CKY/JTvZPjcpXz4l7wc/nVwGPeHX4b+MNUs+rXuoPptHgj4pPQQ/l2HtuwFp8r6uk60+JuiyPgRHhT7wnuY+6ETVPvBoLT91zKY+/VYzP9tuFT/+b6i/JqeUv6q3Xj9HoRG/u3QOvhYJoLwCMgi+sIiZvouJHb2dhWa9fsV6PtH2l759dSe+0JECv9k5kb3iZ5G+H688vuBhsb5pOLG+c7/JvV90s75/AJK97bmTPK7AAr5RXxG/GGZvvuV9Rb+2VtC+oP3eP7HkLL89eQa80mp0PRkULb+8x66+lVb7vgXnEb+wUhu/rPQ8QNtVMD5hYrQ+Tw+pvkepjz6UYLG+Q97tPZDNxT7fKWg8W50qPuPb6j4qwDo+GhYVPfymBD/KVpo+du2vPm06aT7obdk+XcaQPVGK7r/tXIY/4IQ+QD42pD5fNA+/5Lx1PQ4uDb/Rnmy/7SO9vpuVHL9W+/C91gJ7vY75eL/rjjq+1TuAP5+anD4+gIE/5ge6v5K4tL+rLak/mNwZP8ZRJ75s3Fw/OfoFP6Anvr6egRq/6uezPpgHuL+fBh8/SL0FP/hhQb8E+b4/UQzdPQA6VD+L9Dy/hIXbPp6ZTr6OOpG+z6ArvmSqU76Ktfa+EAuOvkqUfD7Yugs+/Jj2PYXaCL+M+8891fHyPW+eLj4QvEc9qocjv7zrM79HAfE+o1cEP5z09D3p6ok+TH5Qv1swMrxuFoS8dCqevidfYj5njF8/h0dAvq6CF79kHC290vdnv1RLiT+1JnQ/C7CLPhtMNDsH43s+P9EWPvmoBD8Q9x+/hzg8vyWUIb/Nn6S+l3w9PvNkEb6e0aa9YVRxPtz9lb4cBNK+GO4avmyht75atY++YAFvvZn1br5qnzm+Dv3mvkbQur6yHLC8L/alvnSYG76hBQ6+CuhVvoPGkr47rdm+rGeqvs839T0pquy+ItcEv8kxI76Z/4Q/Pop7PxAMnz+pFTQ/wdaMPuCokT90Q4M+","SVQ8PsJWyz6YThw+66aAPwXz+T6J5CO+3tRjvyZszz+piIK/HB2HvpUzDr+djme/wBLjvksQnb76k9m+mPqwvnVJqr75IZG+rFWkvv1ghL6eKkU8x9GbvnnGfT3X35C+0ONZvjU8Ub5UmZu+1FhcvnsTq76AXUy9QW4CvpXMgD4tGt48WT6RvZ6fxj0/4rW+4Z1rvij2kL42CAa/PC74vg22ib60aI2+SPMOviGX57/e1BS/P6k0v3dzhz8Yw4w/+w/VvyK0fD/0j1Y/Pi4jP/RofT5MgO49iOOwPuvEAT/iqcI9N/QIP+jnxr1DAFU+5A7IvpD5Gb97BdS+ujcqP/LQVTw59hi9gnq7vivf/D7j6lW/Y8Ajvxf+i7/L5qg/rVbQP0Mqfb3M+hs/xiy2PnEe+j/K/r++/gqsvy4ZfD+3poa/TjlOv6vOVz/eXts+Acw6P7U7gD4v5J8+4SfvPqDZyz67jMQ+OkPHvgorpT+U3YC/CLMGwAge5760hq8+MspHPasC670P8Vw+g+4ePnRgz70B0kI+cTi5Pv9Xtr0hz929e5n8PUNPBz7Kvc6+mkUQvtopsjyIgz6+5ydPPRGUIr3O+j0+Cr3lvTOVDj9h9By/yRAFPz0PqjsUnfI+tK3DvrhWgb/MOjY+TkZ4P9s3WT/kixA9WhA7P9looT/wfyY+5u7CPh2OkT4GRPi+delFvwm8kr5rHkk+lDwmPxpdyT6BDpW+If+APij1Lz5UTuI9fyozPV7Iez3Sxuk+GOcUPmlyAT/lKMM+FvwFPkZsxDxyyYO/eGLmvM/awz7Qth091wydPAdj0j7ZI/Q+RhjiP2nMEUBQZoa/l0Y1v8mYPr9/4pa/G03jvjTHtL4i1fC+PRpPvghRHL6xRgC/KuXsPPtvNbu0Z5w9ukYLv69tir4/MqS+baUgvgTIATz4mY6+kg0MPNDfv71gd6S+jhyjPsEkQ75ealq8iFKnvq0Zjz19E4o9+ZqdvsZikb47DrC+80gpv0koob4oCJ69","q9MBP2k8pr5TdLq+NTTjvnFAwr5chB095qAWPwQaOUBC7AU/6fUzQLGXN77Phuy+dUylPhOgREA0ZUU+HDHrPlMxj74rIC89rDiGvuhQBr/40JG+GQQ8viirXL5ojI2+7WZlvgl0qD29pDK/ZnaNvvH9VUA02qk/6pwgQFxsV74gVkK9J2FLP0GjgL5oW9Y/cgQav/X40T9MnSm+JkmivhSHND/JUBC/0GcUP9rvgr4LsVk+lxzAP1jL9D8xMVW/yV8qv+CweL8X58496MRqP6a3yT5200g/fwksP26rgr5cR7C+JI2bPp1Aqr7vilO/jt2BP5Q/UD9D1Lm9NH69PQ=="],"bias":["siTNvk7eITqHv7w9HmUNPX9ZDL1xvsU6HUtSPfx2fL0oIWQ+/9JdPcnrG75Lubs9jvv3O9hgLz0fvcC8o8yKvAtqIr5JexW+nqwdvgTHTL1CIwC/WLypPV9uA727kAq7u6nlvIVzt72jiO+8JJYNvvF+t7/DpA6/lIY8vL8lKL/3YcO+b3/fvQxvFr9R9BC+jii8vmB1ur5xD4W+5jCZvX0AUb4E0tS9PpZUvglgajxXQA+/Pa0kvjcdP7zLHCc9bbjHvOgzP77gFI++40wSvgfzXb7qag6/5dm9vtizhj0bIIu8q+jRvjVsGT7fmxg+3wauPTlEGD7gDaQ9xBA4vw=="]}},"hash":"21ee756c192cf2a0a2ba03b7aa813d750767844fa4a08270f607c5e017c88a96"} \ No newline at end of file diff --git a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_encoder.ktn.model b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_encoder.ktn.model index f7aff0c5ac..e69de29bb2 100644 --- a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_encoder.ktn.model +++ b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_encoder.ktn.model @@ -1 +0,0 @@ -{"architecture":{"class_name":"Functional","config":{"name":"model","trainable":true,"layers":[{"module":"keras.layers","class_name":"InputLayer","config":{"batch_input_shape":[null,17,17],"dtype":"float32","sparse":false,"ragged":false,"name":"input_1"},"registered_name":null,"name":"input_1","inbound_nodes":[]},{"module":"keras.layers","class_name":"Dense","config":{"name":"dense","trainable":true,"dtype":"float32","units":32,"activation":"linear","use_bias":false,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null},"registered_name":null,"build_config":{"input_shape":[null,17,17]},"name":"dense","inbound_nodes":[[["input_1",0,0,{}]]]},{"module":"keras.layers","class_name":"LSTM","config":{"name":"lstm","trainable":true,"dtype":"float32","return_sequences":true,"return_state":true,"go_backwards":false,"stateful":false,"unroll":false,"time_major":false,"units":64,"activation":"tanh","recurrent_activation":"sigmoid","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"recurrent_initializer":{"module":"keras.initializers","class_name":"Orthogonal","config":{"gain":1.0,"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"unit_forget_bias":true,"kernel_regularizer":null,"recurrent_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"recurrent_constraint":null,"bias_constraint":null,"dropout":0.0,"recurrent_dropout":0.0,"implementation":2},"registered_name":null,"build_config":{"input_shape":[null,17,32]},"name":"lstm","inbound_nodes":[[["dense",0,0,{}]]]},{"module":"keras.layers","class_name":"LSTM","config":{"name":"lstm_1","trainable":true,"dtype":"float32","return_sequences":false,"return_state":true,"go_backwards":false,"stateful":false,"unroll":false,"time_major":false,"units":64,"activation":"tanh","recurrent_activation":"sigmoid","use_bias":true,"kernel_initializer":{"module":"keras.initializers","class_name":"GlorotUniform","config":{"seed":null},"registered_name":null},"recurrent_initializer":{"module":"keras.initializers","class_name":"Orthogonal","config":{"gain":1.0,"seed":null},"registered_name":null},"bias_initializer":{"module":"keras.initializers","class_name":"Zeros","config":{},"registered_name":null},"unit_forget_bias":true,"kernel_regularizer":null,"recurrent_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"recurrent_constraint":null,"bias_constraint":null,"dropout":0.0,"recurrent_dropout":0.0,"implementation":2},"registered_name":null,"build_config":{"input_shape":[null,17,64]},"name":"lstm_1","inbound_nodes":[[["lstm",0,0,{}]]]}],"input_layers":[["input_1",0,0]],"output_layers":[["lstm",0,1],["lstm",0,2],["lstm_1",0,1],["lstm_1",0,2]]},"keras_version":"2.17.0","backend":"tensorflow"},"image_data_format":"channels_last","input_shapes":[[17,17]],"output_shapes":[[64],[64],[64],[64]],"tests":[{"inputs":[{"shape":[17,17],"values":["U8QHvwyCfD/ipXA+5TKAv30oVb60kI0/mMmfPrIpiz8YMxY/IjI5P6E3Bb8rkxq+ukhwv4Oum7/bfYQ/7YTOv1rYhr5vpAI+jOV5vvqcsz+b3Wi+t0nPv45p574KiMw/X/OVPj4MhT9b6sG/mRHzP8ipVr5/iMC/0nEcwJPg6b7xCz6/EyRvv+oLR78DF24+n5ecP4Va/D0kSm4/R+gNP92u+r5K0QS/xZK+Ps47Dz5eSn8+tDamvzCGAj7TGYu+wOgfQDR8Aj/NWIW/ykx4PflkHL/xocU/yFXEvHX9oz7eVfW/RW8xv1t5hb9U+LU+3zfUvrHRgz4j8vK+c60Lv4iTer8TDg5AE/TxvgljaD2NRwW/y7jJP81kGT9RHgLA2BHUv3ZCq74dxQW/Jz+UvpHId7/EBHM/YibkvnIB5r4/YIC8e0TFPXQE+T9DP46/Uj7ov912hD4f+bO/R+jmv+X1Ir+0RBS/QnX2P7F9HcBfRHs/yDSzvFhFcr6R1j+/0KHdP/GCPD/cY9m/oykvP7DAnj/EIG+/CKg7vlhktT5YmSm/xGJ1vb9lsj9/B7a/K4fzv8QdVL+9nQw/3Q26PZjBOb8eLns/fEBhv4zLLj9KBG2/IYnjvwbnDT8s7qe/CI9IPjJc776vC4M/LsMCPvVZob/52Io+KPFOPk86NL8aC5I/Q36XvwHS0z6RPrO/unSyv5mJV76uVYu/reu9P6excj/RXGy/mDgBP4/9jr/SCHE/0sGNvRUh3L62MqO/lUAEv/mJqz4ck8S/g037P+cCQj/0K+6+akQ9vk7foT3BEDk/HDWqP+5t9z2xM8K9ygUgv3Dmur5EG5C9mtTpvS00Fz5yLu6+a5N9P5L0jL/T4JO/hZksP0TNjz9kATa90EDLvkS4UL9XTDg/BGiAPxm2wL6KK3A/5aTPPnSnqj7GvcO+LkFbvp6BM75yKsc+CrNYv9zBuD3hCEM+D36Mv3yeNz9/XGm/EEoMv0GP6T5vHqO+V9VmvF5Clz+PvyVA","cHIBv7Nfjj8Lqu++1ovivv/0xb30Lcq+tY99vVnXn78ObgPAOb4YQLHytT9wYDpA3pnnv35vyrzVh5U+G+HUPz5eHz+CL0o/JjQLvie6eL8Uqsy+sYfAP2oJA79ynpE/czY0vm7TKT+wWJc/tfkawHPeTz8crPe/VyVEPrqtlb+6rLU+F85XP/lRi74M2cO+KagBP3IgST99FUM/Ysfivrg+Sz50Fr28QQv/vWXfhL5YF4A9A7ANwGi7E8BFjsa9yDIrvoMI/L9k/HO/py9pvt77Rr9zLcw/6W/ovjdygT9Aw+Y+oAaOv0Knsb8vMkI/4vmpvwxnRz6ABA6+badLP1Nd/73YEwQ9fgiyP5agbT4FwjC/8Ugjv9uVkT5TXrW+DQfWvykoDb5aLss+sQY8PzXvar2zl20/jHRPP8dvzr5a2BHAfschQGmwj7wYM9e+1toKP4E8wT3Dk5C+630RwGj3Jz4VKAU/TD3HPuG4xT9PYG8/b9GZPjxcmr7N6Q0/b67PPg=="]}],"outputs":[{"shape":[64],"values":["Icd8PluDSr7lsxO+Pa/zPJ7aBbxSY5q+z2MFPmGEJT++N+i+u2qlPkY1Fj8eIhs/p3P2vlL4zr6fZ6e93+plPXc6rD01fac+6QMfv8V0KD5ECRY/sJeQvkbIBj+Tijk+RPl4vqJ3fL7qfwE/04A5PqT5x76jHSY/sy/9PfSmkz3ZARo+ajyIPhOK6b2mE/G9jccMv4XYyb2VY1Q+u0JDP+GtEjyh4Ru+iNatPTs8qr5hF4Q6CxtiPuxHhz4tbAe+4D2DvcnKR78067++rfq7PRlZCD/taTi/W03mvu0Kvrr8cqi8iCDDPHWzR75h79++rhGovd+z5b6BNis/f3FLPg=="]},{"shape":[64],"values":["+4kzP0Mu47454FS+UQkGP4fIwr56+BW/ZiaPPjyEjT/gA0W/GG/6Ph9Eaz+8ClU/rsYzv8a/qb/28hG+Cf7pPeTvvz0+QmU/UqR/v/gMMz4FLDI/0D6tvklvMj+QTWY+0lHPvieZA78E+fk/eJsGPxj4Sb/xAmc/PYpEPiEYhz7MaR4/NsXnPhcXvb9mplO/vvdPvwHtGr6IJ44+aoeKP1BmGj7ncDa/OgHYPhxBU79zun4/BPezPvugqz6Klna/ZKZAvhbssL/VL86/m2aGPv+LeT8Tgoi/nBbGv4AlFLtK34m9dZnmPOQWxb5Cqw+/z3ebv6gPRL+i7sg/XiDVPg=="]},{"shape":[64],"values":["vAT/PcO2jr6/vjO+M7nkPXFWar7VSL8+q+aWPb1FRz2fTLi+DWDBveJRiz4N8nW9FJ6IvucLlr0+bKM+13XzPrQRQb56VNg+xZDRPlzG2j0xdoC+qBg1Pqbygj4clxe+LaXuO79RHb7CUIi+n3dCvu/9DD7GfLQ+ZHGUvc3Edz43JKo+a4RYPTDKgb133LI9kBsGvu2zpj2X7W48yiXhvLBxfz63+HS+2IV7vgVZij6eDxm+lsjPve1/Aj3Wufi9cB0tPs3JFz4V1cc++h+/vviKi76NyVm+d3Qbvezwrb1hnV4+Vk4UPjdBlrvaGoS++c0zvxQJCj8Dcke+ATbEPQ=="]},{"shape":[64],"values":["1EHJPhJcIL8S3w2/jjTgPvr1yb6Oz00/QV2DP8SzEz76pX+/b7CKvswkXD8px9S+mgQVv8C+S76/A5E/6ILxP4TCJ79aTmM/RGxsP3+guT46pS2/somxPgjLYj/KIxe/IEgTPExm4L/K2QW/D/Btv4SjQj8egXw/zkssv6IwBD+wkQQ/avTiPf8Ofb5FuE0+8aNvvvo6HT6smQc90ODZvXejRD+IX/y+a6EWv+v2LT97F+K+FalGvlDDcT3RNFe+up3CPnKBgz5MNzs/ivpJv68NR7/YlxG/h6bdvbwve74ISQ0/HgdRPlhLPL0hK4u/jHX6v/XJAECDivS+UhDxPQ=="]}]}],"trainable_params":{"dense":{"weights":["8j4xPlKyfz52AAO+jFaAPlA4qr5FYlM9rqJvO5mAFj64cDo+pmDoPCExcbuqHVg+LSk2PWwqA760tRE+hY83PjdVPL7EySc+s3iAPWBcHD64k5W+GoItvrdKXT6PWqM887FbvAspjj6cREi+AqtBvol0aD5I+JI+1swnvrJUUb5/OrU9MJD2OVGCGb7VpEc+TXMGvpuxwb2TXJm+2N6Evrc8Pr55U288uKymPgdOfL59SvU9Cw0jvgZnbz7UN+m9RADzvdqDoT6vPnu8Vj5fviyNNT6NRZo+8k+APiGowr1FWUs96SI6vlIYF74g9oU+Yo00vjw7bz6Pe1Y+FdEvPTmiaT6UQ/y9YySVPgXCob7gVGK+sKJWvhOMjr5eEEm+TTkzPYmtyD6IE9o770abvrEHID2jJXe+XgEVvqJ39T713iS+c+RovqkcFj5Nr6m8whu9vpB5k76aVog+FxD2vVv1j75eL7G+UZ/5PEvj6T4sCJM+ns2BPgUOWr7T2X8+VeCKviySgbvXUdY91cxuPViZkD45mH8+EV7JvQ5o1r0Wcok83rGEPh9cIb5NWpu8OxUSvuxUAr2mBrU9QQ/mvJZnir4i34E9M+NUvonI9L1s4Ag+8LYJvgUh170O5QO+BkEDvmK3d74X5a08cwjQvdvO5jxFQTu+w4xMPt0tOr7iJ1U+DBVavUvxUz4yh5U81sFKPm2Zaj5NUXS9OfEWPewBZr5Dlb8+l9Y8vrUvpb5ADjo+prqHvl9Igj4G4aK+V8GHvGe8yrzToBi+CtZ0PLTlVj6nAxE+jzM2vtFCjTyJmGU+kPmGPpGKEL5fx4U+6mn9vZDzmD5UPH090pgMvd9zEb3pg4E+w0WOvn7z3jx4Bqg96F1kvv7fgj12faO8Tm1KPt1KmL1vL2++sg9Zvo8Mj76b6LC9KovaPCbEmDz/5Mk9hGP9vWgXHD1ScRA9O/+tPnCSV75hQka+fszQPUMlSb1fwvg9OY1XPHki671PTkM+zdWqPogujL7xBoE+","WkuUvhfQs729q5e+RP7CPteqPT0FHKE+G28RvleojT5ebKg+VIWGvmZe4744xRM9mMWIvhaolb4lrse9xJJKviyFrT2TpSU+u2eAviE+oT7vXKw90gyAPv89Br1wbA6+zdAAPsEGpD7JHYG+336xPo0qCr6NLne+y62Svgfuo70VhoW+R4Z4vqhcsL1+oku+txCMPuev7b6Z+cI+lfExPl9T6z6CuyM+qBdovhRJjb44F7S+Dfbmvl8CZL6px/09oRVoPqS7Nb4GJMC+IhyOPhCvPr7scWO+B6ZIvBpzRD4ROKS8eHdCvt73i73o7Yo9kEDmvjGVAL0NeY++m56ovvNKsb7kKlG+RJm+vgyCnL00NxO+dfaavsmsZb5zphY+t4EFvlf/170t+rA9t8B/vTYWRb4kzPM9YbSMPahtkD5A0pk+hj+dvh2OpT4x4uu8lO7pvsDkkj75VcI9C/rpPsLVhL7Gd0i+7K9ZvhhsTz2gemC+N2XtvZkU57x/RLS+DV+AvjraHz7aZ5U+R22GPWO8Hb50LCW+ZRkcvvEP+T2OF9g9bX0gPTrVEL4WHXk9h4grvqspZz4RbHY+Yzr7veSpZT4mAPS+yZ5PPjQLbL6jkLy+5PS/PoXp3D7YV9g+WEWkPqtluDzeaxk8iWS0PiV+Ob5rkxu+OizePYBiP77rrJ6+SBw0PXbX6b7iy4O9zUabPh/OVT4HwOE+u4CTvkISor3tLnU+WLW2PrTbFz4Xk0a+iQd8vT9WJL7Esky+SRNqPgZ8ij5yS1Q9ix3CvgPBdj5ESiK9EXSBvhmcHz+gnta94gyoPYuAAT1Hn6C+0QQwv4GnNb/VzmE+KxYKv52nob7OEpA+rUVKPVagHT7eZU6+y1DJvlCOhb4a3+y9wIymvdB5ZrxXDP8+dx/JOxuQjD7ikv8+9S+Fvu++wD7bdXo+apc4PoxGOj65BO2+1p3GPqxGkr4YBQS+m89iPTAZYD4AzN49Rg9fvf24fbyRGu6+NFOsvr8cBT+CYuY9","JM4iu66nr72+KjU9UO/IPQCEzT2BlHK+CxgRPtW9/r057xc+NWUzvdGCvj1oobc+QnygPj2Znj0AGZc+RmCUPbDRWj6lQqu9DKwdveGH7z0qito+DZSZPhyml74J3jY+MMLrvVINkr3D9Wm96zlvvRxiTD5GgNa+mTkSPkulVL5vQwQ+SYcsvprspT1qgBM+TOSnvpxtx75mku29J50qPjxZEj45ur8+9KF4vmIOTrxP/4O+19mPvIJrSz5RQtm+zEuBvtChSr2Mpjk+wOmAvjCM1j3VAxW+ouANPaSKpDsz8OE9mlo/PiBryzyQ37k+kE+6PV90wT1AjEU7U1hkPErEhL5JX789k0eove56ez4QW2y+7LSLvtBf+T3fx0c+cycDvdXeKr6TlW6+Y9PnPmq6Gr5v08y+Fr2Uvg++Tb63SsS9Tjq4PYF1Iz6b6Xu+F2AfPohnAj3bwJI+nbeJvVlYUz6uQmW+gON6uwK+BT413kq9nI6XPu2J+b3heGA9sESWPlxNdr7mua4+Pxwgv/pBDT+QK8++FfCVPq5Y7j46XAS/Se4KvyDXZ70VqUq+MhbuPnEp3z4+2AI/axOfvjLpn74O84I9dIMBP6o7lz09FRA9b3YEPj9raT7vrl++e5cpvA7lRz4OrsC+W034vpMKFr7tJdO+QwjePsrgBD/ZMog+d3/APqlUsT7WYKG++l1GvVumab0gkvY7JwNTPntZjzteu4a+fKiNveupmT2IEdA98vFpPZxkRDzaDmi+AH0Jvc/CRD7zTKM+LgPLvhb0gT0Dssk9Q7GDvngKob3TUZi+8peRvui5YT4vz58+r4xLvesxs74jZJe9DzFHvg=="]},"lstm":{"weights":["xFKPPT5QbT6phMg8DhqVvad4Vr1HeSa+TKBvPNuNAr7Gqwc+A/V6PcIMGj2JULW7Z3ZKvffj+b1w6EC96VAsPVDx8r0khLM9efPpPTDCDz7Srhu+p5KHvE9P3zxsjx89mRx+vZKLpb0FZdO815wsvLmFAz29ShA+uWcpPXDlED5kA+49hIwAvn84+z0pai29+IwOPnfH0r1CQxa+GTZqvDQ/QT6i1zM+knASvkGsDj2BmRw+j7LQPV/0mL0i3g29CIeIPWiRATw6tDs+Sht7vefPwL1iKb+9XkIpvG5sxj05i+W9qh4BPRlliz2niN09NMVDvYlERD6ey8Y9V4BMPZqXQD7h46y9ME5zvciqtL6gPe+9vJl4PcYOj76sAQu+xXemPf8OIL6o6Li9ZkrXvf2STzq/yh0+hM5pvD4Mwz10RLk9EAxDPgvL1j1jbh+986lMPT0zGD5UOZk9dW+jPPGe57wIEi67JiV2vfapRj63OZS9IplXve2mL753Qyu9IB9qvb0+rjwPI5o9OwJJO7VF570s5MO9WE55PbzUBT4sh22+WAwqPkA1Xj3wwUw+KOjzvY9iIr1DkqQ97VY5vZof5T0dl508eL+YvZKZNz4w0hu7p+ZHvrhRAb2unYi9S00YPl1oAT7zYdc9eVGBvjHL3b3jCAM9rM3+vOKL7LxSmoA+Q218vngDqr2mitc9NdfGPbJQZj5PLtA9p5uXuzNo1rwzubW6ccTLvWPoYr2RhhS+gEj+PRbPy7wRAFe9ZtMKPaAn1z1Wkey9UDknPtDB6zu0os89Lk2bPRGf0T1p2ao98u4SvooxFL1gcNI9Hvejvpfywj3rEpG98QCqvSKjAD4gwnc+bq78PVYEXLz6kDy9+e8aPvS0Wr2IDYQ+ZR9pvmf3vr5z3Mw9dmZfvdY4570O2sM9SHGzvAJ5o71nkbO9hOA9vdwhRj28lKy9tNSVPhXAFj3sJVC+yi90PG6roTsz/ge84DNxvWCuDr6Kc/e7BE4gvow4Kr1yXpQ9","zTH1vBhxGbzhMQy+uJ9Cvi0gQr60gRe9DEWFvXVDKj5W0tw9NXsEvnAIxj16bUI8ehgavucnYj0sJuA9a35FPpLITD4mYgS+p9E6Pns+Yb4jBlI+oeLePpVdYT15rSG7/vA3PYD9o70Q4b89IbRrPenaRD3q9Zk9Lpp6PhO1D723I5y+STJIPkWrkr4HwQo+MiJyuZW6k70T9jc+Rm1XPuDlsL5htS4+3QOAvRrdbz1beai+tLwDPU0Ucj5nYh4+4/kVPpbYE76XWR4+ytcePW/foD2gD5+71cUePo57HD7tY7E9qXjtPW7FDTyC8uM9hIxKvjPq/b1EY6k9CIWGvA5VyTyuvII95inHvZmTCD4yGtu7hvSAPfcnSL3FfjE6zygsPst/4r1ZJ4O9nM0tvc908D0zdcS9GO3BvVKFKD53wAA+ZP/oPfLssr0/xBQ+nSAkvrxMtTzHmg09RhuPPVUWHL6vZQY+0QfjPbnInb2p9+G9aQhHPioVWr7mP3Y9eriGPCT3vz1EDG+90lMHvjJvJz2Q8wq9FB8GPfIY9DzpOZm9+0orvkLzJDxX7LE9VP2ZugXQDj6jwoO9xFE0Pp/OE75CoZA91JSMvazrAD7crMm8O/jbvLcJvD11OeE9JLiNPQH3QbwI9Fa9ywgmvi2SNL7a8gM+8asTPhI/KL1IA207Ffq0PbA+mzvzzdC908x4PpRZQ7v/7ui7H3CpvsIlCD4WVyA9n+ACPrLJJT1SVNm9WxG3PVvdOD45s6k9lm3bvTXavD1NV5O+hwXIPXlMKD5ZqPa6co6mvTPTSb7FajW9PO4MvhLL173cOPA9pzw0u4zPfr48D+u83InGvGt4/r2vpAO8TuBnPtyMXzwab9q9eqOkvPb86T17b5o+PuGyvakH0L27P1A9GznjPSdrFD7thYM8LSbOvVTZwzwpIWW97wAsvRTl573Jhrq8P1WWPdMKnz1QYCU9TZ42PAPCvL0+ViC9odm0PqVLVj5LsHe+CJKhvUV/Qb518mg+","N+GCPZDC8LuKqMk97YKNPUahGb3GVgW9CPeyPZwZgD2QvN29WXFHPr0tFb4bybg9QIgbPYW0gj0lq6g7FPfpvd9HYb4N2Qe99FGvPQrPgzu5OYw9Zp0uPLVK3r3BAF8+yVgsvQNT3D3dFjq9VGK/PblWQ75Yt7E+jYFYPlERHL6k74y9zRV6PmsBvTyOR22+7lYGPsrEjTyp4YE9zxVjPvd6LD27FTS+Va00Pv73Bj7OCw4+T5gPvdhK+z0z9jk9QSwcvvRn3DuD9HM9wiEaPk526j38okk9sXq6vW0tUL3z3EI9t9K1vDVHqb1NP1A+Y/H4vC7kOr6d24I9h9lPPCi9/DwlbiG9fdMavVhvCr+5Clq+Rr3APdtBHj4dQ/A9BydvPpuWTT2w6ou9boO2PHe7wL3DWoi+zgWmvYl7JL2K27S9HR5HPT/1gbyLzg6+Y7kZPQLXPT62vEq97RILvp25xjzPPSo+T0utvdXZzT1B7iA+9/RlvTxZEb6bna29t0BFvuLVnb0eJ7i+1Y8LPgsLbT2NNA0+CILiOxRPqz7GLNm7HinsvLX+DL1t2xg+jFUtvzptDr36VfU92sUNPYUrwbqv9yM+DrWoPu/AFD6SfpC9rKO0PleFKL7Vw2u9+FKGvPTMAj7GP908UJjMvCq0B73Y3Mo70PDLvUIsND212OM9RH45vXhs1D1GeHM52OFvPUarBz5dgai9Ycf2vQpbpr0hals96C+xvGpOgT3+SQ6+DtcIvia0fL5KXRu8xqTEOZCf4D3qqHa9xpUHPr2qEz0klcc9cdsRPRFOYb2P3867Fm1ZvWztqzwFvp48QunNPSFVxb0On9i9dfvXPTxmqz3Z7AS+h7a7PSaqub0vouW8ePBHvVCaaDwoz609pp8FPoJEAj47DyY+Su4iPQMXkj3zLnw8XkQ+vlvrhj0Fzuu8jSjqvZv4mj2xB8C8QDuxPZ8rHb5X/Ou9FMkAPpU0Ujwh2j89++95Pf7uDD0PL9Y9wo9DvcMwI77wl/q9","eVGdvcPOX70FRpO9vVt9PU/FHj4eKUc9hJOrvBvLor507fi8Wa9LPdkVKD7Lqck9poB5PXSVFL6sKM+9gV7hPCbOCL5YL7W97U47Punt8r1QlkQ8e0S5vfqTkryAgz0+CN2MO+GhCb3GdAC7AxKhvIW5pT0z8uc8lWXCPaLsoL6olqq91HhavbwMUT7wLo8985XuPQD3m72YaBg+NxSYPULC2zwUgvW9WArevYLprr2fZKm93WwFvJQ4X73vDMK93n1ivWNOCj45ABO6Z2LyPQ0xWb5kSI69oZ32PXAYfb3MpE09/jwJPpxZuLvgTaG9y+VDPUSP/z2cvjQ+XaMovDaraT5ca+O9a2q3PWHJPz1s9MC9KnXDvJaoirz6R/A9fF20vY19E70OrS0+Vm6TvO5Uprxftn49dxYnvvMi6j1epas9GCRsvGnThb3wc8s9Ym43O44DoL16kwo9uhAVPQ13FDyNycE9XKGSPdXv2zjykN+9xe4GPTnV8b1JDI89094yvutc9r2wBlG9+z+LPGZKcz2rgps9JGG4PDhLpT44MSW90mcNvijPOz5GksK9UaInvG2HRj7BKG4+bLhZvgwDbr4BJxY8OQnyvYxgJz10kto+ajW/PPruWb7Lnf+9G66bvaSuNz33g0S+6O6uPZOCBj0hquy9zc6wPITzXT1BPHU9BnO3vEI1lrxTFeK+z8o4vrdlLT0/gBG+ADagvYBiyDxaSGy9RNNQPXU7hL3IoDK+CokNPiAMzj1hl989bwiEvaHDi72Zmms+IJn5POEfEj4pMLI+45EDPlI7JD61DLC8RLhFvWbknj0KRqW9ZBmDPReoYD7Y8YY9jV3aPapjh75yAYE9LukDvj3pKz4l57U9yuGMvb1+hz1FMm+90JmGvrbfL708jqq9Ej/YPGmjIL+AeCa+lFRSPQLvkL2Azjm9iYVIvei0n71DHRs8xG9lvYHUJz4nLrS9C0a6PK+o0jxaFqG7JChuPWLmFT7Bv0W+QapAPic0jDyqKvg7","M+tkvGli3L3foqm9SWoJPuI9E765C5U5+4m9PVNwcz1N8Ny90saPvkMSY72Y95M9CpFOPfG4zjs0RQM8Ud5ivUi6+z1ZuBM+ioi8vK+Dmj3kkdC9HbC3vB/Yh72qyeK9ar4dPlMEBzww1Am+g9xEvj8FPr6xP+O8vyJbuixdAL57YlQ8t07/vK3trTwrsQo+klHxu5VVnb5pYSc9FtSXPcXpaTyTZQC+kLoevprZ171tAQ296A50vmYwTT6/EJ+9LvzzPQhODb2FBkE+NyhkvBBTCj6S+dc9nGQ0PYmeg73A8tk9kCYhPRYrWL289Xq8OiimvUh/+DxvDwU+gk1wvVWo771UQEy7Z1xsO+J9rrw2qwY+BpirPRcDmb1DBXE+ghWxvWbWDj6xtgU+Tt1gPnpsjjy7g0W9SJq7PZpDlT1iE3E9VZMyvif0Rr5bYJ69hETcvc/UVz1bP8q9scFuPthfGj5ud0A+ZvV9vVRd7LwjGPY8G+6BPbOSVz4ViEQ+BCQ0OwMaFb1Q5Ie+NfzHvYBMf73VaHw8E0QJvrHswj26ego7MKw+vQPDjz0m/uY9Y88Bvisflbxsvcy9fdAuu0k5yL0E7U6+a7diO/ovVb5Ujl49UZ6OvL/pDr1TIwY9nbK2PRvyID3EbeI74ySlPg4cTT15a+y9CF2EPUeegD7gLl++RgjNPWaal70pVLs9f61DPXo2Fr0yFWC8mB7Svd7d/TyGU469qAg7vmnR1T0/Oya+13QSPEVBCT4rZCa+1+GivdtgUr3l/oM+rAIIvvowBr5wGdM983vDvNfqlbumdLI9XogkPvS93L2+Efu9Ic/MPpIZ2r3uhs899LRcPvC9B70RUTU+cuydPVLwST6+R1I9grqxPPrmK7tLNrm+/SYqPvp4Sz4TsRk+ewkiPnKIJr43joI9OjhMvp/LaT74Et29NQ40Pb4fDr4z5r68JuGtvrT6CT5BZQi9rq6SPRBItz1PQfg8gDravXleDz5CPrc9vmBOPvqLkT2u6eI8","ajHmvXEhAL6cISy9thSsPiS4yj45c4E6S5VMPBCAFr5GHwQ8hMnevdaFgD3JYCA+4AsHPlR8Xr4XlGW8TjmcvdqMSb2paBK+B3IZvjBGvL1OwsY8SFCEvhTqsD3xwOq94UkfOl+8vr2pQIe9fKp2vV8uh70YUaQ9x46BvZ8KoL71ZTM+PtJ+vjJJAz9zWFm+TFo0vpv3A772ahY9aioUPnmj2D4Ci7E9H5QvvUzVE733gCg/PHYvPrtOLb4KrZ08x1LOvaJC4b3mi8A+x1zyPFBa+j0v5GY+TKbVvT/bCL6KvBU9oXP6vE94Lb7seVS9KLG5Pgtgh7ysipy9do28PfmANr3uWTY8SOP6PYbXHb4J6VW9jyCDPffkB7xkCKI80rRpPVnkUr1pjg691Kb6vfXvFLyPnhE9Lu+DPCVHhb1UBNu97mBqvSpDfT2LKBS9fiabPZyEqL04wOG81h4BPkPAej2VSLI94iaJPUxhOD4uXO+9J8zDPR6Pvryebh2+KeSrOjegOr09api9902Pvew3Gb626ki9OUjRvXwLVL2EII49LeqPPRSGmb19hDy+E3wpPbHaWD7a9BY8/71JvKy55jwCSpu9P835vXHdeT3+Eji93hGvvcdJoL02tmM9xsOdvfXjlz3qZvW9SaLRO+UXCD3rycI9s9Ggve6C2z37e9W9R1rXPMuKg72MTYG8oRLXvAmbzL1v67W8AhWLPTbcRb0K6aS9SuK6vaiV/j3yhAk++WddPZiJTLwpHM88M2IqPdzouL27OEw+x+twPHkUqD33OpY8AtcPvgWtT73I0b+9A3SIvh4p4L0ahm06gVINPY7FrL05Owo+1+61O2HrXz0898Q9l4CUPSHUaj0fRL691s3ePXe7Cb73mKe+hhb9PQM0PD2c6PW8P1j7vScOk75CRq09Y42zPeIisb2jgry9ED7XPQQnmL19G+k9TD2EvhbTWz5Pl7c99EvIO5V8vT02sik+EI84vkUphL7E/sC90V/avLXqT75aNoy+","L9djvlmW9z3BbAE+NQvlPUASiL6vPBK+Pz4XPako8T2byKA9s79PvcIwCz6UirG9wlnRPk7pTr0GYy++z0BDvYZxMT5+09Y9gencviAlc707jJ094QmavHNhoD0IGM89tnDGvcwdOT767n89aQhCPeK3RT7HPIk+qO+dvqw7Ij47dns+XtdgvrxvwD0h7wa+uMKkPea3C77y7WW8/iNiPsVNND1D8NI9a5mAvv3Zur7g6gA9hnO8PeI3rz0Rld+97HlxvkA7ojynWJO97joHvrLEAz25aAg9in4tvXs/B77h+DW+N0KcvX3mPz6Ez5q+SjSIvVi9qr7MZTC+pLe8PE65qz0HY7e9FPFWPmpdpj2DpB++3Q3NPW9iCb6iGg6+hVX3POEznD0Mbuq88uMUPt0/RrzgXFw+gUbcvGMMQbtQjvC84nmuPdY4nD1h1uc9B5YuvQRdyD4MKK+9NCfSPTbsBD2oLZw9zqg3Pk/XE734+Wm+9L/BvS+q7z0SfQy+/nNJPr8jtD2OGgq+t1hMPudKuj2xW6I9QhD/u7/XBr1j+na+M4aevjhAzz1Vc6O8WngOvhQ/rLzS3N29QKaiPYbzUb643bg9f4NCvu8gyr31jWC+un9yvvWgiL4sjKO9TrqAvuC8Uj12pp48BiUcvuIuhr51kZw8QUWvvSP76LxG28o9Z468PQWLnb2M0x+8TGQxvp1EHD16R529SHRevlSgij3iQoG702yIvvuhlb0lmpI9pDYMvhPdFj08MNW9vn4oPYtPhL0ZfdS9vWm9vM5tLLp9eZ+9+N41vkqFGjw593U9SDk3O9pxvLq+7uo90fOuvYBGl72hbNo9apDzve++sz2XnDG+eFBNvkFe2j0XxdS9qO8lPvflQb0FlH09O9nPvVuTsbygpxS9yVsRvv8aQL4AjZu+qUDcPXHKtDtyyRq9qqA/vcngfz0wnju+nA24PSaWbj3zXcQ9a9JtPL+mEj4uYTK9WX84vvPHB72YUh4+ip2qPdZshz2Myhu+","f+eMvIcFBr4cvw++fZYvvng5Ob6eWxS+ZRP8vUr28r21epq9RcdGvhdnKz1lpYK989vdvRY5w70dojK+sKFZPfdP/70S6wK+isGQPXGwxz0Vv0y+5EzdvaYsOr4gOWW9fvAhvSMsFTyisTY9BjksvjMB3b0BNF89GC60PeCA+T3Z5Hi99k8PvpkcP75B9yI+niuJu+DY273vcBI9Xu0OPfykcr1k5uK9xHl7vXsoYj5wFyy+jlahPTmEsb1UuuU93cATvjR6Ar5+viA8Q/8BPfqfor08Yle9JL8TvpvZw7103RS+vVSoPTOjIr6jazc+akFUPSCRvb37zs29UGJ+PqDUrr3XUxg+KG8Ru7S0CL5Iebq9spABvnztRbweSYK981CzvVnlPbsbbu69XdD4POpe9z19HoI9NTmDvtipUb3PKgQ+L80ovujhtj31O6o9oAoUPP3Zrr3yR0G+NQe8PbdVvb3vpZk94E3CPHXx9b0RaBQ9X7obPNpxFr4deFk+j7PoPRoAVr4ZJ8A9QDd9PqtHj71ksQM9RmIXPl73gL6dlD89X923PaA5Or5NchE+qTAQvk8YX74t9Xa9a/PGvQC98r1amL66x3a0PctB9D0jCOO9i91JPdOnoT3VFTc+Nzg4vm/PVr7PB7w9jQDHPbk/PD6n++g7E3iXPHTzCr5z5Iu9OH2JveIunL0rlb091SJ6vYYbDrwHiFw9sXVDPNmCY73gsN478HzsvTKtnb3L3T49bwm8PdTc4LxWY/O9XxgqPQiSXr2X+L86rM1/vsHHg71e/9k9lZwMPmOHCb6F6Z68cpDsPJEZZD0S+i49e0iovaGXZr1mQ+49aySMvY8z0r0E1E09vZrhPVrqjj1hBju+q3cWvhlSLT4rTvQ9yt0cPtTDYjzHrwM+COrhvHB3kD5CRK29OxyvPUPSQ742fc++D9ZavhROgT2iku+98D67O4eRu70CYNK9XsiIvS1ScL5hI5Y8ZIo+PVNzMz7kjte8Hk0SvlcyL77qpT29","rSyzu4nSTz0klNy9jBElvfrzrzzknCy+PqeDPhAiALtRzXQ8ovhlvGbQnT1idKa9LdEcPmY93z1QU7w9yFiXPa0opj1fmIK+BWo8vY3NRD2EUfU8UBc9u/vYCj0sq866/zJ+PtCAH7325Iq9F6qVvBblQrwsNTK9d068PadhFD7XdjY+aU/PPVwzfL3b9d+8pwegvd2Xn776fqm9CqQtPNenAb6TLJO9KDy8PXk1rzwNUOQ90n+QvbHjTr24DsC9Z22wPMxV9r3DxhE9hP9FvPdfmT2qKRw+0dA8vtZ6LDtmlQm+A6fvvT1+ML7WW8g9I+szvu01D756fZC93FSRvaV8jD1TtDE+SF4MvRJyej6fMAm++UAxvWREfj50VIa9RAMQvu+W9z33TQk9b3puPbw2Br0DKD49+IauvcTFhb3suqS+h/GkvJHDbr206ag7BVKPPZ1de7ytviG+0Qr5vBzXhb2Vfg++H2kEvT7/xDyN5Z4926qXvlmT0z2DNNY9NsVKvQb0Or27A729ZSmtvRZ5hz0/F78928hmPsnjV74gGHg9KwAoPSbWZT3nuIi+dk3PPM7JAL5u5t099avLPQud2Dw+CBw9Y5yUu+L8CT6uYI28V6N9PiYBUj0jGrS76MOhvECVobyUYC8+uIZPvhKgkrwTHHO9VpETPRHoYr4knGu9WI6TPe4Ugrxofqw9+MiDvlSUDr0hYoU+/FezPRifzr3CO1k9IQtNvFx0aT3OHiU+Il6JPEvlQ75O4Rw9O7WivVvoUr0ByHu9iMZdvVKoXb1pWfk9pDeJvlyqkD19Ix89/xZhvV1yez4wloU8HyHPPQOgEr7Rems+1cxdvm9KCT4ucys9PQpFPkuZNL7CZMs9sx5QvWXuKDwQBxk+S2eDu/7hE701eLC9n04hvnv9EL32ydE8rfQmPn8zqT7j/BS97zVNPG7fL71lvbi9vC0gPoiXnbwO0Co+dx5GvZA+CT4G/Q49tJbfPX6Kfr7vrLS8QWg6vuvVMr6gEXo9","VMr2PBWKub1WiA4+dX7pvH/wcr6u5r49NkoPPpoQAr7GQ1a9FpFEvtkA3j0YFwM9EzNsPi1ISD4hquK9MkdEPr7TLL7tDX48q/w9vhSbh77cwoI9ucDVvfLZsL33JhC+IFACvllucj26H/c8VhyPvYEyEr2cr4C+Sb4Pvnvwwz2ntrE9Cz8dvTQxur32xQI9SnjYvUY9zrvbL7+9iRmIPdSnf76+XzC+Nf4Cvl3lI765CbW9DhsQvtHKPb6nYQY9n3PIPb7VSj6Ev96+Aih4vQH0Qb6i6Iu+RQyKvS5m2LwOJqa9XlCPvmbjir40dfG79gmtvqJ1Gj6Z8kM8llUHPQOWprxr9wc+OyihPVSWi73Mv7i9pU0GPvsM/T3OCHi7WPAbPpjH2LwQxUS+uhZwvIGOsb3IHGu8TV6wPLDVjz7zYBI+86+7vTFkHj619cy9oUr7PbluHL7ISAO9iSrgurx80T5X6UE8TyyjvXvmFL5pG0w71oxhPWXGDD2CwpM9AfCOPWGYTr1B6fU9x4GXvXaGLT4obwy+QGLJvcDW0jyEqrq9Gmo6Pkz4rL2hwMw9DRo4PuG+Vz5VYB6+bFGVvQ8Kib0OEBi+d1I8PskbpT1hVa+9WwC5OsmRmT1aUoy9CLekvU0KvLy61JY8tP0vPj7fnb34LeI8yBHFvRUAy7uzVqw9fWjhPTWpKD5tRnw9M4JPPQScDD4pAlQ9XGtLvf6Z5LwojpO82yTavdgYgD73iWg9q+VSPvJYZj2WFdq5rM6svQPNgj5bvY++6v18vGleQj6duV+9FfkVPhDHLz3WWIM9FhGMvefsNT68inE8w1vGvek7uL6KIqk9+/SxPTiWub34WgY+yhpSPvL/6L3s8c69TjdSPi/52z2se5q9kIe+Pa/g/D0fXY29GImPvvMXtL1MxCC+i8zCPejhXj1utwo+NOztPVo6DD4GpYi+Z/MLPSstBT0qxUA96+sSvmPf77wnHhI9oa5GPYbuW72jl6+9QZ5XvP+2uj1PqkG+","g2HkPe/+zT3xmtc83uqePd4U7j2RRdK9iZl3Po3GGj24WSG+xHG+PBzxY70ygZw9LXTavOCJqT3wdzq9yGUevqBvK74V2hk8ik5lvf+JFT1oXBW+gS79PArVD70bTAy+MIUUvanLCLywdLA90Yk8PpX8HL7hf+Q9uiOwvIEf0Tz2HQi+Z3zBvowHcT2h8rq98vlSPXbCSrwqtfu82oxOPs4uXr3JgVa9KbCEvcTGJL7i4DY9/O+9PTPe/zzuCtU9o9SXPTCzXD18GZU9PASdve5liD4sNk8+K/1HPbFyED43COw9fHgcPgaKtD0C7Aw9YBosvRp9MjzgY2096479PcUYXDyfALW9pJhLPiCHC75gjOe+OaukPH/G4bkhs249GuwTPgNmgzxa1v28V3nhPXj8tz57KKY9mBC/u9Apjj2whwI90PuHPWD0Fj6tseK9TzKnPYpvxL3kicW9BBC7vTHssj30m1a9mKdfvZ55IT7hRWG+XobovXERmL7tFtI+FFwovrfbCr7iFEu+WtUZPt7IfT6Q03y9y3dBvT+CTTvp9qm+6ZR6PTHg57x7VxE+9AQPv2snRz1XSdY8UTmfPb2qHj4TNoM8v6+Vve6Gqb1LGqs8H0olPIbfQb170hO8lJXqPZ0Z5D1i70C9WxcdPlFwl76NkyO9BJa6PJoCdL2uM4+9ShqMvS87Mj2QEmM92JLnPU0aW70DjXk+xbNqu7lmCr7MuhW+d6hSvTaxcb1N3fs9GgMvvg/jAr3Ku10+NNotPr0ydj233Bk+iy8KvhAgpry3Zgm9cB6BvYpzErxkWjE+gflVPYOUUb6PlIu6ZoEpvZxYaLyxXw8+E1MNvDG7AL5c0kS8/DaFvdE9C76gNQo+IgPUvdJ0pL284uI9QEu5PV7UXT0oybu95csJviPoFL4AZvC9qa5kvqMO1z34BHC9R7c1PKQH773IS7I+WWo0PE3dAT71zh6+lTD+O38fE76DK/A8GOcrPZM2IL5UBt29qnHWPUTzRT3T1O08","OiCDu8VJQj3WpqI86jmTPuL+vrypkAe+F4jsOxanSj147J+9VSGtPRlEvDw8TQo84V8hPafADL35fdy7DZFwOx/8O77TeWs9NvyivOaznT0rtz+9gZbtPKMmQz3uC7G9XgJBPnv9QL1aECC+QEhevrMHe70CZ1Y+na4EvjJqAD5sJM09oEuFvb/lxT2H9fu9mpOkvV0unr03ErE9UlXZPvwsoT3H9M686AnrvH1wEz5VUFE+kkOaPrMlIj0APtc9EqvUvc+d0r0mXxy+v4vhvY2Scz5tBny8M49UPYzJU71Pt2I9fEmjvc71wz28wrI+LApdvaq7mDnLQRa+SnHCPX6vsL3itIg9911xvWYmBr7OFQu9qXr2PD5iOz6pdiy8QEBcPou5wT2TcxO+YTzSu6kIHD5V4Nw8dF1UvR76QL1ddRe+89wOPY8cTzwi+4S96Nn0vZjkZD4Okie9FmjbPBF9CT3qdrO7eSqnO2GgZb3fm8A8DfvqvYrAFb5ePLE9yj+Evgkhz71ImYY9SDoqvkTe+j1b6Nu8RoSRvfEZa75BucW9vbuOPiXRFL4wSpE+vzBCvJ339T0Oeve9EZNMPp7glT2JjL09DlcZvqYkh73Qqei9Oy2ZvXvQ7T2KlAW+tOiyOhrdBT2MDYS9K3GEPTp1+j1G9ya9ZH+5vQ5Q7ryIoPC9JttCPqDt8L1YoOq8dcNJPgY6hT0c9oG9LFbnvSw00zqeFQU9ryJ2vpVxmLxldBG9W35VvUtoUDzXVTS+9yzUvUVf+rxQUxG+oIsBvuyrN77G9W6+knkavrdnPzpb8Nq9Ny2ovYKqcj3KD/c9IZU1Pt2ZSL4mndI9Y/8wva4CirxrVg8+9CHovTyoJz5gLoU9nqFaPW9JEr43LJI9Z6skPoEnVT77mFQ9q5M3PfIsmz4c8WW+hM9bPIjkaD1sbFU7T42BvW6cDz6U5ws8tOgIPuJMHj6C/zY+yDu3vZqobT4dGNW9xZikvNhhEL0K7fQ9s3GlvRvOKLsWR508","rVQHPg+sUbvY4gC8a79kPDI9CL6d4o09RXQpvc3jnT0+a0i9H1pzPm0bkT33cSw9Q7Fyvuwvrbym4qs9phk6PdYyLL5UXok9wrSmPRHUC74LSLW9602wPdSIir0ajt49AFgMva7M/rz1Xe07ehGaPdLCu71Rr/G9QdyUPH1tvz0AgTC8iw6bPSEpkb2fwaE9HAHgvctiqL3WYMg927yLPZ1fKTz+jp09d+GRvQN2cL09hvW8EGMXvssrubyI/eW9VgIxveA5Nb5M0yY+Dke5vb77cjyFoLE9ksxzPVUuxL3QS9c8I6tUvYsaOb662pa80bAIvTdDPz2BvzS8dZWnvb6xYDxReMk819gbPXUsnLyWixC9EBi8PZ+XIr2zg0c+h+ZRPc8IET6BqKS8BxiQvWg1vj0bAJ09hXGLu2r8RL04Vq+9znbEvQmFdj6CMU89+mJsO3i2xL1gcIu9rROHvQRyr706fBu9VCEWvNsGt72atkm9f5tEPq/Jzb2Kns49am7IPVHwWD2Kc/+9OCd3PB7++DwtcV87E3ErviIinz5Es+G8kYDfvRUX1z3AZoI+PCPBvUQADj6CMbA9H1/qPP6e+bzECCQ+bQ+HvRN+9rzm8Ts9Blx0vgnbJj4d/Mk9pxaNPZ/CLzyjoSC+BiWoPpgCYT0EFQC9ZuezPZthhj1mQVW9fxE1vipTBL7uJny9VGowPeTpNL6RpsK9FQ2LPaymPj6gIgy+fuiAvS5sbr04A7o81loNPBn4Iz0r7Xu9bIW+vfR3Vr55v069m9iXPa2Tv70CtkO+G0rBPXh6/r0M8Kc9nA0dPlkHKz7mqiy+y7aLvEmxi77DY/k8Tms/PCnNuj1ofHi8oCb7PKmGqD3RE6i9JZoqPaxigD0vA6W+0ccnPiN0mTyEshY+I84ePtWUEL2mvSu7cO7KveORzb1+VgC9D41PvWl5BL0+WJw8W/GNvhS6GzznbRW+QzCrveiOh72qOpk9PogMvt2AjD5Izay9HYt2PbSAET39HQM9","9BAMvkrUjr0Fa8685CF8PUqPsT4MLGe9J44evhhkbr5SuWI9dZFyvAMIPz1lOV69A+C5vYstjb5wd8q8MEisPm7Htr1H31c9IaBovaw4kT3UWHq93omLvXLw2L0mZaE9UhEMPg6ojj0IzJO99KApPug5hT0nMVA9+urXPVC+sT2MGZM+6wSOvYGr0T5Dk2e89KVIvn8ngbyc6Sg+6LNRvN1i6T4TzQy+pfVqPvtiZrvtEOY+9GslPro7U7uuc/G9OVYgvpHrML41gYs++PW1PbmuvTwQpt88sYvsvBHV/L28q6y9UgMyvYeLRD4JSLC9162gPuHHozr3QUA9CKDmPXB6Xr2oouU79XMlPYSdfL2xSPg9VqN1vTNCdb7wuhG9ePEOPtpGCr2f8h8+nuAWPTEkC7529+M9IAzsvTpFJb4xIN+7AZXjPMuAwD11JCe8QQm5vczuNLybYg4+Sa10PPDPMT700am9vEfKvTQg3L35KvA94SogvjmHYz08kOi9ew6cu4cnbbr0dRc+1f2uu6/xgb1+ydw94888vkg9Fz5DefY9LanjPUH1zT3qQkU90t4TPUtXiL7ahoA+ZMQ3PUKrAT26r7g9RKmKvc6j9L1d1iO9pLnWver7E74p2fu8TvTLvPPrQb2N+Do9Z5R3veP56T3Grsq9WJr2PaKihrxma7m9CkFuvkxrwb3/22u+gNDBvWzGw71SMxs9sCpYugJIpz0iito8/eH6veVTMz1SDwC87yL3vYFHjT1UnDG97pIOvrKvkTzE5mM+HBWPPQYLeL7uV6e9R6p3PQ2AHL04BLU9+ERCvBhzlb0abXc9mXYsvt3QgT69jEq+z6djvoHA0DzbQNG9NJn+vYPOMT7X+W69Io97vjR0g71M0Sg+tLR5vcJVGb2bDgq8SHAmPmQYBj3DR7i9reR9PSx8Gr6Ux/m90JKjPellTL20OsE8vRBLPecYU768yjg9tecDvSxTC758jpY6kShsPCfUzDsMwYm9dI3yPUf7/TtBsR++","nN3aPeURWL4gi0y+az4JPsmrNL4e98I+RbDgvTIL4z1TxPw7SxzdOs+jZj0FGaW9/MR/vtIrAry6e1i9uoX+PbOucT6Njze9uxWOvMuCK7687ju9B5AWO2Wuhj3Hya+9O22svdWTJb71mQ8+dMeFvZtmBr4iGwQ+HZm4Pm7lgD07CJY+P28ovXBDjb7EcAg+i71LPZnKDr4FwNS9dhZTvvwThj3wc509D2gQPpifHD2szZM9Nd4nvUI2pzwDZQe++7LHPSZpj71G/5A936S7O0iGwb1HrRG9ztKUPYCrgTyiwvs9LfHnvTdZVz63n+W8EXUfvsfdNz2mfNo8vBuFvWGo0Tz9CNC9aldvvkThqb0wxNA96Yr1u+2R9L3HsEW+iY4RvrYLcrzb8OM9Ukuvva7vKz0z1Tq+gtmFva06HT2zwmE9R10wvhsTpr1gAdC9s5kWPrGCS76fRsY9GEqmvZ8khD3LYFK9KwaSvFwK0LzBRRY+X9qUvG3RQz2SuiS+IYjWPOr+Bj6+iK09I47QvhR7GD1svP+9D3kjPqefPb2gCso8P+mqu847Lj56RfG8WxgqPr2JjjwgrjS+juHgPWP2Ar68ORQ+kSWkvaUwIr73sL29XHmyPQKjcjyg+QC9Qlwivnzq8b26QQM+eF6kPNGTBT4VOT++MmMxPsz8Fz1mDzu+PKNNPRZBxL2EZKK9X/bXvcCOnL3BMQA9DX6LPcW+ED5n0Le8SsFBvlJAtT0vezk9QAjKPT2fpj0qPc27UNw0PkShUz27QYA93zdnPQToyz143Qa+ELZ9va8fXL5DgTY+7MBTPTn9Tb5TtUC9/7ckvlYvtz3ZWwE9gzjNPY0Paz3Vym69cylBvlVpbj3T+9+8gEAIvRmHLb59KCw994ODvlmKoLqteDU9FRpxPY0c4j3QSCI8x53HPbKCoD2I7f69PUl5PR+l5D160C29kPrwux4Z8DuqTys95qyWPUXql73BBEM++11ivj/zkz1u+++9jCcwPKfJlL2xWuu9","CGjZPI4XXb3SYzy8QiJPPc64yb2d6/Y97K8BPrTQMz0bLJq7+Hk3PTaQT71BBkc+R1tivSEQmz3jIvM9kBI3vX8wPL5E5da9gQItviV9eL2rlvc9ZyWkPWo4zD1KRhy9pChEvLC3Nj7RhoO8pq0IvlLOQz0qm4O+6fQtvmIWK77xA1u8QYKCvRBPML78qei9QWIIPiCuHL6lYhw8qplvvhoejr00l5K8VGfiPXVMDL0xljG8z6IUvs095b0USZc8hWgJPkEloz3yKvE9IrMEvgMh/z0Hjpw9W0hgvunBPL5ukiw9vs0cvqrZDb6rT6c+XgdfPXohhT3O8t49hZC6O3yA2737/wY9hEdvvYDp2r2CON+9w7+tPU8koj2xoLW9UH7hvrZ8fT6ISGy+pGigu1Wieb5IFkI97H8IvQ0fob1I0b+9TlwYvhd+kD52CNU9fu4evIy/U7w5jT6+rt4kvpOVQTyWpcm9I1oEPqxYLT7QkYm729EkvnCq3D4Z2nA9mOJVviVDRz2ZqyC+vFtnvj0hx7zC18684JQNPPOt4b0Gzni9i5jSvcQVDr3vfiu8P1FKvcn0G773Wbi86WtCPSWe5T3Qktm5xZiYvchD6b0AVJU9StENPtiPYDuvoak9rBbIPWZbGr7MtTy91LEsPVkiEL6LUNw9W3ZvPYmDozu/aiC9oeOdPV32mryksJm97gTIPfky7r34GRk8kXhRvX7ww7wpwLi98deXvQllzj29v9099ni8PcWLu72uYoa95aK7PQezeL0MfTm+Z7Zzvj+/uL27qrC+JCDVPBjEjr2jDw48kxBKPmSC4L0852W9oizxO8f4mbsA68O++dPTPZKUH76pQIG+z1I5vWsOvL0L57a9RjnUPXEfd74QvIs9VNfJvQS8u72tABK9FIrFvRbCl70oRr69dzsIPaHHJD7CMw475UIRvm5fiD1/ap49RuhFvS75kjwobOu8W6rJPBc1Yb2gSPE8BSRVvnKeJj5avxs+xgCfvUG/Mr7m1Oi9","N/FePZ80qDzIzsy9Gt1Eveb9bj0e9za+Qji2vTdN1r0X+La8FFkAPqbJjj2HMig+5pvlvdK9OL3pp/m9XOj6PdFTyr10BaI9bZtlPddL373AYiy9yczJvX0Xhb1GmiW8WMVjPuxD7DzdnLE94CpKPhbxur1LPoQ9llJhvcXrejrtZIE9x42POwmu+71qRZK9c73FvXDOnrzB75W9XwgRPjC+nbuKGbG7R0UEPlPux720pwg8WpD1vNSxMj2EZZY9dMMRPqv1NL5ELn09Pf3yvaoAUT6Ji/G9pyJGPvMYgr2pcYW9Qdc7vSTRrTyH0au9HIWWvYYP9D2Um2S8lGXAvCI6Kz6ucOW9YFY0PTTbVL5hHdM9hr3BPanls73M1IK+i4y9vWZsB7wnf5C9LleTvXa+CLxI5Zo9+iX8PVotzLy87c292z7OPcGwJz2reiw+tpgAvX3cO725U+i8f3EdPiGHS70jGOE9lgkXPTdbCz0NpWK9x7Mrva6Umz3XoBK+JaVtPXnV8j3SkrS9/eYzPlE3bb2BT428nj28PToUdr5Z2Y+9R9KjPSrMgbtMYf295L5kvTfeC75KvYc94ld9PTXIQb5xSX49OS1bPWrzA7yIowe+Dn2jvF7S/T2BjZK9FXkIPKxsBT7C48O9akbKveZ6kj1vbMk9MkwqPlz9Db4epY0+AhKyvQjR5j3nRbe9hQzuvfkAbz7zT2W9uoBRvcsocz13WQI+BM0QPqCogL1OoXe+RfR4vfWAhDsJxv69TAK6PcCUPT5w59o9iz48PT2Trr2ssJy9sf4QPgvlBL5fCuO9G8rpvc2R/j1aK4g7A/+5vTNVfT5lCgI+ZHyHPB++Az4XJLa8f/QIvkebvD2nd5C9wxutPVwGoT2W+XE+tu+NO4Pigb7ldwi9KNBSvWSXyj3tF/K9sc2yPU1I+b3EF/A9pKcRPqWk0rwpD4E9x+N3PO5xaz1vkI2+ChIbPv4rUz6lVGs+jihXPuGHhb6CUxi97b+mPbgO0L2BCFq9","6kmhPXFIAr6bugK9PZaJvmQeNL6amhE9mLdKPVY63z3uWIg9J8Y0PbsqAT4nX2q9kfDKPYEuhD1mDcc9JbP6PcrroD3xsSm9JQqzPlCyqb1miWQ+d0uKPnyR4z2JQNW9uXSGPVc3473VZ3Q961YZvRObBj0oQlg+4TKmPZOUCD2H6i++rZLkvar/dz2RQca90RAxvWx38D2T3oa93LQJPRbR5b5QrT6+WFhpvf7yrr3L2wq/qFmpvQ1HDD5OEgO+1zEdvjuLZb0r/rC9ehAMO9Rog7xAFPM8PYHLPYB+qj3I95i+mLnXPfcrkr1Y3AU8DDFVvuQ4cb0TQbk9kaCLvFPCUD3/yuw8V+v6vBIYMTtO9oc87Aanvc1YgL0UGuC9N/A7vE1SwT2ukw4+kW21vWWebj46WKk9lGcnPZt1lzy7CCS9E7orvntNsz2XpXi7p6IevgQzh73LJ0s97/Z9vR2/JL3LDOI8bjTEvQ0pXT3pphE+mOoJvfUPtLzi5SE8G2KUuUI95L1HpDM9b2qdvdY5Ab60QAw+7MAgvgw8iLyWNjI+1EulvMPKy72pkBI9yccOvoarrD1sZd89qEcQvUyqRL22uzY95MHfvB746zzPpZG98IGCPb9F/roUhnY9xX5TO2EkE74ULT69uqj9Pf1cIT7l4Os9ibzwPBCXxb3sNPE76EMcPbcbUL7jngc+aEBaPKMCob26F7U9pXkcPSwEjzyg35Y8VjZuvoFdMz7TXRy+C48Svuf+Yj69aNC8qG9Mvg1Gvr0Lvc0+axW6vfK98L2emJw8Iz/pPD5mUT0EnKW9CKw3PlGjsb1M7VQ9o1HpvB14cz7+OYu97Q1QvraKir3OKYw9cDbovTP4QT2dtKc9eFq1vSBAir22AUi9YYQLvqYeIT4OXcg6tkdJPOgPSb2+GtI9klgzvegvgbs8lHs9Juw6vmpyJj07r8w9xKc/vn6y6z1narI9JMNRvkbECb7taAU9dLKGvVSrWL4IAQe+3ewsvqfv1Lzat2q8","0bS1vUX/CL54HGU9u1wNvSlRWb4lSIE+K397vR31VT71WMm9pEewPbiOmb3YxgC+FbSYvSGc5z1nOq6810skPds+CT7RIF+9clp2vTyM37zQxPu9dxZbPSIW+L3dKic+B71OvopVgb1/rbS9M1WgPHsbIr4FOg0+NRvyPbU9gbruCAo9vFIUvuHQ8DyixC29LIRCvk38x7yqEf09NsmJPinmNz7YB02+coQlPkSJY76lShA85gXOPNZRST53Wkm+WA/uPNnLUL59B3M9/ZEKPkLtlD5y0em61uE5uaINzLyFp3A75a6UvfY7Xj7rIkm+UBsovrR4Rj36tza+5JHrvZ6gib7EcNW9PCHCPdWjRD0Supe+h2H8vTZvCL2pBze+T6LVvIADHL6kHiq9qwS2O7cZmb0KTXG+YjNwvujJBb69Cbi9YIijvYh3Yz60g0C+TQRsPjtOjD7hzJo7ZMvXPWt/WTx0tR09V3GMPW366b1GEOM9FxhwPcbUkT0Pagu+SK0TvshaKD2Ay3C+evPSvU23Pr728Rs9Yl4BPpa5abxsYqe+kE8jvi8ISz3zaJm9SuubvsA2Ab6KZCW9GwKkvTi/erzCJhe9u5+avl3Dmr1+Kqq9RONTviopzD0hfdC9iSgpvhmMEL5dKjS+k5MbPom5VL6JN8A9mzrsvVFw0b0UkoQ9FpU6PPcwtr1RsvY7ZgkdPteZCj7/dWU9aJVkPcfzvT3/TTU++1S6ven9tr02YY68EsIkPfMauT0AiyE90QVsO1oSWD7Pkx69YcMKPl0liz0vcGG9Dtm1PSlKLb5VqhI8f/HKuwXwqz2r0F69cdjTvJZSNz5t0gi+sjG9vdHxP7vL2n887c4fvWnTNT6WX4O8IoFwvVYoqbxrEfQ9wE4wPp70jD7Vkoe9jJ7SPQ8HS7yg/8u8Xg/gPTRYmb2DAfG9/ftMvWoB4D0RXd08XPrcPdHMQD4T0aI9tPYZPdtDVrturN28ElIVvqDuFrwD+oQ9sJE1PpzUKj4vVxe+","IsZAvsW1/z2YcUA+b5d/vurJnj4rMcY9eP2/uz4fdTx90MU9E/aTvXUaib0M7WU9dmvpPKcFDT69HZ295MbYPaI6yD1Y8gY9ubJfPgpzALwGP6s8HqJ4vUJXGb2t6Yc9Q4eDPW3GRb566PY9PjyEvNau+j2F2109kij/vORhMj4Le5M9riSBPcxBAj42300+7iGmvdJSCz1g04m9Fo4MvnVTPD4xPkK7fsHrPe7cJ74/nTu+b5vnPElxIz3a/YM8HuM2PUs5nb0upzw+t2lSPVtP3r0klSo+6o9lPutzcT4iF8E9iNeBPd4Fgr7tZZe9Mxa1vdIrdT1nOAK+3YAtvYKBXD7jG7q9qsgCPtPLqj3x5QA+YOOQPUUYo7y3rjw9fG+kPpLaCb6vEM49KuNkvfU49TyjmLA9EVsEPdxZxjtzxam8IxgBPihTFL4roYi9wC+zvHbIHz2tiZQ9ZjMHvKd4pz20UYC813WFPeJF6b2HL6w8Y2cLPj4wAr4HyJY+VNhjPt5wtD2RCsU9vINdvfAcpTmFL2k+Z+4jvacvTT3h5Di+pPR1vXDDMz7v12i+NNONPAw3gD1g16C9E1+Avol6db7KgQk+pUyOu7bfxT19p+09Z/7DPE9y+r1OF8S7nxAivWvpNb2uWAu+C8Q5vhtbXr6Dxg++GMyWPqBWBr7h27a9B7xwOrtlB700Bpo9wgeSvYfQwT2/y+G8EJvIPrCTAD5Pfnq9T2McPgd0VT0pdE8+SAZOPkMd7D3BkaS9L9G/veW9CD083Fs+mYcYvUK3xT6js1M+B8bnPUClZ72tev09UZ6IvRF+ID7IPhu+xkKQvcznJD49Us4+itOuPRiXPj6onWU+glY1vkF7ub1L3uI9CmWYPIw42D5TGYY+lMuAve4nFb3gfpo9cTGtPd2c1r0ZoJq9rIb9PSG+5L2aqI+9eCCwvL/36j0EtO89uZN4PeiXIb20HJ49mAo0vkliRr74doa9Z5xlPtMVA7y0fFu+yJcHPTUUiz6UL/o9","/6cRvMufWz0DUay9/Oxjvcx3br2hJsi7ggBGvgER3Dtdx809Z3gBvnjGZz2cmkY+cE1wvRJLCL6e5Am+ujvyvYYWxr32WLQ7QXBivYbBdb1MGAa+r18RPs3rWr0BMoQ9pK7RvVw0br6LfOO9aJy0Pb86o72FV4m9+WoKPXIlkz11/hq96DkOPr6YzrsNiq69SLScvOHcTD5buC++xJ+iPQvdmrySr7A8e78PPrIL6bwHNCy9sntyvqLw97ulhbc8TDnOvTC8lD0wWie7NbTePWfq9T2qaTK+jhDVvc14qz17hMy8Yr2nPTl+2b2erwQ9WLCIPMLCRL1EK0E9gJYgPuAb571WYKK9n3VmPT7rVD6ZDQe+UIBDvmB+9Ty5FwQ+f77rvTQ6AT4fz8A9l82AvnEeLL7g+5Q9wVkDPuuGpD14zaM9HtuFvOCAtT4C6hK+MphKvqxNAj119u69ENihvfepP73ewYk9jiUhPJZN8TykTey9LSKsPepZ0z2VO9y9F3ifPUyWrr0icFe9zLcevEKVqbu3yl2+8obGvSQXJDxyZCi+YAaUvS6TU7x6fw69/0hHvPqfST4FR9c9xArAvfTvYL2qO7w8rlY4PaZ0ET2j57M8A1ByvgyClD1RjUe9OXt7PX0Udb3Kc7e9t+xnPl4Xl7xyT1G8/apyPfwpmD1tqBy+9W4ZvVIyf74sife97ETMu+kS4T1vVWu9hLOGvUBuqD0lSSk9+w3BvSiyxb2B9tO85V/HPUHfHD6eMX69bciDPfB9Qr6C0Aa5vLtivABzcz1V9J09e6kbvq5bP73DcLQ9WW/gPIqfgL4RB6m92I5HviKGEb5dkwK+pKQevvRRsD0c28o8E7BZPRZ/Iz7ejLu9RcBhOwMv6z0ynaS9kzVVPSy8nD2p5oE+JnQ4Pr6wtb0QfyG+MMwUvZDQ/b1Lq/I6K16RvVkkvDyDP+g9qxJfvtqior3+J10+rwbfvenMjb16nRy+K+uSveIq7L15rZI9zHTKPXYCgDzDWFo9","P6rJPN0QcT2onCq+jDkCPlWYlz6NEHe9m1aAvB8TK74/SBA92uBQPKRq3LwbRX++ED/xvqjPhb5/fO69dQ9ePWOWMb7DNce9GYjyPUi4tb2vS1G+SaYGvtFKZb0JJRi9OQ6hvb8p5D286x29CTVaPRHl+72MowM8u7OjPWXgH74qrBc9mliLvLqqpD5T1Da+HzhpvjwNEj7ti4S8kr7HvvmCPz6GBBe+kJsMPqyDaLx/SJM+a06NPeV1Iz3fxQw+pwL5uyslpL0peO88P4ZTvVxk8b0wSyC+RtQ+PeRFrL21m+S9ySsSvY1l/T12sSM9Bw6HPEM6v70DPam9yFG8PXF4F74DwWu9qpSkvb0DFz0NGxA9Tl4HvbJ2GL7AxsO9WDGEvmBFyz3/VQ6+bSC2vfZcaT1eT3S8N4ITviJPFL24qp0937FNvrhpCL6qTAG+pwgHvSpu7j3jtxk+kbGCPfwS671Z+yY+WXEQvgmBpD2Tn+e9hIBTvUZADD35WR8+mDQVPscSDr5h9YM9uDcYva0VvLluxTc9NATKvQNV971ZHa29j47fPQCPIr5vTnU8EPT7vA3evb0Bfpa91fY+vewB5jx9EzM+73wvvbC1sT0/WJ89ukJTvK9mFr4yIlK9yiiavZsjNT1WfZk9gCPMvUoW0z1u35Q9qZ79PHCMP72ohFc+egcKPtml2r1OXfY9x+sVvi6M570wahu7y5ZNPRrUjr1wcK09mt9qPfxEgz3CHFI9J6TNvX4eSj1FUMm98KEwPXHxrj0Fx28+b5axvYm4LL2fi08+mr7OvCdVzr1+uSU8jnUwvhNm2D1yv209GUXivYpmsj6nba+9v2mkPZeXiT2wAvG9Z0AavgezkLxNLBk9V10APpGpzT5m458+j9D0PT59hj3/ErM9YmBRPqp00z1i6zI+geiHvQiNqT0bOte96mIMPdey8b2K5yQ+tG5nPio9Tr1bi4w8L9OcvWEHxTxEkh09lKXovVyV1z1of/u9gHh3vRTrTb5gWOm8","md9lvnqNUT6Dmsa9CHLfPdBYBL6opGw8SW+/PZevuDvRV7q9mFR4Pa/WurvV6Je9ZHQoPvywAr5x9R49fEjBPNq0vTykrNG8dReovMQTIr6c4bg8ob6YvQg6BD75PP49n8eau5bGgj1dnge+MHrbPfVrB71PBiE+0qV6PiAcEb6tGSc9OvFNvuU+oL029LK8iu5/vqbRAb4lsPu92/NGvmk8hT6degw+x/SfvjzE/Lz0NEQ+0qGAvolhJD74pGo+7OShPf7LGD2sLww+UIQavmbNHL6I8qu8DgdYPlOdoLwCz6K9OSGjPW7/Lj3xzLY9BTRgvDJY2bxrftE981CUvT8GjzuMvpY88ry8PYKeFT5rSYQ9mpVTvQmQDT4UOyO9h2cKvq2osr1PE1S+ROY9vXwVEz6y9gy+FT5AvmFUh75rCAo+WkpzPS8EZDyw6hI+OWEQvr3wYr52k3a+KszmPN6/Hr5LOb09N3t/vae2Z76nKrg9STGUveZbGD3fzwo+50UsPtvkO740qPK9AXlCPbu7Jz3HGqG8hROxPWSCP74b+Jk+fkZLvTekeL0JS8w5j2cRPvxmmLt2DlS+zWGBvZgNFj6kHFK9W2Q2vXnsiT19Fxu+JJuuvW4w0b0XnK28WZe+PV0EQz0E+dK9k2XgvT10aT7COto4QMXIvSe+or1Caas9BxJZvWM5Cb4Wu689s7jQvVcvqb1XIAM+nBvVPU63fT6kufE8oH3qPdB4CD6tYhQ+AffBPbftCL0+v+i9iu3HO848cj2HWYA9V1y+PUKKPb7538A9VuLQPZaYBjo9zF092yqgPZjN3zxmn0M+g/yxvN5LAz3oEU6+5Ylivg/JBz53OiA+J8cNvls7Sr1Gveq8dgsrPj+dYjyvO3k9PZXIvdSjBz0mDc092SHUvGqzxr15IBi9+zztvZTFQj27wSe9fDdKvOb2erxq6mk8XKMBvngwPT5ewKi9FBLFvChaoj20UbW8m6qFvfScp739qI+9NqqaPWcWlL1Yv5g9","PzSVPQgHmj3Fnaq9RZUOvR/la77dUDy9vLw5PteZX77WfR29RaUavu9h270m4UA98Gw3PfABLz30wqu946azvdgUlDw3bJc8atunPcHOsj2uFqY9a/5qvXp6nbodRpg90BU+PdBq6D3fszE+VfoDPeMORz1LANu9ObfOvanwir0MdKS8UJGbPVw+ujq1NRA+EiPDPDir8z266VS+TuWGPQlsDT4JeUK9tfkNPqyggDyNsmC8jHIVvuSBAz0lG/a8Z1d0vTKGAT7Mdak9OwjevTs/HT7CHTc9ejpDPZ5hRj7PWE89s7OvvaKLnTytnzu9FP0yvIS1w7yITLY9zi2kvN6ZCT6fQ6K9MTkhvdIKxb26R6s8o5JBPg3WAz5k2L699+NhvkarvT2COoe9OxgFPljWDL7GW00+3seAPFJtSb3nhU28CVjsvYUQfT44/Ym9QiXEPJYe7D1RE3c9szlEPQAl1j3A8sU805g7PmcvET3hdpM+aMAaPoq1Hr7uZhQ9+jDevYFoNj3vnKM9lEbJvQIvyz4nwJ49H8z9PWfyvT0Ml5g9NuOTPsZOKj7F5SC7wLDwPY/E7z1/AC++0aJfvjrSNj57eG09tdAMvvvXW74Zp9I9sRiKPcvta71C/uc9rJzXPO1vjL1eXae8gaeUva1KhL3FOoA9FlagPYuzAz7VnIW93KEgvT6Nmj2DLZ6+lAfavPUwPr3tvls+jri2PY6aML0r5/a9g49HvfOGbD4XOzI+GAWhPT6DJbzEUsE9kqkqvb2s/T0wI38+6f6VvWEVGj0K07k9bKAHvbyXDT0637E9s+cQPS/s3b3qIxo+qGsEPhQQIT4tdeW9NIoQPtHfm71cD5y8geY9vte0qbzbWMs9N12jPN45rjx1fbY+IWCtvZ1QUz7P/bO9BHAuPj1g3L4hAG29dTlqPY8Noz2kOT2998QgPW+WET4kDDc9G17IPdpqnz7BSc897UfBvYJ/qz0bMJC9/U9xvXopdz31Gse94KtPPV6DsL0nGty9","Z2+MvKMO4T0h7by9hDL0PERUTr2YF8A9iHDnvF0hmj2s27C8fQwvPQyShD0gSwi8nI7xPF5vg70rlBm9y5sJPgF8Pb1/puQ99DyHvb/exz1wxPU9hhSlPTHylj3eua+7dVSJvT0gtjznVF09qJJgvX+UpjuF9DA+y7lavZ4KBT2wso69Bd/OPNLxc72qROQ9KKJDPVdKIr5V54w9HQ4APqA9tT2fymk899PvPafhM7080Fk+UokmPpPl1j3GZo29IDMAPrl7xL224VO9FRsgvs914j1CnwS+SJ9UPnaZ8D1fYo297eQPPuZI7jyqDKG9jh0svu/xDz4cWoS7SdGzvZSwQz629pE8iDJSvZQ1DT722F0+t0DrPXyBNj51oIO+jThFPftUmj0wzac9EP6Avfz6vr1LIbi8KM1hPL8WKr22jv29q9VevSkrYr1ZkBO9PiImPpwACb2x/DQ+moCJu0NI0D0zZhQ+0QmkuQai9z1bR/m7aWNnvigCr73RD5e8WSL0vclN2z3So3s9iUEIPXy2Ub2LHLY941SNPQ70Jr6AuMQ8ajAfu89sxbnCpCm+JtimvWrxKr5n0gs9YrkePHHkQjyn3jO+GAnYPQ8WRD5v3Kc8/mNHvMrtujvSXDC86KgZPWHEWL3w6BQ9zhAVvgadHb5AEgS+5LCrvcaVFz3Qw689pnmlvpBPGr76MSE9I2UjPkPvXrweOaG9zdFYvdtXA75fzjY+vBkPvgqzHz6BNwk94cUxPa8wkLuUvFI9XeucPSrbLj4P7zm9sCbLPZ6uqTxhEiw+p1s9PnABfry5BOG9kwYRvhL73r2s/wA+YyGpvW+TgD6y7tU+q8Kpvg8D8jw4i1o+0tgEvg8pFb4eJ6Y9qqYJPtNNnTx0+vM9etiXPaCHcr7Xlwo+YEGGvZKnHj71tS8+1zeOPpukIL6boQ8+LAl9vKqCmb3UjpE9CUWUPr3pf73n1/w9VwpevZE3Bz0U6q09fDrJu/Ktzrs99KO9QmVjPRYOS7w7SIQ9","7mZLvXrNCb5m6Wg+Zu6Mvks2w77ptE0+mCr0PXq1Zj5xU0Y+AEn/OlIbg7wFW2Q9etR+vQtV4z2Q8s69TF+XPZGIEj52F2K96z64PJUoiz4/KZO7NU17PfQiTr20ltQ9NFFRvUHbBT0N0qg8KNYpPtJaBD4Cem29J6IjvpRMMj5yrCe+KHPRPX1Cj77EcYI809e2PUGCaT4s0K09ChiYPlWlsr7fTZs9d6l9vmpYCrwV1SC/ncNfvSryvz3h9V+6hwKDvH3CKz57P5u835xRPc86MD7aTiI+Dc+8PefEjD0indI8PM5APnrDgbx9sZs+1yiKvjlC8z24vaU9EolLvbnEEz0pNkw8DYUAvWnbC728Zxs9NhOSvSKuyL2I/xy9CSMlvhXSRj3po2o74ytyvYfv7r1xjiy9knwZPhERHT4oENQ9cPllvhG6eT0mUkY8PLqTPYrMh7wJMva9RCpEvG1pFT1phKm7AhyEvriu+70//Se9yQkGvtHFIz7fO1u9bjrIvb5Whj5fcAe+Tx7vPNDEqj2QWRC9QdpJvBf/NL7zPI+8teU0PpY8OT1kxpc9ns6dvUiAib4ybBk+t5AfviX2zz2H93W+Qy0hvrqUAD46foG9/AKKPWNZO7666wO+lJHDPWbeMj0NVJU9mjUGvX1/jL2vgZ88SWI+vrNphL26iy8+6ZKpvVOkIL7KshQ+mSH/u9kCfT0xSca9TTOXPIgvD756aQY+/pn8OzVBQb65q449DWMbPRwe8r01yvG9pjqkuzoJq71fwtU9uiIsvtIhQj3cDj++/Tihvda9h72zq9y9xjp2vbe4IjyDWxO+d3+kvI/3fD4n/hA+VRetPSyRIT6VA1A9Ke8avmLtxb2QMlA93DikPQ/Sxz4+RLi9EItCvqlj072UoFy8/KlEvIDDiT3xfhG9Imw0vcKUDz1CSRM9WdEGvliYD75LCRw9xigFvmmOJ76BrAe+aWAavlQd7bsppzk9Bv9/vqhLJ77JRZk9OyMhvlp6ST1hY5g8","wQhQO7YKYj66HW89YJ2UvWF3AT4zbUy+JGeBvLsKGr11X0g+St1rvtKXjb6ZAOK90CVvvdDGEr1Z/9q91PnCPbUS4z2y8Ui+NgMwvkU4Qrxcbec8XRLKPQzDLL1/9mQ8wTPnvBBFCLxS/Ce+GPsAPnQKYD4uZTe+26KCvt6Fez40qP89YqhzvWxojr0Dvw89QfoCvfMXpr22Vs09H7c0vqfP7LuaP488rMAtvv0tRrz/1DS+Z6VuvgDnAb6NGIA+dsZNvlKghLtGxTs9NBbMPQFDf74IP6K9WbUDPqYmTb0iYGW8bmSOvDUCjjzUpYS8dGd2Pt4h1T20tre9ZN7WvdKnu73aqhu+bl7UvT2bPz7c2LI+SKvivXQb4j3b84O87NR7vmdbTj0f5ME9G9UDviXrtr1P3i4+V6WqvD4I8r3+FnK7fz/uvf1SLj2Z2ou+19M0vivMnj21/ru9dCx8vTgeTL2eFw2+XTedPX8mIL5t/AS+HXZdPf1T/D1eAwc+r0KSPUkDsrwbP6I+6DZ+vt+pQb44JcS91ibVPPCYsL64kh++uyYTvuFXiz63LYu98ZgGP1iHATvngl6+Ec2vPV33A719vUi+356Gvno9+z3h0dS9BHmYvlL0Hj4AdMe9oVc6vmdgnL59xOk8qMEcPPBnaL188ry9dfcgvjr6Kz67SeQ81e+CvUHXkb07NDG9xZuOvR1fAT68Ms09W08TPmq4Tj1tV9U9a3YRvWWFdrzyMw4+g6p9vfTNuD3WQ647Sh/qvVucFj5/ree9hSvjPZS4NT5CfVE8FgzBPTAq9LycVA29Ao8WvGXKPDw6Ags+pcYJvnMZKz7G8WW9EyzGPLtxgD0qxf+8Y5JfPSeU973Jy4E9amKevY2LhD0xz769zFnSvRr4SL5riP+9EZgMvVhdXLwKubY9lLnqPUic0T0ITUa9gna+PQbj97154Pg86676PKecuL1q7te9Htj/vcslBz5TEY89EFGSPC2hWz2Yi4k9uKFDvTsZYT1Xw4K7","o++lvZrGXD5yFsk9VoQ9vkjDLj7EWMm8+JCJvQdE6LsXEnS9Rw6lPU272T3hjzY+OROnPA8UA767pOE8ScwSPg2r+z13+XC9GjxrvolYQz7Z2tE9tEvLPfDsxD0qYhm+ItSIvTjGML5dCH+967IgPkzEnbxbCie9chRXvFXcKD6kLOc8E+NzPdT61j3mkx6+rhBSvQpejz6d+Ha9tWDRPeKunLy25Lm9YqHbPLmuiT1FCdY908ZzvWBJ+zxEAN48+mgdPo2JYz2veda9jmsNva/6DjzOljs+P0TSvLxEyj077Fw905MNvM1gDz7UQFc9S6zhu60EMb2mQSy+UL8UvZnypb3Otcq9TncOPmBvpL2BydW9GYINvWo7Tz1Qi4+9G6ISvMtXIz0irVM+5oEpPil55z3GoNq9B3WWvVAgoT0t3jw+iS9lPTDVUT1V6Oi9/I1+PUXEsb0eATk+mpjNPTLeQL22xJs9gzxXPsA2DL7EFEM+Ev7oPbO04z0MXlK8531dPnEUZj3u2hw7kPA1vogzTT7OF789sF5HO7bG6D2KwBq9ctVIPYd4ljz+NlG86JuXPVYCJj7HrRm+5gKevUZKar0BiuQ71gzMPcxl4DyfKsK9K+NGvOhWRL5hfs678bYgvU0QEj5krzi9/UR7Pr5iob3QQMO90Ie3PduuBbzF/AQ+xHj4PXNIIj4n6y09fogGvtRaT7zwHjQ+Tj0UPd4t4z1d+ry8uzv1vcl9Wj5zsDW81pO7vXfnWj6aDUu+mHAHPhS2drzv05a+8/RtPhcnUzzTIn89t7sAPtXAq7wacma+E+2hvYoSLb4MtzE++7fYPKikyDwnMB69gWySvfYttr3PJMk9NrEZvmphnr1yvaE9drUNvV/+FL0GfLo932IhPtvnqr12TtM8gcyMPWOmAL6W54s90P8UPpJytL0IZIy65DcCPu8Svj2s7Mu96EvxvUEQdD7lvEi++o+IvN6YIT4MPBg+S7mOvNTzpz103Lu7n2MLPiAwIzz3+769","yMwgvcA+iD3QY809nBm9vdPLAr7Tmhe+nhQZvsMtUD2yN827ujfkPT7xz7wNnjM90nuXvGdhhz0ZXQu9widOPeSWJD3Rn4w9DeT7vYfaF73YUsw9zv4EPvydFz4Cr7C8xMtQPk6knj1g70m+88TEvSzOpz2iQ/u8YXruvU11ML3c6Nc9rtfGPTsEhz0G/Q8+DE8YvaxA771BU649Vq72Pdy8ND3r4yg9386ZvRLaur0+WQK9iRv1PRvIPT5Icze9ot/KPGg2gj2oveA74JQNvsqcmr2aHrU9s7pbve/7QDztsVY9R3fGO7BbobxAMrg9qTjNPSDq2DzkP4a9wHunvbQucz3aCoW9eGajPerstr0pWz8+N2PHPSASsLycgTS+xVBRPS8sgz1MJuq9XC4SPv5DDz7CUGc9lWtNPCPZrz1SlB8+inbxPeCNsLwAywk9fS8qPuQyirzgsAy8hiGDPvySpz3zwxI+h8ggvSxr8T3lW9G74EmuvdzemD3Uz2W9oV8evsUUrT2Vmr4+HEQpvflOGj72pLy9CP3PPSScPj0e8jW8VEruPaVi9D30Mrm9JcipvKpbfrzNmhE9vefCPGYEhj1OjEq8lSeCvYsTHLzjtqi9ysAQPvnmWz0y7gw+XwS2vG4ltr2C0Ha8I3ryu8OrDz3ly/E8kRhiPpYy1j3hnLc9lE/MPTtotz3Djwk+9CEEPkaNuD3EWZ29GDoMPqPLKrvZf5w9TGfLPOLeSb1DOZc9jBD6vYM1Hj4Bsgu+VRLUvdDJgT2z9q49k7wNPkPRlLxC9rw9k3AaPmO5mjjZxTm8hF57Paxavr2Dlxg9m/zvPXXwY7xv9wE+f5CJPnM06Lr4SJo9af6lvH/Qfr0xHTK9OmN2vsq3ML7Gufg9bWM+O+zpIL6NZKI8K3obvu1anr0NjA09cbcLPqU79D2MffM9k022vMaltTy7MyS+fc2gPlYMpj0b/oe7+2xRPlyz8bxGbVw+jIwzPb5mAbwrJLQ9G9wPPpc5Z721Rau9","uQ0ePlMIcT12B2u94Iw3vn1QJL55Wqw9kceCvZUJqj1/1lm8heHWOnoMCL1Zubs9bb3cPlyDsj5lTBU8wNKhPR42HT5C/oE9xt1rPqQRcj7tIie8jdvgveN5Fb0hUqI8nTEjvqp2/z2nnGg+iwM1vc8yVb4vCaa8MfWqvdsNQzw+nzm+MJPiPQkpcryhcUS8DfgXPhK8ID7E8cu9HkagvIdUV77lH569mG8AvQT8Zb10P4W+FCjJPYTCkL22eJ+9tRf9Paj4pD1/pKC8+YhdPQ9MDL0icCU9MzAlPH8Djj3aw7k9YV/pO72jC70Hg607OBszvrKjsj3c+x49QH1EOZ6uo7qg1ui9hKw8Pfdglz3N8gw8Vln3PdFxqL3nNLi9+gwRPb16iL2qj2M847qQPaMml70HRJw9x/e5PXqPjr3FI/A8fbYlvn9p2bwSc4W9VRhou18Ydb1TpRo+9TEQvlBukD1C04C8xYbXvVbrsLwmiKi9XxeAPXTsaz2ovgK+BWX7PccM5T3KfKM9cbQDPtkWSr3/txW+JChtPTJ2Ab2sjC09lLqFvJTQFT5UcCE+qROwvXNBDr6fNfY9iPmwOkXNJzsaK/m9w4hhvcSGPb4D2ts9HBLFPcNFij0bk4g9QzIWPj+KXj40pzU9OBC2PWXuG72Dv/S907UhPdIL5T1aX9m8pkqIvb6npr1WBgu+Pm8oPuYdob11QPs9AnViPvEoBr4Py8a70X1HvbODlbzVEAU+EMisu8v1Jz1kstw9wojfvE10gT3Zqju+s0ZdvXws1r3UI3A9romyPakrBz7P8cQ9Y/mxPno1pj0c3IQ9zHe5vCHonT25bK29cE9QvSfZpj25CO88uWf4vfknaL15UFE9e82LPFqdob1pPOS+dCkgPpvEBz7YvaO9klPWvowbHr1jdei9Pw4gPrMvvb1GKr29EXIOvCs0Jj23LyK+XlaMvrFJXLxZ6w2+OX7rPO9aDD5K1wG+eVA3Pr3zlr4XBSg+NXAFPmIz7bwfFm29","0LVvPqsoJD0wUPe9cB/DvMr6jT5I1gu+rAYIviFBBz0y36s8qOKKvrJ1cT1ozG0+Lw4Nvs3woj2uQUu94p9IPQwRWL7Q3DK5mdr4PRlzpL3SJMk9mcshPs93sb17Ng89ST67vNSG7L0DgB29DicKvpdUhr0bA32+1Db9vaemEzwWfOm8Z9bmPQS9sTxwgko+lt55vi7Ii7wjutY8Ef0QvjLZH74yxqK9IKTuPdW1p700zaW9Z8k8vBeZZD1QtSS+Wf6xvRSOtjzp7Si+GAlUvUaHGL7BNXS9DG/mvSLDIT2drew9c5mkvUTDI76It0y+A0KNPWJUNz79ZIS9zjkvvV5E5jznKd8971pkvR8RWj6NjZ4+1JQsvSJEDr6fc6o93QJcvda5uT3aejY+NlwIvaLTMD21sco+j8DFPWsgpT4iCNw9Nts6vdehdT3YFVu9DC9tPFp2hL50FaY9pr7JvX81dD3MOYI8U8LNuhhMNL5yyGK+6amSPWeP0LspYZO9lZ5/Pq0oGL5cXeI+R5o1viFmU73tgEI+9n19PYT/Ob4g+X09ACXuvb7YFD5LrxE+Wqy+Pp+aXT4m5k49d3vsO/Fh6T3yrhW8chSCPVo9zDyzZSC8IDI1vplKfD0vVdw9ct8RPk7OJr6sTiu+z4elPSTlED5m5eY9PujmPS2aazyKnng9coALPTdfY70NpMg9MuV3Pjkn9z21dxA+Lb+qPDgfL77whAq+EfUHPkLtcz1P2Ne90bIMvk3x2D1UT/88Tg6yPVQ3Wb4pvag9U115PQGtJj7Bj0U9+26NPM7tmzz5fAq+syQ0Pn+lIL0025299O9NvVtEmb2kP2Y9D5D0PWEU8T1fYd89JkdHPdqdA74LTnS9GEd+vggboj235+E9OabDPT+/671Wt409uiDOPWShTL1Kgl2+EbgMPYQE9b0WVMS9S1tIPkjwoD2GzZc99nBcO7xbWD3A50S+Y0oIu2/Gtb2v/NK9quy2vahsnL1FxaY9pd/Fvf3Tzb0mgH08","0yKAPgNzMz4+1VO977HcPS2Skby+AAS+d4O7PWOQEj5HiSo9KdCnvUssurwFnx+8T0lXvX//Bz4anMk9mnOrPTbaFL4Me/w9hN8hPpJXkj1PbgI+WXOUPD3Ru71nEYu+DEovPj8JDr0WvgC9ibCzPdZb3TwsPkg+ziZwvvCmQj55RHA+TC18Pca57j1nxIq9kwgSvsNNPz4e0yo+w8GYPqbIzD0+tC89DKDAvXqC5z0QN9W9DmChPlid7D0rs4k8IHITO+CEAb53O2g9LanaPW7Wcz1jsS8+4KLOPU29lL1jW7E9Vsf5PaU8GTyBsa8+hc4JPmfTA75Lca+95DnDu0k2DL3GxMC8pYCEvbGljbz5VRO78HElPnmf5z0ac9k9Zi/YPWCPhL2k4rG9YHQ2vqufTj4wcyo9VhfgvYdg+j1u6Za+jCDDPJjhmD1BX/S9sS3IPWu1c70fyVI+JloQvTIFGD7EP/a81uBivmbTdD5uTaK+0CkJvdnADj6K1bO9KDJePuJbd7z2yLq9HGetvXPdmb3OJZO9tyhivfGqg75BWgs+dkmnPDBUub0j/yM+6gdIPkv4F76jEJU+dIgHPtPZLzshyeG9mHoHvp5/Lz1QJm+9sf6avSRlhT4pHOw8csHovYZxgD3Sc4Y+rn9bPVr2S70k7JS9X8+yO+wWSb3aPD+9ofAqvrWp0b0Xtm0+9HRIPWsnEj0clNU9gwu4PbX1ob2pMjM8Q/8QvXcuKD4wDcU9npGbvt4CKb65GQ09nGFFvYNf/ju8pQA9eYedPquIEL7Dqb++Yb03vqe9j72ZsgW+igoBvVt2Rj5gAXq9CzKVPXKGzr3FfF4+GZhIvSZziT4LwVW970awvPBCHrzjWJ09RVlZvkSarTx17TW+V82EPqrWd72FwRw+WxwTPVVK3z06nzs+99SrvLN6NDyhX3c9nwT4PFfVH7xxPJc9NPs7vvIkiDzco9k9zICsPCoPaD0MSLa8f2D3PYAcM74szos8h8j0PdIs/j1Ov509","c94xvAqSJL4IiHW9DeLmPdJZe70fPWO8iqkoPb+MFr2ZPK28MNg+vf2MhT36oFG+yGQSvTeSkD0aUMY9oS2HPUaI+Lz9VOq9rkowvj0nuj25Gh0+Yym9vQ+wTbyCJRa+Et45Poa4gz0feaI8e7bcvTvP9r19KFi9MF9BPrt0dz1OTUA+MmV8PMpNLT7NDxk+q1rHPdPteb7TWxG+YBa+PZ7ez7227ie9UmWmvfSF072/Kwk8DXkJvplNdD4UePq9+A0LvW26/r1MFha+qoAuvg+exzpjNJE8+LrjPZhAVruMcx0+WYmuvXNjpj37spE8etn7PcVyTL4ccxm9y/j0PTaeHT6SsRg8PrJPvjRDY72yQSE9kyzAPB+kqT1zz/o9FvqWPcPPUb06+wM9VMQpPInmXLyMAdC9kHCnPHKWF75E6nG9MKjLPWec2L56MEg9yfx8vUeb7b1xi3m9NaMnvZIpFL6C2Si+6gzUPYinwr2gOrk9y6t9PR/eyT2NEUE+2FgIvTuc1L2f5fa9vu6OPeActL1idU092O4zveK8h76ZEBQ+Ad5HPl6C872iHIS+ijBkPStcAL7Lc8o989DxvWE2YjzQGjo9XczBvZjIKL6CYWa+4H2cPGkhGbyfE6g92sipvMYSTr3RaD8+p7M+vol9Gz4KFrM9I+z4PVrcnLu1oow9k65sPXpLGD74ldK8HKSIPdDLhz2yWxo+XB+HPYvwW72+hpC+apWWvG8tkj0Yv609yhcBvpub1L25wcI90MQ2PQ1YWT5sW2U9K4gvPQkzG70WIIa9C1qyutvysLz24eA9FdEPva5bOjyltdI9RuiUPlpynr5pSj8+JXjGPaL+pLwhdAO8ywPJvBtcKz792/i8M3ZBvipDdTyoVKi+GdmwPiYKQL4KjDS9z3dMPM1onz3vLia+w9yTvjNRGD4Z3ma+F/DUPXWNjb2sdY29yLwOvq/brD0CXb+9uuWOvegt8j0F17O9xE9rvo1FLr5PPMm7v7HCPHVXHL50aom9","WK7FvdXcYD08HpW9wLchPreEmj5bNEG+CvTAvJI/fb5iUio9aIsGPj27AT425Zq8gyfWPlF6Sr4+kJw9xD4mvXDQnD1B1gq+zzvEvj/aIr7q1YU+9GWtvoIkdT1nxJA9TU0AvkcMbL2JQ+09TguSPEt7Jz0/V6k96mmKvp8VcTvf/Ok9e8aJu1Jz+T5sLBC+No19OxunE73q2VW9Hr/5PWTdFj67POm8wrQhPgov2byZiMo+w67du1lGZL2CWXY9LybUvAsFpLwRKvm92supPNwqqr0zYu29UCUDvmZDWr15l8O9nmagvQ36Bb7F1De+Gv+VPW/b271JUd29YHvQvQhHZL5rpFA8hczvPRRh4b0g6wQ+0Ze6uofqKT61SmK9NXCfvRaClj0vKum9YYVlvWo31Lzps6I9MLkbPilLJD6Vo+K9a59kvT0nlbvjB+w8z7ONPcAZz71wTom9Vjn7vYOgvj1c2/A9k7Y6vg2z3L2GnxW+KlyYvUe5iD32cDk9XcuyPVb4uz0buPM9Y3fKPWAGwz3iZIw8PoEkvmEaGL6M5S47ZqWsvSTUor3v+xU9esD1vFQbj74e3GY+EHI0vL71/r34Y2K+2N0bvfaONj6Rx1q8N3I+PcUo472izjU8Db28PC7o5z2imJ69SSwCvryhmz2O+409MjwGvo2aWT2N5XE9hxQFPfILYL5SHsu9oEp+vnkOkr35Te861ZAbPp7Xyb2EdQo+vFSgPUGm6j0sCBO+H1qnvePy3L3D2KM9Zl6/vc5Gfr2zHcu8xwVsvX5g972g+ky9dTQGPZsD7r2oLYW8zFIUvmJII71SG9Q9AlG+PbCIcTuEzXM9duyYPX2guL3rrlY9lKclvnj8YDuvrKS81MFZvfvdOL172UA99TOwPNTMijo6hRC+gSBuPZlFEzzdAHo9/LqfO6CF1z1yGdu9Uu2FvV2/Jj0xllM8TGANvtYXij6GYFI8CxEjvtF5tDsXrbs9y2s/PQbQnb4HALS8xUT3u30IcjsDExa9","DKWkPcYb3L2LCBE+T/jEvS0EBT1fIc89GpEkPhqxvD0OH4498Gw3vX9n9b1spam9XhxRPo2lFb1EAhM9nfISvrnROz1xjpa9FQe6O2PSLD3XDBO+gPgfvun+oj2AIBs9FBApPaH8ET43+gs+6jmZPUD/rT2evAe++bf/vTG6mzudkJo+YR9mPTmDLr4riz0+pMbPvbbLPb6TeYu9Qrdcvk0OKj6H7Y8940JBvs7zuTxBt9A503ACvmpp170YCQg8JXHsvYSnoz2Jt389HYMwPl09Ir6NhRU8B2oWPiKOPD5h/LG95gLMPE5DiD0FWZO+Z8+NPeib0r1oobS9zNJCvZXR5L2sFQS+BxlyO5++pz50BBq+F1qpvcyJFz2Oozc9pGxoPSdYh71GW6O8ABIbO96g+j1uK5K+bUmpveggzb1914C9vWDivA1Wiz5oCZ6+QxHFPB1A4j1bytQ9xHmzvXN297zQ1ou7mgMPPtB8F77ykgS+dGbCPSsWEb0An469is+oPfoWW76iVD4+iyBqvo3Mob14KO+9eqA3PjFVOT4cIrO98Tsqvjz9ybsKbrQ92NKQPoxiBT4QWZ699QSEPe/J8r0QxjC8ab0yvqcCxrvjEF69ZHcrvgr1hL2mKRY+lUgGvlZAl733w1o9zgozvu4Fq7wKe/e8zj5bPbLJxTzKdaM9otcIvqa42rtk9oo97Ai9vGQ8Ib7t7CW+vHRDvUaynzyt57k8p+KhvZPqJj4zaJw8scwmvkjF2Ts50NO988yEPfv7lrxesiG9qHL4vdNMoLw2Efc9liG0PUlGAz33mna+xqkWvVcoAj6ouK07BFrDvKdOAb434mW9OlDHPRmxNb0YcQW+8MnFPVpuQz3Vc/49X2qRvXCC1b1RoRq9WWecvVnQ5r3+pAi9RUDmPfEf/r3yX7Y8CDKXvmqpFD1vIHg9eaxGPlmPqr7VPYc7z3wovspn1L37MgG+KmBRPQ4WiD1r7sy9i4dtvQ4e9b1ODyg8fIo0vQq4J70V2cy7","87EPvBybgj0A4Fq+AvxlPt9fqT2fNWE69BRwvp3Eq77Zm0S8rBzPO5Yr5D2GX6G9euQQvuxtqj3uZEM9aT0xvstJmD2XHGi9MNNMvpgtoL0DJiC9heA2PbICLL3ZufW8BHlJPXyW5D3kGJG9xKcFPv1gpr0/nfM9OExHveGNWL6F+nG+D4v9PEx6Tb3gR3Y7K5cJPhI9Hz14Qge++pN1PuEA3jzNrAS+3djNPF2BNj7m+vw9xMuIvXWJm71k+tO9yv+jPSbrCD4qnnE98xdCPuomGz6jZUa+vpCbPAzAUr3UKp68K/YLvh+uAj0BQsM+cJVVuxZsjDxSWUK+ReELPjkEEb54uxQ+56YTPsJVv702E7y8CGbDu5tYW74JPgI+nv+XPSXuHT0ZQ4g+VlIBvi/MJb7z6ts9/Ko4viaEGT42Vgs+oR+huzqD9D3yLD+99H06vT4hZz16BfO9c6VOPdHgLr5gcoO9AMKEvdRWEb1BN489H5DGvet+Nj4+Vt47Ux2DPVYnib2QShS9qbX4PN+rk730Zp67OVuLOpn87T3T6AC+UW0qPXa4cT1Q1nc9yfB+PVln5L3vZSs+LSEGvl5bkr1H2Ay+/5Z2PXvVAj78aiK9U4JgvQiseTwskgk9QjNsvMMrtjw3pog8DFtKPpVb1j0f25q8ew2svo1V7zza1By9AcJQPRe3VLur986+KGoYvthY0j03gC2+87AJOw9bKb0ijQi+LcqNPIeg0r3a9Iy9rdjIvQkLuD2SJz29MqqWvQVhN75eWMu+uXfnutdYGj4lW/c95qUAPugQPb6JdAa9FIGyPfd4273IGm4+8YGTunT+p72rBqi87Cm5vk89kb4Uqqo9lP8Dvxz9IT4W9KS8iu+dPPytJ76/r6Q6uj7bPAbYH73Dwje9lMa9vA7jir55knu+aY8IvEgNSj7PX6K9KC8pPj3ZuT3v4rk94O2BPSO+XLuunLa6Zx0OvjAMxz1BHgi+UqwxvhdVgz5l3U4+mm6tvQplnD3okOC9","d7PSvddr0r2nj6m9/tYVPv+xsj22Uce981cHPvwb2j0Fqko82j0KPebZIj5Mb2I97Xf1vQH6Xj0HpYK91WWhvbiAp71neoq9GsYCvYgTsD0VMyw9fAvIvWSER7wzCwu+f3NjPuWb8D3zHF+9i+zmPT6lNL2AEfA8CZqJPPAZFT7lIhA+y8pPPDVxIrwB8jI9c2cPPsX1Rz4U8ZA9TzRoPEUoD72pKZ49CEdyvd9NMb3Dk4c+YE6JPq/IHjv214W9vSc1PZSqCr5nybY9z4TRPbbcFzuuCXA9L2++vHCt1T15bNw9LqyDPAHmpD7kIdy8jdOtPEuZXj2hcvI9SGyrPYFcUj4EzWu9YNx6PgHHfb07cE0+JlEjvBG17z3cnA++6oyvvLV8fjvK9HA9BiGzPApqCz76s5W8JeeQvMiF9j3X4TM+bbRQPbMEJ7z7V3w9ccCLPkThPD6eXr+9N9gjPnbUL70Vpbc9uBIsPivJvD1TlMW91hdWPZ8W8TzXf689iC9ZPZruIz6S5JY+I9t9vQvXFT6bEMM9L0YNPiBiOD4ZlWK9Zei7PcW12j0zlJQ+UA9RPZKGbD5jZPY99gG4vCHsH74nruq9CGzLvJj+bj3+QoE+GcFIvG7kDz1OTi8+Ht6cvbYiGbpwFmY++9nsPfvAQD6V2b88BUKcPvESgT4BOP09EVevPSwELT20eii9dnFXvMtYor39YeQ9E4iuvagXTT6oX3s+bGM9Pb6r2LwFpYO9xXfvvaRtAb3GRfk8LTFivpxtsj4zblA+aGK0PcD7hD3f1NK8CYosPtR4S72qphw7EgCrvf4+LD0ExqC9obCkPAJOG70kEAW+CcecvX1QKL7PDuu8eJwjPhxZgL12m+s8nMO1PeBhMryxHZC9srCDPSacCr7zlck8kZt2Pg9/Mb4xYbo8FDFyPvPfvb1nCQQ+fEZbPvvNIr4S9Xi+GQgrPpeYgTxhzCQ82hcvvXlAhz2cuxQ+GwKrvtl4JT61b04+/rxcvRDhdj6tRiY9","+nvFPRojMj1AqRW9MK5gvpRPqD2FDc87JJk1vc4GVT5GkYg9GnYHvrIfwj1SPkI9UEhBPnLmOjy4zR4+P0CyvTiolT3h9Z89nGrUPQiwkT6E42m92nZAvb4ZVj5o40C9xhE1PmyEL7zFy7O7xjKsPLRaNLxk3FO8icbivN+uDT4nbHS+I8kTPsJIYr7sT7o+gAU3PtccLDwI09K82wEhPtCygD7d+ro+0Gqeu2nnXD4SdSy+6ihFvizK8D1S7Yq96hA+Pd7r7z28+xY++DF2vU7VDT6U9t0+UHxtPj5cYD7SRNs+7eMIPbpasb0i2Di7frRjPrGQSj2Y6Ka9VDRJPP0EibzsdwE+/XUBPvd5T706G+Y9LMQKvmvUrb3+xiW+mqkDvoTc+jxbkWq93nOEvXn33Tycoko9Wvh4PSCcNj3gPoq8tRLiuZc5NT11rsK9pxHNvaIQoT2yNq888e/ZPN5EG77bIx6+4yr9uxoDNrxrKYk9iHkLPoprsz3r7i68Wm8vvs9q5z0ecxq+Pn0AvpYAtb37Djy9HpzdPafwDj7uQM09yAEVvsufKjx3eSw9FktAvroVCL7MBAE+n4xtPnlrGb5hIxG+06S+vWUHWr2KHqI9gFAovmxZVL1wyKU9x4oYvhxgITzU7Ay9k9YovfxPCjvkxCw+UIKnO34KyL2jQMy9IKPCvbfYX7yAZKM9UhDNPahmL767DHe+2aX0PdaHNDt356897KIfvWOggr6D3r49cQiFva+PcT3Mev49T3H/PTgiUD0X3+G8mwVYPYa4LL01fp69RZYjPnj/MDw9F4U8LzgMPXtZEj3Sr5M9lxA6vSdRTT6zwX67Rk02vdurgr3M4dw9yh2mPVHr1L14Vxw93gRDPWuFYb3/Fgw+hHlsvTRXMb4lKPm9SxUEPnf9Lz1oWwy9tdKcvQ/pQrzv0hc99gOdPZstZzxf7Sk+Bh6FPd2jIr53YDQ9sPsGvn/Fnb1rKwc9MQ+avaltwDteqg897uWAvWXnM77Tjdi9","gVCDvaffqr6Kzrg9omATvZu1CT3GnDW+7dAnPKVeWb3B9dg9c+oyvimZIb5hobq9eD6lvSqmqzz7r/G9yo4EPp+BFz6RvUM71H5RPkdJTz7ze4s9Gi80PaJ8vz3+N5u8lCRLPXykQb2hCaW9/niovW+sjT0c8JA9JNzfvQHDPTw6JZC9zoRBPnVdy73UWfk8v53evVx2cTt4UDs+3nDPvfvZhL70zag88uoHPbvewz2Hjgi+ZQMtPNh0g76EPv88sve3vVIk4j2UrqO9Fg6ePjNv7r0LGv+91zoVvtBYDrzwjg89JJEWvsQ/Qj3RVh+8pI6rPftVuD2L8AG9V7IVvXx+yrynuZ49rbo4vrjo+zxHWUg7bqvGvOv6Eb5c8es9a+QaPoCvsTy6QY895OahviztB7+aFwy+j8gYPvZpVr4vhUW+5LIIvoSM2L0C762+EnaFPd8grT4acdK9RRuzvUt5Ej0YMlq8/aajvoT2gb0Sdh8+a4CqPY1XOT2PQ4i9dJMJPotjwr0b9xi+1lXFPIhFtb2wH1Y9O/MqPt0WNT09zAc+WvUHPdhW2D2KaZo9G5+IPmCY8T1eXa879EENPbnq672z8Ti+H5zOPaDdo73i1Kw98TWbPYVqBr4mPvm9Gg7KvQ7VoT10xBc8JBV/vPclvj18wF+5xl/AvI82jTxjEJI8rvAqPnijFz3o5Sy8ThU6vRCsLT46F1+9uXHPvTc1pbyBSyK+BSH8vTcJCr5YlbU9USoSvZKEDD7yi+s848cZvqIBkD0efrO8LGelvah8yz3KqVe8w9aVPX09Yj7ZcqW8TgWgPbB+DD3qLJu92DokvSOV2rwJWU8+F0dUvW90CD1QtqE8Ua6YPeerwj3Vsqq8Ya36vYF1Uj3w6+k9MAsEPsqXgbwwWoa9AGAau2ogsL0arS+9ump1Prq7Hj3VPEA8f3ZnPZEPbj0KXlS+OkNKPQoBAz3QzKc9qSFavUji8b1CLIK948UYPjW1Bb4yxoi7D9Y+POxHc72fGxo+","Lm1jPhhWW70KzKi849qhvquvPLyMK4y9DJSGvUn/4D14rTg9VuyYPUTK6T35ILq9VbkjvSuX2r3FBIC9ZP+CPG7HJD505XW7RO/rvvney70du/48KVMAPsPC0DyNL5q8MZmtPVG60r6Ygue9HpXbPXAif7hrljm9ZqbpPaaGg7zVUMI9HILgPagNy739B0W+9h9jvERMNb2m9P88CRQfvm4MjzxGrxW+3igCvcEKir3rFP8802S1va/Jyj1NxcI9JrgTvo6TIz2HkxC+VEy6u8QGSLxI5Ka93NwtvAMhCz4OlkG9AXvAPdsdSLxkBG4+nweTvY9RF76yrPq98HwPPiITuTwFNRK+64tLPQx4Kz4Y8A0+0MYRvb4QpDwWt4y9y3pOPkpgg76QoCK+YvnpPcyha73wuaO944CtPfPeiD1N4KO9G9gDvZDPGL1m0Ak+jku+vRgHx7x1Iho9yshuPb1Ih7zNicg8H5Ifvi7jnL1TnBg93+4+vqfFir4k6iU+faLQPTHlM7xAm2693JmSPlHEUj0tsvq8WZuzvYjoJL5N0Jq8+1+RvpeiqLzwIdA85uNUvsofNb0nfeW9njUCPpZTK77iuFw9ebVAvTqbFj4vdKu+nn8evWVIXD1ojfq99kSBPeZxBj6/CUK9Gf+TPZtK7T1lHwY+9/O2PauehL1U+jQ+FpUIPUl+ab1Tj2A+utaBPrsZ+L0Dnrk8p/wPPpAacD2U4vo8V/gIvYtiQL5QuD6+QYGBvJnL27unmgI9D0o+vaqmrjyHcwC+sGQqvVfFtD3lE8M71GsVPdWw/jzJxRC7iE19vZQYDL5Z/gY+sREaPoTJjL0BCwS+tsiwvUTSwT6Bxxs+92AkPo5dij1e+z09ekr1vYyMFT0wXYw9If2hPvdg8j3ZCSs+qaNVvbElAT9Vfug9tnclPkfCKb00Pts9733Ivd8mmT4WhEw9/z3AvY6Xij06zym+bObYvSmnk7yI6828cY9bOxWNCr6PAJ0+nSa0vTsLhj3+r1u9","mx+xvYS8ejyw6YK96LQLvk5/pLz7ebq9pHSKvcgXNL07eyK9sfNUPWeHzD1sFh0988KZvFHJkT0s1oo9MJs4PVQpsD05zwa+rV60PUx2HD6uGfo8UB1VPdMFJD25+lE9FWA6PakKBD7i/rs7vMMiPuq/Hj4mOcQ9OVi4vPArV75DTHO9yOCEPdh9Qz1xJOQ9b+O9vdiI5j2l0J69JoAKPryn4DzuVW27biPTPSm1m7tFRf87ZOHDPV7FuT3MN7C9bOn7O8SEBz6sESg9j36DvUKVyrz2pGk+wERgPfrMmT3cUE8+HRSzvPIoZjwef0E8TJYpPhl4zD28kZY8BInMPBy7Q71sdcA8L9WDvZrOY76rxoy8ByIrPvn1eDzoDSw+DwaxPYmpIr3Zr6+9C4k4Ph1qmr0Uoiu+9x0VviuvCz21Mag9o0g2Pdwg4j1wHqW9+/RxvScnDL5pW7a82/HRPWfDj742fA69/M4lvN1b0D1/bOu9pHFJPXS0IT3+Juw9YgXpvVmeQj1kmeS7fRMiPpZ2fjzxO1297JVVvict1bwTqdQ9YZJjvX+cKD2KVoW+820+vtrs+L3PvoK9zFnhu5i1F71UoTG9LqjDPWvpNjsFAoG9AlYpPsiNOL0CK9a9cHuJPWSjgb3WcFa+CZGIvtx2BT7OeR48vOFUPRqm370G2JO9yd3ovbG7Ez32Ixc+luLtvcYTTT57Ts09bT9+Pay7tbyyWAm+srl0O73wDr0lTiA9ZU3NvfWabbxtlNi9nLyXPoK347s7VOw8RLYkvVP7bL0G2Y+9HdHePY3NrL3X4zK+kvgmPH+7Ej3WmoA9yZe4vWcuTz025n4+l5YCvUGbcj5xI9E6jhavPYaw17ybS3Q9OnEEvilMID6GQ7u8uiAAPpRSQ7183xS9RypQvpPF0j0Vr5I8pIYBucATmL5u6o48QYYsvhaHy70tL0Q9QX8LPZnbQj44blw9iOoUPYa/m71v0eC9vYA1PqfRSr6C5ea9UK8OvrxRvr0MURu9","ULxpvkmjNT5rAIs7ODpHPQRGEr66RDS+ilQpvu2ngb1uFqm6bJ0jPmIS4j3zX3q8FAIrvIDF7Tx4hXe9wjP0PTUPyj1lg+A8aM78vS5SQb3S0hw+KaT7vfTNxL3jToE9+4qVPVEMzT2Ydz+8Q70Fvko6Db7Ilnc9a0YBvTkIBr7VfRw+TEm8vQLmDD4W1XS+49a+vCDr6DzBNsC9vA8xPl4ubL1KVLO+VpJSveJ0UTweTQ2+qWgVPn/k+zxRlT2+xcWbPM4T4z3Fd2S+a8f9vcv+jT0lgou9ZLijvVuIg71c+we+yovbPZ1bCT7lFjk+OnxMvqyIn71nCMO7gjyEPKnxQL6jdwM8wjCNvX/uMT2mFBK91kD7PQs9Er4t7RI+KCUCvl4nJj2/XBw+4bgZvmgfDj4/mqY8Z60zPdnmMb7B/Bq+91oaPnsCir2kAMO9CxvXPTL+Kb3JwWE+geu6PU2Bqb5nbcQ8h0lgPqDBhb3YiWe808uKPHfolT06jx49Zp7tunE1zr0uNS0+BLGjvAYtNz35zZi8IrYdvY6uADyY1iI99kw7vUKGKb11QEc9CnO1PWRUsjyTg4A+9PoJPj6Um7y0vRO+s95TvWF4Z75LRxM+3J8Hvmhi+T3XFzw9+0C6vRguKD48eMQ9zKkHPMcPGr4AO2471BrnvGHLqrxXo/E91GsvvvbltDyrBMm9IcB/PcJ7nL3O89S9txeXPnIUy72a3oE8odQ0vsIR5L03GAM9p+sOPvndEz0YT/28Df7yO0lMJT7eobG+fafnPUWp0L0U4yq+enA/PXuGMb4MKr+9lygzvhKstT1ZnN49lTjRPULbQL4gsg8+RxIEvdpWCj5REHS9dnAyvfMUULz+YSG9QRIuvawuiz4XLp++kJ7aPYXdFT20Hsw9N+Otvjaf8TtikJC+6k7yO2KeKbyEjoe9iUsau7aMoj3DLE+948zXvVP4Hr6kZLU9ZXPoPUblkj1htoa9cS8gvWNCo77T2Au+ct9zvV5v1L3Z7nW7","YwWuvU6AUb1mr889bE9FPabli72WGJe9AoDyvaTGYr2L6C++Ee/wvRSoNz3Uv9K9W1bIvQRoCb5WLNE9ybknPIojRj51W+89Cw7zvWEI3r2pTNm9Mi//PQIdF7x+d189kMjMPZ/nlD1R1Q0+JeSYPbLUyj0fiWi9AkF6vm8dgbs7kOs7VgPRPFLpeT44PTk+Z3o3vnkRDb7K27k9C/ShPkJibL2rcaI6ilNrPXK/Ir7Uzzs+BiUTuym2sL4G4vm9wwAEvALO5L19Hb49cnuZPYtOsD1dz7u+EAHfvVkPAr5G5Bi8zfe4PXV/f7z4qyW+liWkvIwGhL2rTJs8LrHrPYycSL1SX5u8J4TFPTETej755xK+CfAIPYn41rz9Dx8+yRZqvSHjwT2rXwI+9hWhvPXCvr5XXPu9OYG6PWkhTr6K5Ue9biKpPapzCL7hYfU9f5CiPGM9zT7VEUg+RiSEPbmcYL13svy909GSvoVGHj279gC9wLxpvue/7j1966u9KMU+PjI7mb23udI8qFkfviQWeb3voAe+ZVJxPv5FKztiDke+sok6vufu6bvqLAq+hxUePqJmST7QxRc9rYsUvfKwu7wXhme9uxUdvf4flTwcxw2+odhBvooOc733Raa9CF1NvDvJbDwWvkm9EWwTPvqfVb6M0lC9lL0iPQpwvj0="],"recurrent_weights":["J2OUvdblN71QfzO+VgrRvlXQUD79UPU8+q7BPJnCRj6IRtQ9M8g4vqzXDLtD1N+9pwdnPZHdoLxMg5k+MWKoPUAtlz1Z5E0964o9vpkhqzwC7sq9q5ZePJouWT4pWgg9KGcTPtEVkDwWKRE87ayrutfqkD6sNjc+vi+bPGMSbT4NPWy9DLSBPkRsiT6d1ss9+lUKPtAcEL7bM7U925jBPR7XUj6ja608Ub84PjC+WD1w8BA8qdUMvIJ5KTyPLVY+SlsAPYAxhby70fi9/+33vQhWbz62ZC28MUATvuXnMD3GH2a9XzNYvaVsaT5wF5892dnpPa6WOj5pkie+d1AVPhHTyj1n3NM9IrSDPM5Dmr0Gixo+qUdKvb7qFbs4eli8cnR9vrIhwzxfT+W9kMW9vZpDhD12irS92ukEP7yUYT4PYUc+hjZWPgiQu72pVPQ8pt08vFEzd73SwNQ8seRXvjd8Fr0G5kG9LylOva0rIT1/0Ci9tCmWvdSqyDwRDOu9PJpqPTEcxD7hi4g+zDzkvfxW575pe4q9J2cTvqmh3r0sI8486pigPSaryr1teK08buVWPXnPAT6ykeS9+N8FPqYZtztU74I+BOwWvh/Jpz0tWFo8R3rCvHNCrjyyPnY+dtBqvBD3yj3WXOE8+6kDPtDZyj2RsDg+ZVsZvFavjD7peM49lZ+Mvu2zAr4r4Ie9eUc6PoTAODxC9a28HHkcvicp+bx7SWS9skEVvhrzYDu2M5S9IkRpvgUnir2HSmI9DAtZvpJfUr1eAgw9vs8xPnftuL3Wsps+956RveqkDL5xPqK8rPNJvle6x7x2jdg9MQrJO58yJ7z38/88447+PXsRzr2juZA9cIb9Pbk0fzxxyJm+GVUzvVbOhr2ziFq9B+VyvfGF1L2X5PQ9TBXsO+cuOz30Lms7p1DWvfYQDb6DfWs8wZJIPb+EYT6lxqA8qlKXPIM1+rxsuww93MxWvBtfWz0cnje9JUT1O5O9BD1wmT+9I0lbvb03mD0M4nA9","L5nRO1OESr5OwF491LnePNUgiD2gXJ+9J56LvZ1tDj49rFw+6ccrvLDlxj0EDw0+o0eBPu0M9ruBQrG9uicGvfdq0j6vbDi9Oj59PIQgAT/91SE8a9eVPWSNg71HCTw+DnggvWRaGj4frdm8GobhPWmmG75xug4+PtyjPlmAxb2ytRU+lmkAPNpaUT77DIM9x1VoPlimRTyHM3i7IMqPPTDcPz4IWJY+IehLPiGqJT5O2Q0+1HGAPm31gr2T1eQ+rzaaPQubL76MaOy9V1PuPUSPtz2pufa9hHRrvIdomr0hQVm8KoaRvXFtqb2QZQs+CZawPYScnzzy46m8CTijPjkGEr0H6Wq+56lRPeKsi76mf4q96qvvva+C9b1ld5u9twXnPfXSGL7WGgK+kyk1PVzalr65/P295Pjevhr1Ib33p669F3QsPUnfAb7uSBU8eixBvGz8Ir6X7U292yQKvdNOwb1VsLK984a6vfboZ7xbfFi+eJooPXpaML4Oppq+u+icvvmujb3LMCm+NZy7vVlqi76WApe91cYrvlCEBz3ObQe+cJvZvQORjb5/BCS9AJOcvcJMjb46SF++VV+mOmQee7zF3Mu8UOCWvTEVGj3rn329epChvdqJVr2iAYK9BbbEvT0Rmb1jzyq+9/WXPdu1Nr4XiTu+3bQGvrJryLz/2W29efEEvc2chr5QrjW+gVK1vF7Oljw0Ip+9wdyhu0GQKL6lHPm8v2YGvVSOBb5ar969PnqovW6tmr5+XbW9KXADvFPQfb5dUL29owVXvlNclLweuuq9iW6ZvVXbIr4rZx+9hp0GvlbWxb3+h7e7esKBvD31TL0oIPq6BGsBvim85L2N/Z6+ck0yvSImRT39VN292KSNvdGyor1LbPs94zkJvs79hL10W1C8RqYevuhu6bx9m0K9lyodvPmupL3P3ru9EzvDvVhAsL0qdGS84Hw3vuOfprwsZ96916rYvpuypr3S4Ey9HfMOvmNH9ryuKRW9TjusvWUkB77x0f69","K1SHvdg8eD3JuSc9ev6xvSdlJr7RRGG8Huv1vOxCn71TjtY9jp6VPXvVcrzMVLy9+XnDPMrFGT7Y8BQ9a3lxvHSxxL36Mfe9ctd7vUIISzxzqma+IfCpuv9vBb06LIa9DRSWPJ5aYDwig8m8sNz3vU9fYD3sa348M9XovQY/Uj6ykyY8t2Npu2kpp73//UC9UOqAveXlnT0KnbY90k/ZPDn3OT2fvpq9HGs+PRvJ7zvN9rA90I2WuxMaEjwCuKC9hWykveltE72h4oY+LjoRPJy9YT3Srmi9bRexPBhe5T3VsOq9peogPbhEjzyb4R+9FuGmPVlr/TzAboY9907kvWIJkbyWkVm9EiuJvflilr16+8k8IQDwvU9LED0wic69/TbYvKIsE74N71o9RZtyvlz6E75qcu+9wI0ovc1pirzAKey9BnJNvKSWXjy5PJ2+xwGWvKj0nD3VSH29uXNtvZQWwry2pju+HFr1u6bA4j3qFn49MAxWvS5hsr2vPOO8jpq6vrstT71e7X886z43OwjXm7w0+U28MgaLvaRSmL35zuy9Ra4Svuizbr5bazG+qJIPvrotbb3ajG69vfx+vlk+SDs62c68EZzzvHLPoL4BaBG9zDVSvR0pjT0HpwW/YvM8PhMjdL1Pu0s9Umo+u0qbO75hv8k8de4OPtviP74sUuE9rvNDPukDDb2gdoi9ahBcPkdTKT4V2mY9JzpwvkJrjz1WXZ099eAgPiRxrr3xl1w+WvEvvXf7zz0fdpA+K8kSvAgQID5iwnc9Io8lvU9qV73SrwC+ds4DvrameL3z2kA9lb5mPtJJ1T1pwAy8gk4JvBesCr4cCyQ+GPpBPY+ORzwJiE09w8i9PMb95jxK4i09n1E8O2uEeD4yo/Y8pf8IPlLJjjwbmQe+2erovPYtsT041SG9rEMCP8IDsL4DMJA+BC50PjFiI7qZ7BQ+l8tEvGQctbwtPh6+HUftPHjTrj4sGFk+sYJAPrPbWr25UW8+3V92Ozq+DTzC87s9","gUrSPZvAjj3gfZM8AM8APhQbyT0pexA+v9VxPUd9Sz59LSK9u58cPv/pwT3oNxg+qKbHPRTF0zw/6L48CJHSvG/NCz5Q730+h6lOPRS57T1Ev169/Kl1PTNomTwcCRQ+a/5TPRxRCz4tr9w8V+9NPb59rz25h7M+JjgBPa4yVb1xBsk9CCyqvnAYUL0/Xyi+SXyvvLkqaT5x5RO+yNGwPSUGYrnqQ4Q9ngruPfR0yD2MEpU+II9HPpNwWj1YmvA9DNIHPCzWMT4JuRs+VmNPPrSAFb40XTA7PftIPcQj+T4HH9U+IvXnPWR97jx9pMm9Wu4ivsqePr1FvAw9HsIBvfsqnj03GYs9igOPPZ6AVr3pqyu9Z+3hvSTYYTvOVay+J/ISvbqjTL7loxk+BJ8/PeDnPDshF5Y9NZ0NPQX+Ij7rTuI9mV8RPpyJizrr34g8YvaMvulN0T0zO+q8AzbdvDDetD3/vs49W3ugPcWHhD2hpvS9wnSZvmmHi71OxmG+U212uyfGUb2NJPY97Xe8PShf/r3ROaI+7lRdvlawcb53p5I9qJoOvgaD2Tymjfu731RTvrvjUz7VrLK+vT/hPYC0CL2+WxU+EXViPoxmqTxjloU9ZVmHPp0qSz2icxc+leYoPkFs9DzG9yu+EoGFvhvHxT391lk96QwHPZmzBzzTTVk9twAJPkdFLT3YQiA+aZ2WPQbViT0wBeO8efihPY0k1j36daA+E2wMvteAiz7yuHo+vBdePZY6VT0TFuM8QHzwPXPWHD2reoo9UqUJPvfFKj2m2qA7hxu+vS5anj29IdU6fYSRPikFuD0Su/e9Mo10vAKD6D0F/Js9Jat2PRu5oT5FX5K8A/YpPdU+/r2lfUk988qHPeTg/T0tJii94Xm2PW4aJTn9AqU+qCaiuth5mzxyEgE+6yovPVVUCT3PHKM+LozOPUqmvr0ltn8+fsJevs/qLb5O0kI90dW4Pv/sCb6fMms9bBSUPdqFmbvz8zq91KzTvW1ncD1YpsQ9","J116PQOlKj4F5IW81aPCvZYEizyYQz+9XxFBPSMiCL0k+jS+wg0zPgg5qj3C1wU+RkoHPoFNXT2YaoS+eQcSvcZaAD3wx2o+VcQbvuI0cjyenpY9FxnwuyDZNj12a4E9uYEWPmrZK74/KXE+AXiSvcEi17yTxkM+ph9LPueCUD5AcwA+T1qaPMwntD0FUCm8VuECviuC+zymtIK97k6cvRhEszzE+nq8mTQpPc1ULL2q3Ko9VOg6uL4uHD4KWiO+Et53vfYdQT7j+Ds+O4WvOq7unj0EB/c8dJP7PRisBD4WzBQ+nkZxPQyDFT4Us4Y76DN5PpFlhzwV27E9cfZNPaxGlj7H/oQ8BUn2PcIySb0hx/28mmxjveNKCT6wVrU9XB8pPhoSWL2A+kw+kTC4Pb/n3T2CZdG9aWE+PqX+fz6kMZA83qcfPiOIXr3PM1a+S5emPQflFD76kZI9sJPdvemglz0pEvU9RQEvPouU5T2ORBA+q1wyO7XhET3H18u9pDT9O6sCUj43M189szfHvcQAKT7IC7g7UVM8PkYYTb4TwSI+XaNVPbq3gz3b/T29IE4KPqD+eD6JMDQ+k1ydPid6xj7OL+89eh6tPRhAPDw6Eb09pku6vH3uNrzHppo+7K7XPdOt1T2rgpE9tAh3PPkwHD5FdJk9WqTBPZX+fj0OkTS9QeAivMSKO7eMNMQ+V0onPS+5yL38nog9OkabvYZi+z1qVaA8Uj63PZ5nMr2Shx08bwXMvZV9Gz1Tmj2+NPIYPs+R3D2pc8g99sMnvO9BmT09BD09GtADPhlA+r3fN+09BxaFPI8Aqz1lGxW9X8UHPZ3xDz5J1E+9lL/XPetoRL3/aAK+Mzv1PSyyO70DdVC89mDAvccGOD7Pxoi91vfyPEL8hbzArju9KYqiPJdqqbxSspa8EmnPvToIIL0emAm+sAFNPQswrDxNTYM8BOiivmXqQ7xePDO+wLfWvRInHD2lsr49ywpjPPqaPT0io529wj7APRnipz3aqVI8","AmIJPl4pILzfQ109WfGKPWDynL3Gqpo9ct2RvcTrkz1Nqpg9OJd+u9IV8Dw3Pzw+GCmMPUVKYz08Ta47BgoXPqR3jT10XKC9FCuzPFoRuTzGBZg93Ve8PW6UBb5MTDU9uqCzPULnjT5YPxG9maZBviochz0McA4+EtiRPW15Az45tbo+QZ4suyXRPD48kQo+68s3Pmiosbw9L9Q92YX5PbMuIT7MIBc+iXAEPr2Hgz0iN9w9idVFPRGhmr3LL6I+ZB7FvBVoN75A62+9Uik2PoIOkj1Dong96tMIPl2klz7t84K9/g/6vU87izwjNKM8TF8+PkZp2j1AteW9iMvaPY0vUL3QjTs+TH8zPRyQez7ZSwY+yBuovOBkQT2oyj0+jOHmPdCv6z3Vyqk9L78tvqDRZD48TKs82jcPP2PZOT3wNE88o6AyPs6tdj7gOb29x4T9PXRLWj7J7989Jqy5PduyCT5MKRc+5IyFPWoPVLyzNhc+pvfdPY9FtT5oxYg+GzCQPXbJBT1JAdU9spsgPqtMVD4It8k8y2HGuu/BkT2ZJCQ+eafRPP2ZqT41uVQ+WyokPg55pj4fgZ09IEJBPp/B+r0awuU9J7Y2PsY/zbxscR+8sDTdPbitqLtPZh69ajLIPeUIfj1F4AQ+UTOWPrd47D0kDEG8lj6xO/DdeL1wjCE+7u4ZPu3xBj4r1Uw+enNhvK5QqD1q/U0+hLeUPsRWYruevkE+QxfUPfo1Dj3S5fk8CkHUPVsB6z5tUsQ9wf5nPk71kD4bbF+9LiS0PiQsiz0C4b49TH0KPs+f5D3DudY990gxPXqKxb1TTWI98RIcPHE3Gr6teZo9eq5aPRb/lj23QdQ+6EBLPVhnb72tIx09n5gkPpaXZ7ugMZQ9/43KPbeBKT62tLU9GlW3vBxoVr3SueK8+tgsPkV8kj7vH4A9Gk8aPpHYSz5XpY49LCQePtf7CT6RDO09PcF0Ph5tXT4H8Qu9W0/OPQwmhDyphMg9OYUJvVk/Dz7t5Zw9","uIyIPXR9iTwkvy28y2eYvZ3MzTyjdAc9RuTOO0VsTL6G5xm9BYxsPcbOmD25sJ08Uvb4On7hl73s7o28f/74vUiFn7ze/eg7PSaRvSVQVz3GyIw+rpbhPSAq6Lxapgk8XPcLPscHaTxtkl49cVaBPOkVib0OGWC9QuyfPO7z+DzdPSy81Ajfve5QETujorG7pOKAvfJifjyxOq29olnjO4Y4t7ytSym9QFf9vd/GWj63BVY9KA3lPNXNCL2/y5y9Do+aPQQTAz4z/ii+gi3LPIvOKT12NlI9V2AGvauvWL0K+/g7DuDrPM0LDb5ZzAO9ObCXPW2syL2ZMS490QiSvRbnaT7FWw+9n/+fPZnnAT4dXoe9GaKfPOZiIr0Xsg++bpoyvSATGr1Ztc69aM90Pq9OFz5KglI9Q+wBPuq3Lr1dpHA+QCMkPXBQsj1fJZ8+rAgPvPSohj3bTWI+4bBSPoqwVT2tZc4+N2/pvCGaRLwB9dA9bKXgve3Ygj1EmEy9PIqTPgMxGj4+u4U9xIr4PaDdEry6d2099uvPPbjbaTwpL/w8QQSfPVMraT7ZgCw+n37IPJrWejv+iQq+bcgFPr6ScT7I5qK8qS2ivWU5tT78asc9Fb++vIongj1BenM+FjIHPoxFPb255UM99L3EvSItST4AYPc9QC4KPTEvpj1uvnM9nHsQvieHob3mZB6+JOduvSdEU7oIzzK+BdvRvSCg470oz8u9r6vGPIaNL73yh/q8lOP/vdws6b3HDI69v7sFvl2tC74zeyi+UcfkvYjMSTyqMZe8/Wlxvr+60z1kSZG9EAlNvpr39r2khiG9d7WLviR+gbyZFHy+9oSVvnGIA7348hQ9Idhtvv74FL5pFIy+kqHPPIqcf73Cz4y9t/xdvk5rUL3QSE++v6hYvvNRob2f+Z298XfmvqNZdT7tta694YZSPMh6Ab5D9VU8McI8vsa3Kr4Ezgm+GgDyPWPY6jn75T++mI99u/UNJL25+4i9NkrZvPKG8b0kffk8","26ZWvu/aHLz9Toa9w7Ywvd+kQb2Cr2m9OabgvfsXNL6RMS69JIRpvXUHvjxYEs68ydUyvXa6ZT1lcrO+0mOIvHrzub1vgk2+ecjgvTlFtDzaSu+9lk0TvZtd27279cC9X7pWPREnFb1XQQq+DUEevhMGZTs28W++7vJ/vFr9fj0LR+O9Hoa7vlOym72LhL89aZn5vVdhoT3nNGS+0DgePZvfTDwWNti9StKNvFn9lrzD7ze+3leTvcJSZb2TU5O+UPO3vRNWC74xZjy8+w0zvdb73r2rq2E9PGNoPQBi0L6KTQW+vV2yvQPbyL1UWQQ8gP0EveqzDT0ZdkS8tLfjPI8HsL3JuME9/oJZva3PM72Df2S9JTMZPuWX371K8lk8/BSEPNq9WLxDVf07SCv3vDh8nD3Q+5U9b3vFvbU6QT1cgCu9/48dvQJnuL1Y+rQ81scjvrF16zyr1oe7Wo+EvYS5vDx/R/883Y9uPZXLkbzkg9A9HCnFPaU3cz2hSPS8MxMfvdAcJjt/wrw888zSOzhGnL1ov6W9dHVZPMevgz0GOVO9w02FvKl/ib1tnY89P3y5PY6A0r1f76A9gSmCPRMhpTxBJKW8k8SAPlWrPz3ZUSi9CtM0vjGldj1v+4i++c7rPLkPj7xp3OM9EYKkPCSytj3El4s83J0qvmGErjzRZRG+R6JsveXtJb07Ziq8xQRSPXhEo70wi1c9VLpVPToEkr3/leq91ACHPbAqb76diV6+DmlNvUlaW71E+5u9A/BTvhBu0bzp+ee9A2CDvrfnfL11CmU8LUsTvVU9Sb7imtg8KatlvnUKiD26ahs9pMr6vRkYVL35zOi9EsKcu9Hpur7Tzny9ByukPPgdDr6DNjG9C7Rzvct1E77TNy2969GZvZIKhL5Ej9s6NKY0vsWq/r0oIka8NLW2vD8Umb0LiVi8+Z62PE7k77tZxoq+vmYDvXM8z7wPwfq9+UmbviTkkj2Qtla93MHbvJfUx7uYa9m9b65NPaNa6j0oeNC9","L/PZPadvbr118ES8EMERvrqV7T17UB0+LK2AvZnTiz7v4y09fbssPiAIFDw8ijY+DG+pPjx06Dx0Jw8/f+EfPvx5PD4F9eQ9JSc4Pm/+0zwttoi79wirvYlT3j3nwZ48BFamPX651ru4ZhO9/lm4PfEZPbw2+ME9+LwGPpfmRT200II95607PeP+OT6a9yU+KiwSPuciP736Xow+FMZlvesRDbwH/6g81+9bPQTJgT6uL789v6Y6PjfPsT0Ok/q9jBztvcKj4LxPqw8+sIq8PVRrIz1LvHw9PsElPeWzRr1sG6y90pjvPUJ5iz4FPvY9WW10PpTRRj4mEPs9OWHnvTj+0LrEcjE96G0BPt/OGj6njjA++KGMPWWDwj01DDc9VG+DPHe1G70XcaM6G1H1O1n8yT0bI9e7byVpPsTQTz2VzBg+2PxRPlMZwrx1WU4+B4atPQjGzj3w6PE9GQjVPTrlQj6FBpQ9/4mfPTQYjj0TLWI9k7GqPVIAlr2UlHM841xtvdThGT77atY83M+dvBN1Cz0WEfw9Ebg1PnkpozzvmiU+Rhl/PrNwCz30WiA9qrTCvL5fMD5s5u89ggSUvSRgij4RItE9UYgkPfokez3JUsY7HY59vXpHWruQRTQ+0Es8PjAeEr2Hu7Y9WxZHPRGtnz1xj+C8b6l4PXJd6Lxlx489bnNfux8HBzwiKUW8naa4PfZhFL3P8BE+/3T5vV2Hk72lYEg7SuYjvFClcb1SQhy9S7fxvQKPej1xfUi8BAqDvch07Ty1Oz89BiHqvIsO6z1Jaz8+9/QYPZ28gb2vuis+EZkpPSm/qTwGIE096rHzOwAsKj6il9s9SXu8u2Dpa7y1CtG9VsmFPuhRm73AdSE9dbcXPTSsK74Qg4S9Xb0fPQX0irwL+HS9xqbbPTTUOj1A3J87oyXTvQpRKT6VCaO9tXdTvdVfgb01PG29e+gIvD8Kgj7d+Ma83dFiPWhfEzxxGma8orPDPf+qAjxDeVe9XbihPAtTHz3C9MI9","KSRDPqw7XbsUqdU8H0W2O6Jwd70UQJU9e6DbPAW1Wb1jOSW9WbfSPYRE4T3EJ68+CWCGPhRvRj2Ng0I92sYkPXeeUz5ESEI9D3DNPJDFnz4PNOQ9s8tAvWz+5b3jq9w9337CvcBOiD5SmZ4779kFPU3czT148Mc99tYWPprzgjzdiYI+xEEGPZ0/yT3d0349j3ghvBws4Dx5AsM9GBiwPcJI/D19bFI9NeHpPV8mkj1fxMk9c663Pbl2hL0jx7k+kWDLPPEjrbn9mQC+F/hcPuK8lL1V+f07+DVHveSI3T7M+Pm9Is+SvTctI73KTbC7R6fYPHnHSr2fC5i8xzcpPgiX/Lu6nfW9WuA1PWI8przZ0Ci+EpnAOpuE0L0Ft4a9GxyaPYgrTL7jEAm6sfvBvZrriL41aPM8Nv6lvTNXPrw2wSE9xZPdvRtSf70gTMI9Vtn+u2kgdLy/Som9yRFVPg8AA72t5Vq9XX7UvbzVw73U4+O8IwdAvoQItr7K/ly+JOlkvVS10T1im+28TckUvh5WQ75Fwp29BwYIvhemiL2y4Wy+f+dAvENRsb6CWwq+U55QvmqmMr3eTh6+eYs2vZrKtj3mF4A9P4ZLvvYBKjuUTGW+2zQkPVq0Ej3zZAS9KkWFvY2zDr79xBK+iD65vFOYSr55lKm9XpL1vWLGfL1Tft+8bFG6vdTUdb28ny++ArAavU8M3L0ZoAS+k8ZsvQ8W/r0tfSS+A1yBPJwwTL4INQG+7yaqvavqzb6ilQi+3w9avoomf77alv88KCpCvYK9wT3GVhK9R6X/vcCWN7ziOPC8lmbNPSESs746Q5y9R1eOPEkq8D2y0pS9ljQgPeZfG7yBkMG+HtJuPXZUezgQiAW+wlcdvkk3V76qG4G9nYGzvZ04f72lghS+XhgCvjIfHb5QRzy9H1JZvaucVr0qU4i+zUmRvYPsrL2C8kE8b8NQvEfMWT2HEZi9oOAMvsA9Db4eajO9gmdUvY2Z7b2Bdlu8nTaVvM2URjto0FS8","e33TPEsPoLucXlo9/N1VvJvnoL1GaAs+d3CmvMGkmj2I+L07zXscPin/pr3NySU9V4NRvEY8tz1N+SQ8oKNqvVFBED1Mhji95Sb8vZqmCb1VvwO+b1AMvkixjr3fgU8975XGvX/C6z1874S94jUAvvYCBbyn4l+8LDN3PSlqxzsofZK9T72vPEzMLL0t7My92Mp2PZQylzr6DKg979eGPSqRAD2qbv88d/CjPXOqjD27RyI+mOgPveDflb1c0Im9uq90PR++njzeXoc9RehPOyVesz3nlNe9tDFZvcpDFr21lLQ902dfvfO/pz3AScK9cdgkPRYahz1eUra95bRIvcWy5r0w30U9/7CqPPXP+L0MFow9E/U2uuq/ij3nTXi9WDdHvm21Eb78ASA9pgqCvqFAO71/Rps9PWH/vO65mb0OtnO+usPOOOeCDb4c3jO+NyaWvUEpeb0NvlE9jWRevqkdtzwIfPq+WckNO5RFJz3y2Qo8qunUvYqVPr3dO7O9u56PviGOi73NHDW+n1g2ulHSBzxnwD68OPpNvpRm6Tp5byW9XFgkvQBnAb2+o+69nHDEvaGAmL0JIFE9lfcDv8QYZDzLA848ZG50PaGgfr6XoB+9Zo1cPUEPmr0HW7K+JzUDPsxCxL2RN1u9Nr/lvXMJKL6dYyO9IVfXPST3ib4Tkbo9JULCPQnTJz0JuGk9o4nnvWkZcr7oitK8G+0Tv2kYs75HTCS+wijQPR55yrzLlBc+iUAJvt+u5T7ha48+nnm6ven1lj2qvaQ+okigOgAyyT2lT6E98aDbPTFeuD2jMp+9NaAdPaDgVz5Wkos+AGGovNOmRT6BzD++j18XPkj7fD2a9I48Ny5DPZ15FD5FDG8+Kw1LvKlMBT1Ln8+94lVNPuV3Ej03DPs93Uo/vnja0Lo0PzY+7sifPo5WaD4o3Pi8/eeZvYcj3b18kgU++A+yu5YXDr5ZD2w+V07aviP78jxf2dA95d3lu5KF9zuzpI88jdE4Pjehrj31SgK+","6j0PPrSS/j2Gv5Y9xnqZPtZAAT7uZli+f1YtPTTLsL0A5XA8Cc44PkKufT5Tlgg+dUPZPZzPeL6z0uU+BTFCvCkpEz25VpE+Z0O7PTFBNT63SPK7lPbzPO6rFz7FhLc9RfgUPoQveztIZ5s9CbGuvTxDPr2ntms8zxfavfdSKz7oXMU8cjijPoe5lL1RsJM+M0dVPhqE+r1w2rG4XczSvTz2gDx93o89yzEZPEbJPT1+H1E+bTEnvSxKnr3ab6g9+ez/PNYpIT7umxq+RzkxPT5RrTwpxzi9VgqSPuutez6fUWq+e6KPPsxlvb1DYGQ80XYhPrYFvbzVR9s83bzOvu1LBb4Y3My9d+bsPQ4l1L037Ig+BWi4PqWbK77HeWi+peNpPtBGHT20ETo+s0zPvSi5Xj0DExQ8YbgQPnm6y7xq8Ay9mCDkPZ2ocD4Ddhk9PGZQPou7JD57QLk9No0iPHDcAr5K3j49flPaPVrnlD0KGyQ9BXfzPmO3H74JP6I834uRPnhRSj3G9go+/Vuqvd6Bzz3mR8+9Q0OsvO5fhr1OV9k9iQ0hvNEUXD3E7hK9E/UYvGCjQD3vQK49Y+OGvffRgT20hAe+ksBrvgkow70hoiW+9HKpPBU3yb3KCVG9iOxQvYOTkz2LZWk8AwSjPvrAPryac0c9KTX9vAxP0rlurp8+h1MEvV0Nar6Diou9mi+kvizt+juk1zm+IkRmvsuwxL3qQZa/TmMvvgOwnz1bZ6i8IulJvVQ3q70bnKk9e8tYvTO1A71HfUi+wIyKPgGlhD3WLuS9I31BvVHMkz0Tjwm+MSxNPhS48r1kRSy96QzfveujK77A4rq9kSKzPQoW2b0WE4i6LT0GvjMyxD1folu9yvqBvTimnz6SB7K9i6ynvau+nT7KK5S+KqE6PQNL/r6wnqy+AlUdvtnWoL0zKpQ9XR6AvOvnZ72i8t88DUQNvQm6/b3CVYG9FBQePuRzGz5k/x++Th2cveVkir2Tuom+WgWoPUOqCr78YIG+","Gz1uPUxZjDxILRS+SLeQvjwSOr3EdP09d/t1vZD01T3FgIK9L294vt9Ak7vshjU9ZgAmvjdM572UhxC+rOravETFkb2XgU+94VCgPbBdnr35/c69OeEGvvnA0L1GljM9eB0IPgO+LL7F7+69dG6YvII/Hb4iRR6+cqnhvrAZtb5NO+K9/D+gvbNVCL7RchW+WNWuvqo/hb4aDAi9yz0cvlv4U76b/p49s7IvPeL1qL5juzW+/PcMvRtnFb4jo+w9Ue4Bvjm5iL0cWlK+65YIPU8F3L5hnrG9mfKxPVItRT22umi7knKEviX1Yb0Q4wy+s+w5vrI8Or5AnjO96fxlPhMyIb7Ypik9ExqovTF6Ab193ZY9Hd6SukbMV72Y//u9Z2nNPZXiHD0z64u9/ROXvqfPoL0+YTI9orOMvpOlpLgyXVu9p5yFvrAmUL2m5Ci8yRWLvCPzCT0HaDa+GaAKvgOIITzbSA6+SYyyvWboOb5clLY9IXCLvdQ/fbwrbiW+G3xjvh8lCL9e+FS9JSxrPYmQg77tJTO92G6hvJS7JTwZ86w99jUjvsAiob0+QCs9y5mWviBffL6NO5G+I7y4vqACl718772937q+vRWRjL1WtYe9JiNIPS9zED7YA/O+MUbovalPbL71we69V4pGvPIzkL6nDIq99/kqvjO8o73e4yE9rkMpPdkR0L16PXY86rQ5va0aqD3NYLS8woX0PWpJ2j1Bw/k9N60KvUTYCztzJNq8ftq0vD8GwL1DGqM9wg9lPMui2L1MF9499jJ3vfQ9db7JPQU9XyTDvAQDbTvYLBW+FvpwPJ35GL4omIi9biaRu21udD21Z3A9JPFdvY+A7T2cShI8LGOLveRDZL3G8Tk8JSJvPZRvSj5qwDy9DyLKPMKqgj1B/6m7G1wDPgiAUD6BqRQ+1yTePWQVnr2urZU7IyCuvbktiT503PW8n0ytvVN2x73D11e9M16GPXsiOL3j+iw+ohNBPkoH/r0fpSK8n5bGvdaPHrx+wH49","mneavV59Ib75SH2+JLYJvdmw6z1TJqW7z12fvXNOu708mW2+K0G6vvbq6bz7h5i+GrEkvr8Q0L1TlYg8GjW+vG25arplUCo9hSozvqjGdL7m5Fg9TPXmPbVPwbxglXa+KyBxPZA5W772ILk9/JxHvTenjT16rJm9vDVuvpNfJb5QvJK+FwEtva2OB76ngV69tUPbvdmMjTyzQKm9AXhVPdeDKb6eZWc9w1ikvqXafjx+IUe89hanvr7OBDz3EUW+JTQIvu3tHr5QZbM9OaJpvjGwCz7KiZi9d5mTO1HV0L4s1Gk+O/6Svcrp1j3HgRq9wqKUvi65eLson489JMQjvjkyNT6liho+xxW4vSl5pD1Y/hM9cXkXPk+KQLugNeo8KTQgPIJBND4DVEQ9YcaXvfn9KT7y0D8+mR1+PlZ1Pz4LZqU8aFWBPa9L5rzDxw67a/e6PaP8uz1AQ5w9cnrsvenLnD1qx2w9Pf7hPs+53z1R1Q4+gsCVPRcJYj6VK5E+k42kPVHkELv2Bwg+TytdPM4AIj7MTow9ioYiPcmlNj2Mr5Q+qZByPfhlpj4FieY9lqU9PoV+hb2dDwg+VrLHPhBWH73QMNw6IvcYPjJ/IzohOn8+DPK8PW/WEj0k2co8IILbvOQY5T2uCuM9uu04PkrE2j2Pv+y8ZTMUPj8H0zyKSXq9+ZyoPdQDuTwl3ig+wPg2PUSbHj2Ld+09kNsLPlCWXT1Qljg90lOYPfBhQD4jp6s9Wt7LPauWwT4iKOe860MUPtgwvz6vXfI9x2chvALjSD5ypUA+Pi4JPgRYsj0Oqak9jrcsPnHvjz2Lttw95FKbPE7z2b2at/88aNRpPdGzYj2iVgU/w5N6PNiLlzxn0x8+q6OVvUNu0j2aPSm+LIl1PV3kdD3Z1YG6+kpiPbRgUT7jUlm9W1HRvJcTnj1jTZ4+I40QPiiBwD1EbtE93FpqPoE8DD6t7Qo9MQZkPqeSND59agI+U2uDPnBGET39exo+D/KwPeR5AT4k5mw+","pgMgPUs5Br5QOya+YU6pPbwYvbvqcBI9xbUAPo0kK72rjkU97uHKvboJ6zxQ4Eu9w93vu49jML6/zSQ9PIZJvRAuMjy4nL890a5UPpGnkz2zb4k9eyWEPsCkHD4Y2ZC8FQcYPunbzDuyIq096UOPPjD6Az7rIc49dq9aPWCdJD6ASqe9/ruAvbpWlj5Aoui8q0lbPXf0abwtNAM+T9FbPVQ4hTwSeOS97QCHPeXCA72y+Ou9ccytPKlIbb1vPwE8kB4ivZLbg72sahg+BVQmPQzhTL1Szdk9NeNwvEbkRT0LGYQ9YL1fvYg/5b3atC29SlaCvU7tRD1j1Z89hbQrPbEiHj6yVdo9ZK/UPMibCz4M4X89+UV8vRTBqb0eXhy8NaNFPg869LmUdgM+HSwmPmKFAT4Bajs99nr7vPVBJryPqvM9UdpQPA2WHD4TLVw+s4AXPTV6zj0I1kK9uXCGPleSirzjEKo+0nDovaszET36c629rkS7PfQ1Oj1zxmq8vqSfPu0qc73f2/g9ro/hPXqYUD0zkFo8BmM7vfzs4z1WLAE9hqEQPgqaGj7ofqM92DazPQiQxj06WbI9WQYQP1G2cj3TLe68DAf7vGqaWj4oA8W7ehW7PbeaGD4mSSU+x4ZfvgrttD0nbKU9OeLaPa1Jaj7qwZw9njLXvX+flz0AZk0+zlMFvktycD6ric09KjU3vsulOr0s2z69Xr3LPgGaAj72Y3Q+IB+Cu+sOvz4FJRe8wZSUPOkrwLo9vzI+WSdCvTqqCr54qm4+5PzCPCfah72R1D6+G8DxPfYnh7tftky+kMQAvcKCjzyp1yY+rl6yPJzKQL0NBDC+uCrxvjDQmD7K9G49aIIWvWNYgj3upZQ+K1VzPT2ew72f06o++AsXvr2GhrwaKlG+ADAzPpVTAz/NJfM88V4WvBdYcj2axqA+CnEovsV6Bb7iBZI9hKsPP6uyIT0aFTE9MjH/PAt2Ir/skBY++FabPVwk1Ts3Y368/H40PkV6tLy9FY+9","5PONvXUbeT27bRG+sv5DvihIZr4/hRE+AO6kPQabLTwZ64g+FF1CPgnJjD2myGA9H8+6vWYmOj119/2+4higvBsPUb3CjtK+aMCovFNsar5SIaQ96yhfvdX7pDygxMy8LxzHPZRfXT3Dv8s9v/6HPDyJzL3PrsC9/EWHPjE2tD6ISIM+hOHvvYa1f7yFebs9YLRAPtNAIr7OJVy9miV5PvCYFr1+qqS9kZqBvXodRr32bUa+UI2avoqtdT0SVCm/vUQjvpXaIr4DN709zz27uwShkbylDa28+GS+PS2f/r0bKR6/eSY0vV9keD7ZnRw+YT03vpiG1L3mBTk+HHgjvvIHCb6e8R69miv/vUItir0aGGa9ndHQvbjiibwL80E9SLB7PdZ3nr0syta9ICxfPZtrc75ViPy9tGg7PuRU0z1NOx++O2RXvae3Oj3iLfo92EgWvUoaob5vL2Y9dvBOPC55gztRzRa+AesHPmSf+DzudCW8s0E3vgBKOz5taGU9j9tCvnC79rsZYVu+2omdOv66jbz8q2G9FJbzvHFCub2POJE9UTIYPcH5WrxuUww96AG1O2G5Cb+vGlM8oWHzvFcATjyycJm+NKX9vTcX97tW8Lm9Wz54vozZKr53TjC+bovkPa2q8r0Qn+29DLfOPLdyn73gsjy9RIRUPkZE+j1dvbs9PPhuu+pKAL02oSE94gJ2vUDGFz7bhQo+tgK0PX8Qwb4kMgU/ojf/PubsRT2mcx2/dUC3PtEReT4YjhO9V0EvPr8qBj6JhDs8M84wvSeFhD6IobA9I9PmPQaiKT4+huk+FGg/PY11bbw8Ud895K0xPrY4Bz+srko+N9qKOG66Bj8rqsA+aTpBPizyxT7WYZw+jTQgvmvnF767sgM+tZMlPVhwRL2Tc1g9sBZ0vkLL2j0h3dQ9A7DyvXWGA718sZs+6OVFPZ2TrT4d8aM9y23xPpfUWj4O31I8M9TOPt60Qj3RVHW7qfUGvuZQBb6R5ie+/waPPt71Dz4XExW+","227DPXX4eL0t4nU+3rqjvbsg8z5IPL6+vATkPdVWYT6WNsg9NO2ku655rD7E5c07Hf3pvONCkb4Iomg+e78UP8BHkr5gpRO+ZMfzPXPiiD24DSa+5k1QPobZKT7lBaq9PcY6PlfzrD2t9lk+/OY+O3OFKb6qS5a+oKDCPpZHtT2a4R8+osShvqwfNr03ngg+09iCPqcHPD2QIyo+uSBLvse4er0jI6S+4SB4vrqyfr7LF+88/tSFPlixqz7sQVw+X/CPvc9S7zzxoKu52gbdvHUCkr6Xd7C+xYfVPdb3ib5ocQM/ItmCO5YDtb4MbRI+LscwPphU5Tw8t+K7+3VzPrqGhb7/oUi94PuCPXIGdT5yTgI+xauiPTUqyj0KiNY+tajDPdTFGr5TpfM+NuKIPbvFYTwrJAq9xUJcvcBwzT47ZDk+LBXtvMP5wD0uk54+53eBvgjKAD62NFE+ITervbBw5j5VZN89sQ4KPfIWVT7v9aw8Obu7PlmcMj5hq5Q9z2uGPQ5CGr8ipUe+bxOVvE3M/D2GKrg+wMquveXgoz2tZxc+isESvgtJID5NKuk9nJPXPMcA6j7TX7k+dFu0Pp71Ej7iemU+9GTKvrJOLD4v9Sa+Bo+Gvv+PSz6CM7w+jflLPuOiVb42wta+8ybZPUKNBD42czo+Q8qpPIDfjj7pbNC82wMMPtJvID7/Q528V2uFvWM0wb1dCGs9FRECvVLgiL3MCLM9ngPWPZHlDL6VsCM+6S/IPW+THT5g5+k7RKr5PMQ/1r25vj48/UNjvtBhrT1NObY+t0t1Plzd6rv2QJM+Ho0aPllNk71fZwG9XFzIPWnVBz4lJmu+VU6UPU3DMz31woE+U0OOPl6rCj4gIxu9ctyTPpI93bxtuoQ96a4NPmoxjDwnjJE+V5UjPkunEj3sFOM+q+8yPtZGjbyR7GG9bcwCPvK3Jz0VERK92EY4PhLNTT6I08094AT7uXZXgz1Vzay+dxFmPvX8JT3jbMy9HxQHvn/eXb0Tj1K+","l5NJPvvhcTwN70m7/Rk9PkdlBbw+76i9Vsv3vLuPV77ldWK9KLsdvqfAC799dAC+7pMBPoTCzb3P73i+/9+hPfAsWD7avi69g6q/PTvJGj7smh++YcoZvYDln72akX8+jS4aPkwfWT6/qJ2+yGMOPsFwJL6oc+i9aXghPukrtr3QSc69z17UPnvi4bw5Z9A9zj/0Pc9gJr2x+aU8LxUFvVjmgj6BZEI9VTq/PDS1LDzr9/u9ZLL5vTdyUr4sQI48Gu0CPo7YyD1YiM89k0wNPvuHBL9duom97y7MPEBQi7x1zPM9UoO7PDbCQ74qVNg8Po1pvZCqF74ELRe+55HOPitB4b1tP4q9JaqmvRx+zz1GWo+8RmC6vVd4Jr7M2iO9jMA4vNo1Xb64TZa9GclFvUM/rr3bw/m9qcEDvuymq72bMEi9wp5xvirfgb0F9lw9m+GVvatPob1GWdM72bLOvLQN6bxHUwq9izSrvjC+sLvvWoc9ER7BvrtUKr7nrw++Ox3evRhpnL16QPQ6TDIlvSyP4L0TtYy+hgEsvRWtDL7YDIq+DiZTPadyPT4KNZu9AF68vZZsPbzCFwS+AXmYPam20T3S3DW+DIJovpInvL0ab1y+l04FvAE4jL0uK+69EkBIPTkzBb4yGMa90SgHvWTgPDs0jwu+/5Ubvcm3Vj2B9ha+lEmoPefCNr4A+qy+h+gHPe85NL0yydO9OTE1vYlkDr7MytG9C9mTvTuyJL4FwB699myevanwh75W6P+96TQfPW7Ks763qQi+nrYsvlQTvTwYe+g9FOCPvCL4Tb0MSqq8CSh0vQQ/qb7SHMi8HT5PvVBb5Dx2YbO7ZDucvWJ7/7y2Uoy+8f7YvU6yij0CB5q+nms8vdCHrb7FTp89sChAu092kb6RRnq+zLabPZaWo7yPRQi+/RJRvlv3Bz1n+cS+jvZ/vrAi77yFfJm996C1vdyic73SJ6W8jgBAvuSjzbwoS+e9W0IOO9LVs72JM2y+8T87vdBvwbxBJgo+","f/gVPVT4Ezyn6RE8bkhHvkMNDb7iExI8BF0evrNwlL3/7JS9OeJTPkym1b1JNBI+D2ejvMPoaT4TrJ29a+2HPUvnITwbL1e+sc0hPXkPUb5oSCA96LxKvurE4r0XfgQ+Nykrvui9Kz2jPDm+g/MMvon1CT7aSYC9zOKWPfHNk7wL3b89RxdxPTnMXb7H8BI8DCnAvX5b8T0yHCg93AIXPVWULj7oUSg9LHRbvLqUGD0l+1y8lq2WvEm5Gz6hVGI94meCPdGzzz1cYgg+t+phvSm8tTym1Ru+TRf1PdcBH75wn6W9w2Q7PfORXL0xiKm8g1cmve0Eg71X1Da+ZW8fvrfG+L2j0aS8UsuUvJ1PA70yxTS9p254vKUYtzxZRTa9bAejvpesMr5vP3i99lV8vlL7Fr7GDYS9gBQHPk7uhL1Noo2+sI8xPRn2Fr7TPzq+HMS4vcB4nD0eTdk9XixPvqVZWrxmvb++kVG5PaRYBr0T9iK90uf3vWM0k76rjyW+19EMvloNMz1RxA6+dSJNvA5Zm71/Bwq9GfP/vKL1mb34QTU9FxrIvXehor6b4ly+9EUavtEXzr25U/O9zqjhvnCxM721Gbm9nKWBPCsoHr44yAK+nkELPYA4mz0nYZ++nXTFuzUlr70iaJO9vOgXvpsOobwohkq+PnKSPa6XTL5oQpc+bHv2PRAaz706Avu+jmtUPmVelz3qPI49QIe7PcYEhrzuSQ+9JLL6O5QTcTstw18+/P0qvYOT1j65QBo83/ssvg0gVz3Kcq+900QdvWrJsD1is5U+l9AQvR9+ez3E+JA9JuU4PUc6kz7ampM+r6EZPrF8BT6f7aw9dzfEPlOX67xK/Xw+fRUXPs7u/zzF00O+3n6uvrENRb5zjxe+dkaLPiM4dz7REAs/+qaLvS0fazuWL0U9jtuYPhPaRj6T9Rm//WBSPQXhaD6tu7a69JMzvU5LmDysGs68OdtZPSlmprooZ9k9uEEEPdl6qT14PxU+08vLvXLoH75dhNM+","DxUQPt5DSz6qG/S8AHgWPj77mz2clgW8vMufPPjjhj3no6S9NejXvapNuT06wrC6EsD1PTfN+zxzhgs/ur2mPS2YdDyec1o+nk0fuy6VpTxpbDo9bGuNPfsmMj7BRds9FbJ4PTgQN70XIs4+trKgPJnoyj0ZKLo8+uISvGhiVr36J3I+9JDOPn/YXz5QcSC+vX09PjjEN71sF10++M+QPNzjUz44/4C+Ov06PUgR5b2qh989VFKCPTZea70xAdw+a5HPPmxkTz7lA0u+3dOAvcMwj7vmOfG8gAAAPSKPGz5RdXw+ee3cPPQvtz3Z7dS7nI1BPLUDej5XlKo85/w9Pnensj2fouU9+cwNvhTyIz2/AQI9biQ5vdSzAj2AhwM4bKlZve3fiDyQI567/Md3vS0afr1opCu+YYGoPknSNb2jM6k95UshPvYo6D3KmWk9K6kIPuXV8j7SAKE9b4Lzvcc3hr0ZnP29NmINPHLuWT5Av709qubTvVRAHrwUOD49CuQjvLw15Lz3BMs+f9Q1vaDALj76UQC+sGBFPkVzsL1+ZXW+J7CAvQ3ymT38Zi88vE30vD5jar1wEhW+pBFXPbr4B75Vmp89G6wbvSLarztUkic8KXOSveCyFT2euik987g8vbzHD73jxzO9+f9TPVF6o71ozfI829BCvsrP3L0JGRM+9rXCvBFohD2/qS8+90w4uvHYNj3cjci851o+vK5vNT7oChA+FmaZvCONmT3neow+2oiTt3j88z0EbR47HilGPYKCbL1JWQk9fu6UPiAvUL1dkpg9Ta6oPEdknj2LD9W9PTqyPjgGMb0CuAs+L3ydPR7JkbwNfIY+dCLWPYbZyD5ezKY9XNZ8PQxxMb57dL89zcA+PcdaXL3LJDM9XVOsPSTb8D2oSh8+64GRPSXLsr0C2uQ9Hl67vOLwuD73kwA9NJ/bvUHqUb26uzo+nAP8vXKT1r2lQOs9vkvZvVFolbw5xig8I86EPJomyz33BSQ+k1+lPC3tcb6aeYo+","l17nva3MDr2Ye6M9P8XKvNExbb4LtM29yAeovVQWZr7pOYW73TUevULICr7WH2G8OsadvsnQMDtLqwC/Wl0yvmgOpDshkzu9T4DxvXQES70PFzO9q3qRvtsSSL6VHKe9JQ8tvsrFTL4+CQq+SpLWPMOzPr027IC+VA5wvs+lhb524tI86Ks/Pm/SpL01Pi6+MCcPvouG7r2Gxxu+1/+7vF65hb4y0KM9dTaHvnCMLL6KsiK+ZN2HvbCeCb69cF++AUxcvVSL973s9n++ANgDPa3qS75yGfS9VQIuvuzcYD4h0I690M/WvW4F3r346uW9ykjavS6s1r0N+uG9Pl1UO63dMDyxb829RDYFvgNqHr5jrny9uYucvPguFb648hg92zrlvfQ3gL3jIJK9LhDKvWTb0jrFkXi9v93Yvt8RFL7rwpG+QFZ3vh9QGz3xn0y+jQQEPF5VF771CMa9gmSHPNmwZL1tjCO+b1+IvSNOJb65siq77pR1PWJOa71Br1c8i6/LvPqoAb8u/CG+6h2KPRCk9b3Zcgq9Fj5dvdPEpT0drba9jl31vXXptrzACTA9iT0Yvq1+QT4w9G29XfN1vhlcpL0Rn72+oKuHvbjun76E2sq73s8QvHuzhr55CzS+ymcNviLm8rx1Rqu9HbmyvaMWlL0PWvC9NKpjuolQeb2idgc8kHpUvVw7a71rrks8jmGKPfhdizy55hm+zsWQPUdBHL23n1o889GOvT0tJj5olsi9GyFNPlkiYL1lj7A9YDpvPVGVcL03xpY9kbUYPfFn+r0YQQ69Qb32PCjUiD1Omro8Bg1oPhg6Kb07TjM9bi8APZfNnr2kVys9SRaRPBLuVL2Ptcu7q9IYvQjVXrz3MkW9nRW2vcbVNz3EilM+uqR9PQ6PUj3O2lq9R7rNPGPWyj3qDce9/ad3PZC+ez0AMS+9z93dveeiBT7m2hK9WuGvPKmTW70fzQY+3HN0vS7tf71Yjz09daQAvmJbqr0l/r29GEAOvPFOqL0b01a8","oeaWvUEnyrrLQIa9lFV9vVjAij3xosa9Um9kvAdaL7y3+IK+yVCDPc2AqL2uXiS+FO5YvqeNkTzOFRG8OvtYPbJyM76qOxE+ltg8O++Wo74xTqc9Irl8vXFFaD2HHtm93oY7PYk8yL5vzLa9CeOXPJftTL1dT8+9gzSMvoQEyrxSX6i+3IUsPWNrkbzTHK29f5tUvbGK4LyS+NC8XLG8vb6NPr7Cr4m+6qS0vWWTNr6XJjK+SuEBvqqzwr0CTsW+amr+vSS+1L3qv/88sPKvvXFPzL1FE1e9T3iHvI/2iL54lri6jy3ivaLbZbyvPqi9PtEavsqWpb1nZII9dPfFvX4pC76lWHg+JVOuvTuvDb1PLg6+TRLkPR25Gr4dHem+CdxsvTm/BTwwPyM9zV2BPprR3L4EYDi9fdQ8vtqt1r0Wg+G90SBmPsCCET46qzy+xtImPq5sCj71C5+99o1ePhFtUL186D6906tMvqCbAz6iiUA+0+bBvuc7AD+KGM+9EwQdvy12/76Y1QU+uqQWvr3+Er9YHkm9OIQJvRtlbr7gijK+8itOPu11G74GhKq+A3VCvj8o375G6NC5HdGuPcy9w71vE3E+VaAVvN3rT74dQT48QuVNPdqgBL555cq+KMJAP+emAT6C0Sw+blOGPrFnnz3XQb6+RmJDvm09GL6uarM+W1D/PPiT/jxntcS9bSv0PbbKEb1LIIi+daK/u+3HNL4ebDI82KXzvRRfLb7dJze+8H5LPdIrxb705p++iwVRvr2nGL50dyc9vZImvN5dZT4oezI9XhvevTkamztSrpu+o3mIPTJ5wrymi6894FIZPnutDr6v1am+nWdsvde0ibxhwBe/+/B7Pd9PsryiWW6+U2bAPc+Ycz4pVRW9AE+MvuhlKD0y3IG+qeyWPeh5N71HBQ49KWFLviZJxz4Cu4S+WOsyvRYfJL7RFUe+LwsCPnMxaT4UFVi9QVzuPQJtzT4aJxY9EAw+PgNrZbxEs7u+tDZ0vtsfm75v5c87","PY98vkrQlr3SLbQ9IFX8PcgZDT6slCM91+n4vbKBuz07Ca89xO0hPcwrfD1F+KW7QK5jvcmsIj5EE909o06cPe9CrD5MaXY+RpfpPe7sAb2lPRo+hGakvspLVL5sly0+eD0BvsZcAD7WprG9F0iBvKxIszxVyg0+PsnRPOHUsr49Wk88XdxGvoxFtj0ZrBI9awx2PmQU1b1DXI48PB8Xvkia7D1w0T0+YPyEvjw6iL1HQ569mC9mvsdHHb7hic28P/9NvZDKmj5xg94+LGCGPSzLjr5ICDY+9kqSvii1Rj1CzpY+XLJUvEW9b77Wj6k8BMkmPr+KTD4VKEy+hSlfvqfbIL7OXJW9/s7nPQ7yVj7soFo+ne8AvtYxPb0wFnu925cCvsBoeb4yXHi+UJZ2vjOebz2IGXa9kdsLPqMaO7wJ0PY9lUu+vfaRNT4btNq+PN+YPP4s7j2+Asw9ESeDvjcpu7xlNfI9N+gsvZTPWT6LCYY9d6xpvEmh/L5OHC6+nrdovv0PIb6e6+c8IQPqvZuVGr5tKXg+pDVMPX/itL6lURi+8OkrvsGxjz5Jw7S9XuEQvvoVHT4KqjU9daRSO4g9O76wQ26+p72sPRkQtr0Gyos9sRg7Psr3mb6UhjI+fdI5vgDHAr79zKI+pBQAvnhXCz40LLo8GVbaPYJ6ZD3/zb49mfCQPqvQiD0eLDk+fVRZPo8cGj07LuS8B34tPSBpDz5Cglg+14jUPW0EVL2TeGQ+jXc9Pa9rpr2h7qg9yYTLPaRSgD3tNfk90coOvQVdHTz55ka9MfBiPZf7O71J4tA9zu5uvT8VKD1udUA9pU2RvD+a5Dv87Zs+/CaiPlzDtT1nCWW8wtBNPVn2Sj4iPrG9JOa6PaZXtT12fOQ9VqjOPS9Zyz21MY2+53opPjfEED5lXuc9+gU3PuWO4L2j3P28+2BpPZ1b1zzwEQK9qxm4PLDZzj0Y0aI91OY1ve9t3T0Mli0+QtYbPttWkD0nASY+qx3mPdiO2D15Nzy9","t1orPi4foT3geAk++PAevBu9vb2rpM48tz8rPWw/djxGQ9c7WqOBPf4JRj3/M0s9+KBYPa339zyS8sQ+/BKWu8i0Dz4RaEI+peYJPYmgBz4nPQy9oHU4vZSskD09nr49mSkHPozY7j3qRI49qHGAO4dzuDxZy/s9w4kQPiyyLD7Syfc9kR6EPnqHGr2ZuMS9h5xEPbOzhz3HbQw9E9McvcE8Kz2d2oI9/x4GPhZcvTyPH7U9s5GRPRh2xj2nVyI+pfhZvE+fzT3QfYM8XbWKvAH/a737Tjo9mbRnvQHRuj71dhM+dCDIPeHlnr2N89k9TEp7O+JsKD7wEQ49Q8SZvdsKPrtNtuy9QHHZPIBqYD3Z1hC9toVbvdAZ5T0n36o9XKxevcZAr7xY3u89XhJBvVsZBz3Lueo9kgfSPfZcu7ymAV48gmADPn8FRj1R6Gy9Vh5JvcnmLz6RHs09m8WGu7bq2j1iOO69/eOdPKeOHj6Zel298NlHPYkio70H5466oJsevrZeizxC7oI+bQMivecgwrzCZRw9q0a9vTGcor2xcxE8rhOYPdBNdr7NOtO64l/evAEt5LpEC7C97J4yvIeWrb3Xd7A7lBpbvmYWrD0Re5a8t9afPdbiLL2gbbY8xrYIvU0BNz01EAG++sOFPbmMqLxFmL09jXtnPHNuuTzEK3E9I4mCPVru0Tw5oWQ8HYm6PWkujz0T2YU7Fw0nvNJimz0Jino9bw+APaKBFD5LmDM+F769vSovqj0b3YQ92pCKPROxhL1Ax8M9HcufPlwKAT2wdlE8e66YPTBEaj0wRxC++XQjPoM3Oz19Kgg9gFw+PMb6SLzk00w+A9S1PC2wcj6/mOU7q+HWvG8x6Lx8MSI+9J1sPR/eqryThiM9RlXWPXiJkL3WVKI+YP6OPbDaLLtzHrc91t10vLAkKD6tIyc+nw2JvTq3xD1tTwc+2Hc+vfsDOTxjr0E9K4tGPnFxkj3qiDQ9GyoKvC1XGj6grD29Qn4pvYyk9r1ioWA+","1ksZvU00kz4w+YS+PyZgvpLpaT2FcAK+5ascvXBAn70/5gO+R/KBvtxUGz6rx5q9ASJRPk2vGb69EyA/GkKiPS+qWL1NeJW9WG6lPtd3kT64ItU9D8eKPYIi0D0pDge+vzelPU1/Fj7FU9U9OY2NvTETOr1p/2k9Z2qLvjMw77xomTi+1X2wPg8VQD7Uj629kLWDPvvbGr6nWtg9n3a6PpETLj3ad6E90H3svcqxlz4aqcE8kY5NvlGp8r4XxZY81MBCPUKI/D22wLs9qe5PvR52Lb7/l5I9Z7KFOWUQ8L10JSG+JYhcPvQg5z5Kv8Y+lN2YPcjKxz06gCg+DRZLPgDdIj55WaA9IMfGvPjYZT6bapk9wGTIvZbmvzym3PY+RizEvZSDeT4oGT89dri2vM1iQb2XytW7gaiUPsXwID4lP+K8DnGXPdYWbT0DYKE87JIrvuMsOr1nSfe9j4eHPvMS+zzryAC+782fvk2GQ763Gbg9S1Yxvr+BMD+K6CG+Kzq5vtkWSr7jd4Y+5y+TvPC0tD0+HYE94zHqPZnGPb7a0T+95XwDPkIiDT7V5cG8JkY3vTVWUDwHA7a8mdMbvgQsnb2RxiY+LaywvnLIkj7Ubtu83A6kvYxqnrsXrqk8yjOWPQ8ulr7ZvYQ+1ZzXPGF9xz0i+iU+P7gOPnFehL4kjdI9dQ3cO9s4oD1WK9I9spDGPXrlsT30/cm84DT6PBZJSDziP5A9ZnySO5HXUby+hiQ9GTV7vXpTDL2gQIW9Pk1pPUUEIb67JLA+oHmtvdZE8zx1dOQ9omY0vm9wWL7whEe+ujZcvlQfnz1yu2w9yXa9PEQ3EbxvFG8+RaVgvKUvtL36nGw9mM7zPQRvrLxYZZc9vloKPZe/p71s+TA+DR/ovfGLszsUc/o9zV72vXpGv70o7yw9WWUVvhfLXz37b4077MQ5PomrAb4h2aI9hdwAPcbHyb0ufh4+7jGkveHqQj355+M9vbkQvjtJQj63juS9iCUdPpkiSz5yqyU9","u0mzPVnmrrxpV/e9oIwCPqTRJT7AlfS7XelCvHpsHb4LJe89NKHwvvWTNLwfqlk9DlUyPkXlnj0sKH2+xinfPVzjuDsvVJm8GVzaPYreqj4R4Ce+H5YrPVQF270e+Wo+UV2mvLr1lj2Pfao9iVwNvgH64r2MkYs+ArlJPzkr7L0yj2O+5sWoPc219LsxlQE9gulwujB9i7xZb6A7mRutPbxGqD1ffoA+YzRKPSex7D0OLDi+qoPFve3Z5r3FWH69GMf1vXvgqL1a3649VxV+vh5+u73P/hs83Oflvb2/C70wfXI8P7xkPHxMgj2dCa28qgfpPEGMrzx8Elw9JvT5PF8ELD1wqpg83Hq6Oli8xz3dgcC9181XvaW81zsN4/M7dkAAPu8teD0Zulk7jv+SPpFMWz5mK7M9TQM8PoZSEL4QxCs8os/kPMYT4T7sBya+wUmUPRg6Xr6iUrK+rFc5Pkm8YrwN86U8kwQDv2AvAL4SsBS+gI+pvB/UxL6R41s7VaWJvuNPjj5aNoS8UAQBPsnCcr0CByY+JJDpvcqKhDzgu2i+wsdwvYE+pz4SgEO+VX3Nvth00D0+/j6+u4QBPryMhL6uMDY+yGwovJL/u71Uzhg++4umPfGl9T3P+eA915N3PRJ3pb3QLC6+ZHWIPhSopj6bHAQ68GY1vZlFLr2zoka+57CbPPH4OL4sPco9V7s2Pq8NMb6Oa769JRBhvtzFtz1P1eC8JuqAPdOSHj6l8Cg9oZcmPiNp3D4/rr4+VtYKvokWeT4QFeW9m049PhiiGD4ctra+08prvkP+Tb5QiBU9fNFzPTc3Iz0FjRu9zfArvk7DMb2JfRY+3Ra/vhcn3Ty6JWI+cHvFPaGzq707owC+8qxLPmlDYz19Mmi++dayvWNOpb3ENaS+HI0XvkJAmDyh22u+PVwTv8ftfT4iqqC9lqqiPvuijz5XFNq9ztFIvtEeqj2Pj3i+mr9Yvrv2wr1kO589qc3SPbD11bw3yhi+xF6oPjautr0rArI9","npHIvQ7ztj0gGd+8oFhVPUlPTj6NnR49BOtnPT4slD5YW0w91F7BPVqwHz3lgme+HjwbPvZkkjy0geI95hicvIuiC7uBgJw8Ya+avvkr0D3zJf+89BI5PRMbg7511VC+wjMvPvR+Ur3PHIa+O5WIvi7YGT2SO5q8HWVIPJmejj6LSAQ88iM1ve3oCb2cOcS92k+pvpAmM7weNv090qRvPotEUL6wpus9PcTSPZ4Mzrx4+/E9kxytvsg3UT0SjgW+gUkMvkLpSD6pEH8+ZUo1PjWU5bzxWLq9qrRvPFoHXD0dUmQ9uZxPPo+fG77jxWQ8U+o7vk7ipL6dPTo9H2zxPVUynb2S6ES+BaSqvnV0hr4aofo91jfvvRX/rr0zBO897d3HvHWEDr7rbjy+RqW7PDLVzD2rRge93nqyPCHqp7uyzYQ+gt4yPqVsiz7yJTY91s2OvW5pNj50dUC+lBQlvj4XYb2XGk682YYHvVmLOj7VRoi9krSPvlAJSj7KaYg9NAKXvZOkIL7gneG8u+v1vWLegDu2kGc+kj6fvdQc/b1yLyW/QOZWPcUWg75QDf07ViYePmIxnL4xNL0+knlWPoJN5b7sGQw8mj20vTWgjDzxyiY+eYqUPN/aT76tfYW+fuXEPSfsHb1RmU69bPZNPjj+Zr6/nvO92DYbPSNhFz7Oqp69uwn0PGfmOT1Gkys9zY4vPTcKcD3PWT8+Hk09PXiVF76W2js+OONTPZf9JT2pvEA+5AARPmKaFj45Els9RtC9vD3p9T0M/t49guBGvWDBBj4zCGI9/bFkPpkYML0Ilbs96T5yPbpmqz4yL5o94ZLvPHCMZz4AmwU+RsLeu6/MtTuvRCi+7/cmvdQGZT3NEyw+eTWFPQ26Ij0rcRY+HyUjPtcDOz1kdU0+bo4sPv8SEj5+awO+XOawPk7k4DwDrBg+OAGbvRM82jthlKs8BRI4PnvdJD7nISA+3AbXvL8EgT13Djo9QiYnPuwpiD1D4qw9GPAFPhyxKz66rpQ9","Cq4cPP2R+zzNVBM97NM4PgU8lr194yM+U2uCPrYOLz5qzVU+5+1XPT3wXj0FwNM95xHWPb6RMbxKAew9nfdbPSuxVz0lgUQ+zcgpPoh1eD0+BLI9PHTrPa7j5j1oeJI9AIECPku4WD7mCq8++BIePTBWsj2BP7I91ZYEPZIDuTwkOIK9GTWoPlRzSz02CUc8hfwjPuk3ub2fgjw+axyxvVOZsL3Dovg9azxkvc9cP737PZM9czT1PS/dOj2n39e8vrpLvIPApT7RMAo9hdAFPoZUOD6bRgQ9Nh8pPnlCoT7OmaA9JHEUPdQs27yRBsM9m2jgPTg1eb3GMkQ+Ld4AvpfFwr0PZH49qfHTPIVtRj6eIAQ+thMIvik5lz2iP1G+12LiPAEaGL6EkX08D5JNPMmjmr0pIJC94Ka+PY5CYD2FRs482jKsPcaT8Dxd83Y9dgRBPs4EcT2pNRu9zEzsvSRBxT2LzT69JD0hPnXtED7WsMm8c80BPrZxXL1Kk7a9yXO/PFhJCz3pgaA9an6JvUpq572zPa68QIvsPFNa872elBs+tH8bvivepz27C289bUq+uzH5tb2n40m+HimQvSIP0D0VN6e9hKmtvpG+7Txhn+C93BynOyf1P73aLB69Sc3SPY54OT2Ol4K9zD8MvQo9Cbv532C97PppPNn8+T2diB+9TJ88vQOJ5z3sQJ89rEI2vejFFj4TeG+9R5r9PX4Tjz0+SUM9B/TBPdBIYz7j1Tk+7z0APTkilrwOr5S9HiTPPt9nXr31cRS8kvWNPlZDyTxRrIk9GkU1PF0fjz2ZnYQ7aRMFPyoC8jzzK6W88+4DPnTqpz27Iys9MEtcu+y0uD67gxy+RHoqPrWYqz3VLZE9CoAZPU0Ppjzk1a49yv6RPabgrT2NnqQ95KYBPSrCFD7TRqQ9wCdvvAaFrD3NDMW9D5kBvgiNU72NTEW9BZUlPqtK5z3MZBk9wG1MPlXui71vzkE9AnuAvSnCHz4XYTw+TActvYh67r0NwYg+","+c79PYS/AT6sVxw9P0oevj2KdD57zxY8egG1PAozKD0JTnO+5mQZPhHEXD1JdWM+xRCYPsAF9z1ADOw9IuCIPVmZcD47FUI9To2hPUYaDb4vzs49Jh4avE1lh76p8eu9wk5ePgsuPz6ozI28XB3cPSD+Mz7nK6e9MYVFPnMGNT5lNzA+TT8KPiUslj3ZkTM9tCRVvPcDQD49U6I+cdqAPSOUyz3sHBQ+KRkevd1hhj25jQy+1zEnvjueYz4BYTY+n6dsvrpP8j2tBSA9KKrxvUT9Jz7vgnM+UDPwvQgZxL20c6U923w7PszWkz1c8tA8JZNBPquShb2KRTu9FqxyPQu0k72uYjE9NhQdvSYVbz3XtP09NfqKPdCZCr3YpWG95qDlvmDypjyMjB8+XMLGPHtwpzqXR3488fqpPuXLOD2qcR49inygPjjyVz2Tuwo9kGqFPkfFz7y7/JQ9lRZhPpcMoTxrWCw+mOKfvYAf+z07gma9lsGJPc7vJz6ElB++3O43vHlQ7D4JcX49gT7gPCcWKz0TCNY96O+QvQ5tkL2VHOY6sbeXPBWqSD6FaPG9vcV7PpaEID75Z649jGt0PiYHvD6b6eo8SS2IvXvjQD4NxBY+HwIsPSA/6z0E0Yw+Xtk6PpwgFD7Nhmo9KL6SPWI24z2y8oA+HwK0PUZsbz4BOjk+X4GVvRVr5733DSK9itAcPtJC2j2U0Z6816jLuly8Wb07b7E9FwusvEBoEb4TiJS9EXzFvfTJiD2wkpw80QohPZj7E7ytulg++4esPUZS2D1E6I0+NFRnPfTlOD7oTgS9RRo+PGvhIj1Bkfg9VzsKvFRMh7wQzqq8CvyUPNOp0L3bA0o9Rs0KPgDBHD05h9W9QK/ivdCqor7yLKa9yg44visDQr0PLyg+QI7BvYNSkDzhD/C8xraRvbPpAr0CKEi+oS0qPvghbb1r5/M9bl9QvqQCizz84RC9OWWUPPA1fD5i9LO8xbvbvVgijz21uq+8qCaNPaEzXT244fk9","PYqsPRyRkLz+HwI9RzKnPV0oZD7fJ4o9mNNnPfYTYj3Q6Wi7eBWYPX6xFz7c62s+BilnPi6Egb16QoK9iUsPPiLBcTwXR3a85RQYvXboJj7o8VK8oWTEPbBCuD0QODY+yioGPRXcq73Stpa9XRGAvfOJX7wdlbk87UhJPiRhIb428no+rLAMPTDfFz4Q3+U9O/wwPgZKv7zcA4o9fXgPPg6N7j17Mj4+IzkTPibMFj6z48090lSCPXg5Db1wc7I+Fwe9vQwWSb3lVBC7eKTfPhVWDr0MOzC9G014PQjCTT3sn7O9HRgbPfdqlz27OEQ9RcfvPZqu+z3+VDy7SzKMPuwMuz27ZI49sawwvWR8fb7m3ps+RjFNPa8jBb1j5ss9mM/KPdun+Tx9xt095nU9PTVtMT7gRLE9qUfYPbIaFj5z7JM8eGOvutpVlz1p90a9C5t6PRK7djwJVIO9dRK0PSyA3T0VNuw9A2edPrvhKz6LL8o9r4JtPronfj5GDOI9OSnvPVFAvr16GAY+E9iCPaO2X711zcS7vDCFvD2/Rj5jUPA9zMVNPsFGFz9uEsQ+BqjWPKioELy+6iA+5wqYPebvlT2xotK9dwMSPq5pr72KGZU9yADsPMzQ2z2p34o+HDKgPLn0Sz65hTM+2K84vCJU+TzzOTk9uHFjPnsnDT0HK8w9DOunvcy1PT4tFWc+A7igPJMB0T0fVh4+UV7yPF9HJj5+QBA+N3tuPFcVpj0FdhE+p+S0PEucCD8HJOY9v0wNPtz2iz4DbZg9PBagPT0Z+L0g5Z69/LQfPkoLED4HSd49ptDSPbB3MD5uCP499RwovT9osz2nBco8kqmRPaAIgz1y9/o+FDgmPmnTsr3KaBs+NxSUPfm7VLs9xE69KyJXvU3KBj5vhjY+4RcnPfh5hT3Lmvm7SvMJPieHVz57np8++VExPvw2FT5Ezky8URWsPWsMDj32jLQ9fAV9Pge5iD1fGy8+QOUvvYcaqb2MN/66PINTPthVIz1dPig8","3OJqvWgoODwaokC9DOEAPgb9iT3RXMq9pSt1PYhnUr0cOZG9st8FOwDIkD2LQqu7h5YPvuFouLx2nQk+ZZQYvcrOrjo2Jl8+SHs+PTi3cb2/txA+XWuIPlvyFD46tPy9rWEAPdnJOjybcCA9VlwZPTh9sT3nT/G9HlCqu3Mf0j0nuI68xLMAPU870T4mFUo8eTJLPQY4QT1lDqa94FIEPdBWJ74GBK29GMkdPGEkUD2iRHE9lsbFPNla9L1lg4W9qg4cvlwnVT3HCGE97xWEvWJeS73+GAA9hSD1O7Wh0rww8mS8nCcyvbURij0iBPE9rgfKvU1GtDym35+8blkvPIbFYj128Es9c4MmPlqoCz4kbre9o7KePd7Bo7w128E9RzgePhiYDz0yRQw94wOHPl6lIT4WdoY6NdAbPWCUg71keZo+MTlAva6H0jwbasQ+A6smPptaUD37o2a9DeoGPnD0LDwFBwo/9ZrhvepAOD0PF1A9j5ZpPiFqdj3qcuI9wzWkPgB96z0tbrg9hh3cPTWgKT4fJ+093H5YPkpSNTw88lo9VjwmPhN6Jz7raKg9J7WAO01fgT1+PAC+bYUbPyzJBj6MyBY8L+MoPTFstT7b8Y49QUjpPAUH0TwzB4I+efSBvbH6Jz5Uzte8n54NO+8wrT2YDqg94RGfvYs6pD7ePYM8vVMfPRdLyjxTU5y+nxbIPUnMnL6X/Qm+/jmHvZBe5j3gIeK9W0N5vqCDID3zFg28tD/OOyM9JL4y3AO+PHGyPOx2Hb6YHwI96euKvddBi7yYNbi99BNBvq1eSr7sCgm9xwDMPfUFkzz6ogK+YueQvmwiXr48YJK+2wy8vucDvjwGkOk9qLMLOqBGWb0/1Oq9nehovm+JhTx3/xU9/YrzvUUwD71RS8a+WseJvXLV6L0cDhU+eRR8vRr7JL6MROm98cEgvVpfSr4vAjK8Vg/RvTjGEb0ZmlG+uP0YPgqthbybAXy+7QyxPd9Qpr3+o8G97z5FPcQaLb7+56i9","HaYpvrRdzr1PK168iPFcvmYxsT1HI2C9IbHSvSOfOT1NWhO+UHHHvNdBbb4XRAO+pkpOvAWo1DyOsL2+fUOVvaXuJb4iOnu+pWSLvfIDwD0YZgm+hw30vbYzdbxV15+9z6eDvmdmVL4zi0S+hFiFvV5QHL0tydM9cGfuuw3hqDyx7bw9nZznvuw9E74JkAu9ltZzu6yTgTt3wCi8Gu52vJ7sXL4XTCu97ymbvdlxxD3Uvce9IhboPRyT0DyJFmm+kWdsvAhJoL4ctUC++fnkvRCcFr5rXLm8C4Oqvci5e77QnQ2+r1pIPDZMGD2Wc+u9jdQMvuJgw71WQyW+m6lpvYkbnL3zhqA9BUEbPDu71r3I4xG+GTJevZBAUb3onJQ9aRryvUq0pD0bBQa+BvKTPRmi+r1yQVE82nX2PEsEbz09EaI9qfttvmZIuz22HvO9lAySvuvlrL3TcDg+jSaPvCJx1b2dxDW+4gcCPg8YV75KVBo9tST9vQ9VLj2YOQ88Jfwqvl1ceLy50Ye9C91EPbn8Ab1FcrE9QRRXPVpSi709EQU+cVgAPclaGDvMcEE9Yi/EPS7yrr1EYec9tArsPR9piL2L3V+8SaQbPuwGGDwM2IU9z7GWvfGD3D3bnuS8SKgivVOgFz3mP7C9179aPezVpT2zLgy9gQWDvFLzIr5gZti9g8vxO7EDZj49IAW+2yzDvSRgrT2QlsO7R7lKvv89aL4DkPu8DZK9vbH8nL4xiEC9k4gmvlCPjj3c2rW9W/S5vvLnyT0U5Te+lnx2vf/gNL5f1fA9flyKO4qfMr5a+mc9d/aRvqgpV73rTIM8YwmOvJhGQj1wpo6+yjplvlD3sr0sqFC9ZI8WvsXPFr3WIhu+a91nvgjK6zwh0e69S0jLvfqv+L0FQLo8fgHwvS1+NL5zrWq+ijwEvu6ZAr50ClQ+GdrrvQNGGT4W/TS+nKzVvCHjgb3z9rS8a0JOvvm51D2tncK92aMtvmSylb7uVpW+6Oh1vl01bD2BqT++","bquRPtrFIb2obna8duOZvcJ+Pj7hAyQ+TV0zPtNhrT7phQA+/TE7Pl3VMT5VNsI9jKoZPntf+D16O/o+6cTgPG7M0DyA/zQ+MemcPsaghjzG2bs94eOxPHwQG7xLEQU9WHWsPXXEAD53DBw+3qiuPbAWhT15lls+kQFxPrvKFT5diqI99ILtO4UtGT62Cg0+K9OAPqs4aj7OODk+AeocPuH0fz7/w4k6JvwgvT+Ghz7zQAi9QIukPQLuPD7xN6E9CkofPdsprzzksDs9FXXkO3d4QT40+7A9hKujPcVanb4Jmo4818sIPWlw3z1Cfou7+cs5PlailT0d6EE+f7FnvU2dhj5WbRI9YNzbPYJnoj7gTx+9DCQPPgDcOT1TZIY9xbCkvQJGDD7QoO09I8FJvRYyDD7SXP88FojvPnbreD2GD6A93f2VPkU8Ej6EyA8+OPJkPQDKpD0ITmA98P0rPcRhJD4XXbM9ylmNPc0CET4eWwO+l0adO3w1zj1uxYm99KHgPQie/z7syyo9OUZ8vS4Ipj2jNkq8IOh3PtiaaT2P1Eq954pBPfNttj2joAE+DslVPdupCT4RRCk+I1JFPtgnpD7ppFc+IQsUu8XX+z28fsQ8FMY+Pty1Fj7K5+89Kg8fPrj5Cz4lEAo+8lK0PZ9rlrti1D49EjBYPq13pj2XiME9LzIAvvS+sjydWoA9u2s1vc40yL2L6kE+f7Axvlo947xp7uk7COyKPv55IL2eBDw8X2BAvvTLIj5nPes8UwVQvvp3dD33i7E7BkTMPQBAdT7WgZ4+8kaxPeFL3b0UARQ9yNxGvX0Lt7zYbYI9MBnMvYpH4DxrG2m9TsdKPfXB+b1/tu49/Yw/PWkY6r2eCOo9hu+ivQo7Zr5L15U8F+GZvSFBsL1Mw549pJe7OzEI672XxDo9EZ4kPZeFpL33eNc9Hl9QPXto270QN2S9F0uDvdC0uT3i/5w9mFaXvYTgVT7raQg8+k0pPQBdIj1s0YM9p4TPOpVBBj7C95g9","ifKGPoyc+bzAIOI9nKCxPUdaBj2zMcI9fjWYvf6YvT3kkSU+k24tPji+hD3FadI+U3cEPmZDfDwDbf67AUGNPT5IkT45/Do94IpTPeE0jj4om6y8xV1LPUWuMzyBzyA+m4h1PcbmxD6RYr08ov8VO3AnCz6rATU9rRdjPl12iTw79Qk/mvIxPhpoQT2eOn+9CtU9Pm6gKD7PMlc9SPBlPQ+q1D15ICY9YO2lPbYb3T0QMe49LmpjPK9pJb3nVPo+eCLZPQjbnLwIjuM8BrmBPtjvgj0NYae8yl7/PdwtZj4Rv5m9tfycPVGS3DyQNRg9/fP4PeEB3D0D95G9p0ClPLjrPL2o5gu+KpcGvvklGb3T4n+75ZyxvcTH+rtbz7i9UVLMPGxShT5XEBI9ftk0Pe0nqz2SEx6+FyrMvppHALyHf8Q8XWCyve0hz708PoC9nQXpPIrxG761Vhi+KvrMPbwcq779sue+OIqqPEOT7721NJa9ywmrvsqASb44tzq+KsBxvrR+tL7+Woe9w3Aavezdbr7fg+i9tbYBvZ2uyD3R8u+9OtQVPZNCcr4d/yU8xUiCPZs0Dz7LhiO+ULJUPWyrF77NZxq+UPQGvpAGTL1ZkhE9E1WEvYd4ub0NDsc9DnwJvauIWbx+Vg++fiWLvdl3Nj0ztm6+mdXbvRKJPb0lgU2+VDLfujgCF71XrmK+kSIIvkmIkD2ZY5e8FXBePLJ9/73JY+c989i3Ozc1Nb3HCRE9F5FZPVTg575Iwlc9uJPIOUSThb6J45W9plKoPEfRgL2a2Q89MoEhvU8yij3CJuy9dZeOPcN2UT6XkxO9Umb2vSFx+T3vQ0S+ziNOPrfaLz4Puee+kHfMuw2atj2t9Lo78gXFPS8krL1N/ok9FnOCvSsdsLvL/Hi9wWw3vt+P+71ayYc9CYhtPW8XB7yp1gO9XoGDvcdS6b2serq+hlUVvacIljtP2/y8qj/KvbgiB71K0jS8OtMwvsGj8D10BM29B5Ekva+xIj1HC769","h/3LvRIu7Tzz/1a9UmbcvV58XL3x5TS8qWyJvsMhD74OEC09TNFsvbZTTz2IsJs7cZznvawI5D34tM68+orVvZHWn7217Mi9syHjPWcQWL4t14a+6K8dOqbNPD6loCa9+dx7vA0kPj5OktE9AV2fvcMNST1u5bG8QSMsvXdhw72NDKc89vjYvd2OxD3XpEA9EdfrPH5Y0T01DrQ9w0K9PKNFkz1QCgc91E3fO8DOhb3uRsQ962gUPkXR4Dzf0QQ+gby9vpgCT77bg7Y9sEYCverDBD34nFE7M4GtPWG0/r3pZTq+ADEvvasQfT1IdEE9W+8rPgdy2j0J6CC93MqxvUxFx7wd4eC9NbTlvZUXar2ONDu+L/mAvBXgij3Tevy95pXLvhDDRr584BO+GdTqveYHlb5YsCw9UEWxPKUa0jyKxZq9cFzPPNS+gr4tWlC+JsCAvTTyzDy9Mlo9EF5QvlPhEj4V7Ci+sFyuPc/vv7z9uAa9SwUNvmdwoL4RYDO+hi8Fvr6NBz4emVe+pIx7vYV4Rr4srAu+xaycvTvS9r322NW9oQ7sPHvNXr5X7pm8ApXrvFpHOb6dkqC+doZOvF3XOz4hQLC9nQPgO88b+D2gmFe9WzyZPdKcN74K6H28LRM+PdLVb72NFXC9YrA0vsMTQb6UIji+t96PvUATGb54lDi9/VXzOmchVzyiGCw+9LUtPZd6O76qUDa9xq4QvBFXRb0cLCk+kw+EPkt64D0eXnQ+H+vQPT+UPj7t3mg+797hvXdXOj55YKq9RhOdPhAbCD7Bnx0+TZBKPQ154jwkJuS9nZlBvequlDzz6v49loGjvZsbGD+dJCE+dPAJvUnH97zR+zI7LF8sPmQgUT2hCf09vnymu+8GE715gRw91Ui+PjVdDzpQY8k93XRoPgV2UD1W4iS+JNyPPXc8hD3O25C9whWjPm2UkD3ZCqE9+fJSPoI80jyKLZo+4iB+PJnoYr4X7VS8rygWPpop1z3mpiY8l/ztPcnrcD1aL0o+","sy2gPNhFgDvqRCG+rUwqPuNdZL0E7gI9gqPaPZ7z77rB+BE+QFSEvVQeJD4dSpo+7ScLPR5QW76AYbk9qErjvfKsDT57Du49gg4vPAKpDL6Cnps8tThxPVmcyT62SVE8UiJAvfQ5hT4DvQM+YOWwvLdgaL3zTHC8sGInuFfyrz6DQRC+HjSiPq38wz3A1by9YbhLPlnx+T2lJ6I+XOpKO4cRAj6WuBY+Z0AfPsYsTD2DOR8+KXrOPaIMAj3BVAK+V9vqPnZUij6GZV09jWiYvklGMT5xOSS9z9mBve25ez1swK89ocfuPSjFq7xBCf+8JohKPY56orysLnG8/pT/PIbIi704rUs9KtQhveUxoj5OLee9suUhPYEcFbyPlAY91wHjPSE3Db5H8MA92DI2vYKIkLwi0TA9nj/rPU00Hb7aZWO9K6POPbScYj2LyZC8aRAWPr6ZLD4pt0I+rwQAPi54xjyLdDM9ijR3PtJBRLzZZBQ82/FWvEMdm71hu5C+pTwqPI/S7T3eVqo9JvaYvRVTmjzVwvE9unQqvXeX0Lwn2cs9hRDMvEwa1r0Ygjm+x2LbPaiRvzxbxOY7Uy29PSXCgL7Fz5u957YOPpTKE76FZDm95n8jPWKxQL6m7LC8ioeFvPUDdz3lbvm7X3v0vBGEJ7yBqsU8z353vnJoCD7rh6o96gwRvddkmr1dBKG9aYdEvIe5cz0EaPq85HcfPmfDFT6LVYo9qFF+PeAjKj4CzeU9EMRxPqogXj1O+kI9iuByPVhWN736FJG8nNN8vWLHBL2G+Lk912F8vYfIUD56RQy+GfAxPiKsOL5qKlC7Zja4vIqz4j3S2+m8L4DnPdRRmj7gmuc8UO0TPi/MTj2rWCs9pneQPJxm7r3CqkU7QQYZPJuqWD7HMpQ+/gA4Phw/ID2evLc90dgjPnObiT0bme88xGtvPb/hr72i9ys8qb+7PbB9Bj4EFGE9G6WOPRpo/jwzcTq9DjSOvpDGIT6Rd44+3CY/PuXGmb2z+7A+","qFbsPX8Crb25nkI+antAvqizpT2w9YE+M1ckPgmGqL1m89I9txsju94jzb1ueq69ac3CPXfWGD7t4iK999HQvpIK7ztpyJI+0ybRuxL5fz3rjwk+ZgDwvVmN2r1ina49ATaDPu+yYb0dcyw8MzM+PqGHmb4mgGs+bsklPRcfcb6LAim+ZCIOvEFAgT3Er8O9b1oGvlBbIj2BQiK8MWKQvbT4s7vMbRS+z/JRviAuOj7DuBM+WbQTvKPkZz5K9xS+50R4vFwC5L3aRju+E4xmPfNduj7OR3c9lIR5vWVXxDzlh5q8mQCnPer/mr3KO8U90HT+PEKX7Ttujo09eE5mPbXKY70qQCY8gyrBPLWfE767NcU9TE21vb7acD33Tdu903jpPdUJvj2KNAe9TfX5u/7NBDxufBE9iHJHvlG75Dt91JM9t437vRUNhL3NgY29Tzq7vIhxZ76Viro920PHOos/LT0C0HM9PDJhPeYF0b28ixm9jtsYvhAzlb6kfws9EDuOvvdywL6N+wW9upFWviV5KT0pZym+xANOPTzGxL0wPKQ9B2swPpesBb3bqW895gpovTTVab4M6cg94Pkovq8/2b01Qy2+pkeyveN4xbuoGsM805cCPnZwAL0ZF7K+sOUKvr2ogDvK+m+9qXmxvQbxiT5W8OG9lKTUPRXIdb6DCXC9X6Y2PXOsI7397eM9ZXefvSYFMj7mwEa9Ec+zPfgYqLyDbdq9z8WHu+8L/jxSXQi8LkWIvf6y0z1JQr09ppwiPiRxgT0cz+y8reCKPKlGbb5Zw5C92X7/vfVJujvegaw7hCNWvdd0Rj2ta5E9gNyuvUkPPr7rqL09RzY+PfJwl7yT7fY9sScnPpuTxD2iKuM95OdRvrYuJLym5bm9dZZJvtkjtbylIHY97+32PU+57D3eLUS+O6VqvQsakr2aWjk9+j3PPDkZeT5Geru90n7APcoaYD7qlma+KQdSPYNOobwFTc29mywIPvblzT1ith+9Si8FPle/aD2NnFQ+","jncpvSxiAz4jzws+0yJmPh8l8D2WnzO9sWVWPe8afT592g0+mrATPlS6lD0XBaw9L1hpPVOi4z3sIQK+QLmsvdcVLr31loa9IBLSPVDyp74NebQ9SsSdvs3V9Tyarn09i1EfPmIqO768IxO+5PFOPbOcsjzArTM+saE2PS1TvbxfqJw9hCMHvoBXzD3Yo5i9pPwnPbERcj3OKUu9++hGPovOOj6ifaa9ifidPciBqL1MjzM+jsguPn6wPz7qa3y7f61Nvg7YPj0+Fh2+NToSvuWa2D21Qpe9mc89vS2Ucb72s5C+zfMMu0HgGz4PEhQ+0sLcvS4zZD7R1ls+I7W4PWa6O74Mjru8a19HvjzViLv4Z569FLwhvvS+F76oyTS+JnHgPVjAWr6hfLk9y63vPZqkkb2sG6K8W0xWvTlu3j2e6CI9zc/EvbWwrL0A6+u9ltXgvMatBr28ZVu9AQsBPixSNb4Kau+7O/eIPLGbPr3ogs+9qyy9vZe0rL6UULy9YemOvmK8MD0251G+hOGjvJhFnr745845vFY8vj3XDL3GqA++vNDWvA5TDr9HDQ06XcacvStsbr3oCLK9yl2LvkKkg762Aem9wcImvprJkL1Yl+K9RVmTvRN2n72VYRi8wk/svYvmCr5C+R++YJHGvTXsKb4zC0493g+bvREpFjwDA9m92TJnvXrhu72h4Zo9APhdvV2jyb2g39W9ZjQzu/YFhD395ZS8d2YGPZGfgr2CX1m974+RvFar4L7KzZG5kR3RvKcobb45mUq+UfdAvdwYxDyqCms9vSgpvDeit71+QJW98UdrvvKNwb12wEO9O9j+vOJFED1MmMS9Uvw1vqUlGD1wR92+gxbSve13dj0Zwti9EUSMveIvnrvKVhU+vazRvSLcWr6Pyvo810TxPWsHHL0aGoa9Zk0DvXMrbb6CVvk7F0u6vQEaIr1qRPm9AZL3vCK16b20aVa+dHjtvZXn471mEgG9Z1Fevh6aRL2MeIm+BxnmvQT5NDyELAm+","MtjHu3blNj7BgLE9aK6MvO0eLb78YgM+oiGkvQTUzD2iFLA9Wdp4PZACqT0f2du9g7DePSEbzjwdmpi9OpCKvSuwjj2TpLk8VeCNPYsbXj16wCu+fYDevYZwWb1gXoi9IWUIvnVlmb1H2u68wG3OvQduBD78l+S9kQKlPQAWHT1CjAo9w9IQvUVXhD3px6k8f64evbAmVT0Ki1c+J/E3Phy8nrw03eA9zLdLPRJiM72oJsg8Kx3jPUjW0j3hv3c95ViVvGvwHr5kHWc9ZIS1PW1wJT291oS9QPCwPdGsjr1HP1O8vv4RvNWGAT6bs3a7sbaEvLBNVL3Rw9k83O7JuxolLr51yuG6HoSHvaKgPLsuZ4K9l5f6vcPfnb1f7ee9wVGxvco5k71+fau8tncavWPy9r1X0AI9hccFPebJNb0jVi+9jxAuPVMNRD2lSx2+sAeMvYeTBb63gPG9WdN5vmabrzyuRim+3HuGvUp0/rtHOhw9ml8TvpnlXr5Tkk67BQFrvp2Jdr6hHFc8DysoPGKRC740jIG+LCc+PdSwMr6T3E++uH09vbxRCL56gfy94Oo/vnElKb5PUH49b81wvvAvJb4pzQ69ikW/u2jgrL6zRY69SS8LPfCZo7x+9wG+RvS+PTx0Eb1+ymw9Y+edvWXjp70fdMW9iqrMOgj7tr4w3wq+d1mqPBrS0z2grF0+hej4vaq5ab10nTA9P8JLvT4yqL4TOT0+PJQaPo/ohLzkI2K92UV/PDLnCT1ogtI9LaY/PoQWLj6gat49lckYPQFTKzsyBOk6uGl/PojfdL0gaGw9hydLvTt77T0gP4Q8289nPJJClD0ADgY/iMznvc0/3b2pBRa+qvj6PSq4Mb1iJgk9Y7oKPpC9cT2+pYE9rwO5PpbSFr3WrDo+AI9bvflHXT2XmLK9plGBPn7qC76K1Ws9l8fnPe+RGz6lUrI8tmRJvgBcATylPRa+yGQaPnHh7jyrbas+cydWPa05Mj7rfSQ+2YLNPR8tgD75GsY9","JIU7vcj1gj3S9489pGWdPm6Krj0XCZE8aiFoPcUeHz7i1ks+qsW5PRNOWD2/HGs96GIdves1kzy1+SC+oiRZPjOWFb11UdE9HPaNPufHCT7F2/S8Dbymved3jT7kGxQ+W5thPS2RTT5Pjms+k3/fPVkRzz0G92K8Up+nPl6GqzwREju8JATePbNh3T3Qhti9Sd7FPfFMVT3a/mM8/vdkvlHdAD1zuSU+O+YQvuACkzym/OE9qR7tPYMyIj6Tgb4+lCtkPlkegz69hwK+IPikPaIg3z1eEhK64HfkPXVDwT66SZg+YbhLvf7Jir1R1TW8JXnIPWy22T2YEl89hxO8PG8co7wbs/A8r7ugO0/rUD0J/JM9bwGLO8uM+z0JWJQ8202ZPcgGuTwSIRQ+UP/Qu3VKBr4lkaK9NvmUvKbAfrx+4eK91vPDPdEZzjsj3nU9HEBpPutTmr1Dcsy85eI1vNWs2z1mACK8zjn5vGDMlT1NFuq8qmRfPta4zLw5lpW9+wPpPeUjcb342tq7ilk/vW9LWbsQdoC8ehYevv4X3bxP3oo9aU1NOgwunLx0PRu+Q6aDvetdTD6IzNc9Tz8tvTdKCb2BYcc9leKKvpFYiL1UOZO96KSIPmlPYL7sAeg8uaZEvVYuC72VKMu8IBQGPkHvCjxogCU+RU2/PewDe7xmZQQ+FD6PvGEKDz5yc3A9erFOPUZxrz3JT6i8D4CIPXT/hT0FGdG+YKqTvpQITz5i8Pc9sTljPNlfg71Xegw9SH61Pr+JQL7CBMI92vqzPjGimj0TFrs8z9FDPQgfrz3+AuW9XKEbP5I3/j1OigU8j5K5PTkTnD7v1QU+P0W5PT4th7zopDK9dgqFPYyBZj7OdIg+qz6dPVgXqj2ZcOK8im1UPoLqcT558JO9QZBwPsJ8eD1FOT+8lDd4vQRkFD17cxK+zp3cvR42wj10H1I+WcWMPXbQxbtcTck9yoGTPjuoSz0HZpi8NMg2vjFynTxvU4s91tJwPb2dar0ts0Q+","Lbm7PLwVrr2ss/e89UGVvhvSJby5LLq9LRpWvezBi75fBIQ8au40PrbmGb2AJwi90VArvtAVgj6neQ2/1yTLveBcib39SkC+tmgMvB82hbwyvBI72sZEvTZRLL5nP0M9AOaJvCA8d74mBqa93oxtPaQ+i7wpKBU+9g+Ov7kA275Fag0+mo4RPuSxtrsYzLw9XC5xvnhvg70Gtp+9KIv1PnHyHb5jGCq+tDFjvuw2tj1Kx9k9gw7wvSPqwb0kyDK+lwbjvZcMBz3dSqq9F4iePETc2LxYkgg+hr7IPDvCBD1tk4O9RxIfvor2GL1kt/W93dW0va0IFr4uj7a9YICJvVxt6jsZLMW9agg1vn0xjL65MpY9aQ8KPmST9r3Vk6y+ndTAPMpa371EQSE+UoWfvq1OAL7GSbI+3JN5vjSz5j1RgJ6+t2aDvlFc9D2YRm69s/a+ve1fmL6UfgM+6EyFPT0HXb4xuAK++XQRvuYkL74bPl+8sWmmvkfV7r0ySHi+aYpGvgvk174yBqW97yApvuzoxb3ARKq9C0JvvX+5jL3Ovlk9uzkuvJ76vLyikCo++w7ivpR3Qb4zDay+Qa6RvhlK6r2R1Fy+PyohO1kaq72zVr29FqQ3vInJVT5juaO+1Yuovmp4V7+czcu9QkmBPWxJP7x2uwG+3QsPvnQk771KHz49Q+4+vUHSCT3TxT89wN2qPWsQgjz+6z87IJyVPjsHxT0fElm+GE1AvoK6oj2Ux569WKMmvk71SL3FsVG+IhCCO+YdZ75AycW9TunEvf8DsL4wpm49qjQ4PfZArz30MJu+sQPXPASXVDu2wK49WwgpO7YTkr5NMby8eFi7u9gjyjxeL5e7+eBivZ2WRD0HK9Y94YaMvtyW87w2Xcs8ny4avXrbEb7MHLg9vGdovezEK74+wdW+YTIevvl2QD18r7o9dxmEPRcAeD4f1w69Q1tJPSiyvDwUIUK9Z7npvJhdib3yc/e8L08yPkW8371I8uu8uQfGvQPDa703WoK9","bZJQvYyPgj1ZWCS+rfVnvWt2Cj1plXE+22MFPY4U9z1heX49+36IPB8Muj0Ggre+DIEkvoXXEz2t1pG8Bavwu6CoXb7sElC9RXrSvSpon73/Yfo9eX8rPRdbfDw5NCE9Qd+yPa2qA74sAY49u0s5vc8zCD6tixg/wArwvXqA4723aOi91xBsvrEE3zzIggC+1MLHPbzHkTpbESc9N2ceunXeSr2y9QO8/L86vjKtwb3EYMU9of/lvWTyjT7vD629DPH4vtNseL5cMem4a/GAvg9PsT5lUUQ9t81QvRIWLL576cw9udyFPAFkmL1Fp6w8Sp79vqXNoD3FVAy5UqONPISWsz0SYue8XzQZvsM9kzonyIi+r6RrvsaT472wolo+ZhL4vPwUL7zQUwy9VlqcvfOnJb7CVn69YBGqPU6k1jwYSs694mWivdCQzT20/26+ruL9PVD6Eb29T2E9pS09PrJ/Wr2LWqu9RIVPvga+uLuvAby+jcZNvh43Ob5Pf+29cKBjvV45mr1GXXm+AfpDPNcDC76Wffa7t0QSPYDJz72Hlm++Sp3OvWd60r7yBXC+l/RLvRKzkL0NDjq/y0xKvQ/OOL4HbB4+959yPAUFVD2iYoY9XY6EPeLS+b1xePa9drAIvj/TqL0s4fG9XZILPjrzfb3XfaS9HCRAPA20Ab2srA6+JjZDPU7QJr5aRIu9acBGvVwfCr7dyeq9KD8CvGFmIj1Kati9fEFtvQdvKL0BMxS9LmVxvYDsvb60vAI++yxEPV93kb5rZGA9+k4pPbyQTD31v0G9od4ZvilaCr5FRs48nEUUvkmIBr6N+SS9fZgIvXxtkTuMuX09d9zevce2o77vGbu+vNc5vu91Gj2HdQe96ZaQPFUI7z1rLtg9UMozvRgQYb5823c+XjPfvV93UDtCKmY8D20TvorSML4/KZ29L+agvvmORz6tDRS+5h+FvB7DCD5flgi+/HDmvrF1Gr0JW9m8Vg5Uu0825T1PXbe7fDaDPVUla73m8G2+","ehSBvQZXbz3KzvM9oxXGvbRXN71wZ7G8J4uNPAEaCj0t4Sa9sbRnPr7Bsr2+LfE99hyEvEXsmz0IifK9B37uvTuVnb26J9C8CTqMu0eE1Luvge+8qMoivqoAhL72MME9EQQVvlCptzzaG1w9EmAivpVMXrxQDQE92L7BvZJBkT6oiua9xSOXPWKgjL02yfO7FSQ/vtq3Cz2X0q29yyc6PtyTC74HE7O9ThHivN72hb1D86M9Ro89Phz1N75suze95s5GPl0HBr1pe3y+hDOFPJ1sJz6Ntr6+AVcVPZ/bXz0YmbY7OfYePoCto73uXko9c2yqvRF8mDxbXVA95jA1Pa2xHb61lTK++JUevhrIFL2bVQK9/TNnPTUcjDuVWYC9aUsjviifKz3KNMW8nckUvrUwLL46NSE9C26fPMpQuLycNdS9t8g5vQak3rzJD2G+hNqNu+U1lj3YXnM8pNI8vtFaiT2u9Ny+v9uoPbEa5T0xDA6+KCoYvnB4JL62Mvs9V9jsvXekKr2eBbe9KtU3vnB0fr3Oige+93mNPaUyKbuVlbK+Gz82vvyBDb8UFku9ne4JPknJtb6GRCM9/6rdvvRRejwLiAO+rt8uvTBldr4bfw89+iViO6P6cjxwLB2/kuSmPbOVxbxTXcU8XnL2vSEZiL7fSzG+goNPPYmh2773ec88BrLwPU92ub1fjIo+mBYMPuvUmL7933u9B4Sbvq/WUL5REcq+/U9IPtQQMD1oUTW+Hh1nvjJoNT7rqbI+p+5NOypctD7B4my93FpavZXbjjyIl4k+qLuUvZB4Mj5/EdG8tjVFveLZsr02MNu906wMPv9wED2CJmw/Yffevc5zx73po0q+cn7kvYz25T2C6jW+17dLvhCo3D3sktm9OG6ZPkW1HzuGwKc9pyJ/vn/4bb2qTYA7ovo+PnADmD0B+YS9l4aVvKCeszyCdP67rKQFv3LSGD4zB9U+TfU1PvM5Ij8NbcQ+iOMlPdQRRD1Gd0g+mXUhPsSynz2OwtM+","54M3Pqlr7j3qySk+NYPUPirAnj4Bm7g8Et6jPd64Mz4kk6I9apbwPcolUz7x+B4++tRbPopfx717MoA+s5V3vQLJjD5jcq49IAtzPVg5DD95PqO8Ae5LvA/vFD74urI97ljHPRFqebxUgA0+/FrBvLGHNL2o16G9wBRzvgY8OT5lND898AeePoQXh77XmVA+INkePaLSmT389rI+pT7TvQGD7T0WiQu+GaMXO0qJfD4KdoG8XqIKvP3Ogb7Ol58+NzeDvU3OpT52ysC7gumKPYaPXD59Cmo90NSsPZ7XCz/3oSE+JSgRPRQJ0T3otC8+hn7mvnsLzjv5n829TROePudNOL2K1Uu9xqYvvAHlV74CKoo+xVWwPlDPajyC6Ci+5dhjPNQvzT2RqbY96ndpvbATb7xcmC49tHOWPkX56T13Y5M9zB2/Pv/Vy72lSa49G4A/PmgVCD6/EUE93pcQvdC5orxGipG9u9oQvsGI072hCVQ86xy2Pi8DoT18JWC+bbUSPuWEobyiXL09k9SevZGYM74RiIQ9kyifPR3cyTw8fVI+i9fuPYWdeT4C1QK+aSziPbeCED3bd/Q+WM4XPvDvPz49GUk9yiB2Pe9/qL3MXJ2+Ocl3Prsbw73Gf8+8LPauPXwpOb7wouy9n8uQPY8IXb0B3Yk9EdgnvQUyvr0oQV4+CO28PROkSb54Er89cQVnPY/427xkkam95xj7veyLWz2coG+/vIJbvtK6kTzE+hA+zN87vt+ILb4l65S8i4k7PYyohr3moJY9PSx+Pincpb3FUME9Cu2QPUk1+jtgXVI7FamUPiV/+b0Lk0Y+piR3PUfU1LwAgxg99TgPPm+2Nz5Dboy8mtLEPQF0dz6nTYE9+8/kO7zks70diQm+VJhWPsMhfj2nTb+9OLjKPcGnRz6WxNq9RPZOPWVrHj+eRKK+YMi0vdy2XT6emOc+nPcRvVj6oj2H28a93cdJPrX1wD0wqYo9/7f+PXjO7T0w/y4+PNyUPY/P8b0NXUy+","cHUDPez+CT7p+o291053Pn0lET67mJu8ZgomPL62rz7M3+a8cakJvrdRDz178m692A0/PWdCBD0vIiE+r/wYPjQuND5n4gy944ElO8dNdb1LJJm8blxkPv3qRD5J5Qi+C/GSvKlZ8z12KvU9wkPpPTyU+z2/qkU+NYwfPQvVmT6MNDI+D727Pa4vQj6esqE9PquEPo/plT5OZiQ+QQhiPnGqFD6yQy09f/GPPoz+rT2gG9O9HgjlPhNlLD5xk4S9sd0ePZFuQ75F2Rc+QJUOPdPCtLwguIS9rIiKPrkn9T2AyLO9/1qiPWQO5TwMCEY+kDmBPiEkLz6/GJe6ekMdPvi8BL3Owv29IXF9PUNL+j2rQDg9BMsRPV0xOT1YTxy9NeGnPU0F8b02RV4+ZpHdPXjik73j+9S9i1HkPiXS5D1+zDM+74xwPvJr2T1ECn4+SBcgvWvfpD0o5gk+m3UFPrg8rj2jo4W9CmqJPZcxzj1WJeE8e21zvVL6a70iluq8G4mHPmAe4T6gJQI+bBWTui5q+Txx5uO78PnHPn99b70f4RY+54V8PI7ziT39qc+9WDMZPTNfJr0uWfw9929GPi25RD6vhw8+5E3svaK2L771ZKC6Y2rOvdGQRL0SeGk9mUQvPnfz9L0XMUA+12MbvstQNr2lAOI9KjjCvV4/iz7PeRY9r76bPUzEtr3tDcW9PtYsvf9poTz6JpG9JnBdPgFXLjppjyM9XiZ7PVEfujsZiPG8Qw+6vF7CHj0nw6M9QwjJvOaSqj0/sQa96HglPY8d9z6ZrIc9HCmMPTBDKbwKWOQ9so0bvqWDEj3A1JU99Ai1PVK2Db44bv890QSJPBeyuTxAccC8x8TGvTg+1j3q33s9eY5mPUaKiT2NGvq9TwQLvi3INzwvjqU8gszou0vPBz0tmsW95uCyvSaRC744I3Q8oxztvUC/cj0//LK9vegMveJ40b3kLou9KlwuPdEvbT7q6Ps9rGt7PBBRQz1Jgng9uN8tvuODDbxhEZy9","Xi5NvCDKJz09fkm9oZuhPUnDML5i5NM9VccavszZ0L2X5gk9ZbbSvbgApj3jMIc+E6ObveN4obu9sg4+YxpRPbe7DzyYZbq9h0LYPUQx7T6s/L48//iuPPeOvbyPYUk+ocTGvSo2cT7xWys9btNHOwbwQb3EeG29VwIdPhEuED6Hztc937KSPaInED7Zs5m9QypZPp7I6LypTZ49HuV3vNG4vD2T9j0+LhpyPf34VT0nXaM7oUI6PTVv/L08Uuo+jsR+O8Y1ur2S+c28EUaBPkHzB77BJ/a8VIGvPUoaV7485QK+TgQmvMDW4b3Cvg0+yehXPr3+GL7ePiW+DRdfPgYgRz3vgJe9OTldvFRSUz7kw5A+KNhxPuFLyD0EaiE+rfs+vAt8Hz45q/M8TKGMO14iTD64w509AcmuPgYCPD7Esrg952u9PTo1UT6ZaxA8glf0PTEDuT0+by0+zFG1PR5Y4D2g44c+PJSTPqEAgj3h6Qs8fQwmPiVYkj6vT68+5g3ou46KjLzrnAA+X7oJPcoFBD5dj3I+eijHPSs2Aj5rEok+z4foPQqm5j607Hw+SAKZPXiQQT5sn6c9R8HyPQyuOD3J6+s9IdtDPnrr8z1x+jA9LKqyPZj/rT1qOzI8Eg0sPtOggD2boRA+pHSYPYJDST4HfqQ9h2e/urdfer0tQWk9ncisPZ00LD0ln1c+ItuCPmN4zT1nut49EGGRPVwuGT01Rkw+19gBPa9LBz7HPm49YHGLPMb94D4DJ0g9X0j/PQw8hz5miqc80ZaNPgtjpz0kUJg99O+HPZbOnT2mklA9F5ACPgTd6j3men89KpHaPSoAVr1VP5s9DxI+Psd6Jj7Vefo+1TxdPYu3RbwqnxA+3Ka1PUwyTj0M6Bi9OBSSPXwlkD28QMg8aVHHPFhBHz5xW++8JiUhvQEZyj1Uf9480/CEPkEb5z1jfG0936ypPVmdZD1iSqM8f9DKPrlJ0z0X4589DV6cPanZST0lVXq96oPpPd/Gaj4OGhY9","+/STvfcUxr0fpxi94Rz/PUSqo71ef7G9GyddPVQUvL1f4Kw93kJyvJt0/jwvhh07Sj+7vODkRL34IZ89TwHpvRFnAb3b1389TzFgvS9KAzwGhCM+wXqPPTOZxjzl0h6+cGUfPhPFbr5Ssn47scPPvKjeJL2vmQc9/2gTPc6IDLtxR8Y8FSKMPCfq4D3boZy8xr9kPUSl9LvFA4K8aDMYvseQAD3EQPO9lo18O7VZnT1wuYy9Wd66PO/gc722pUK9uQrIPTEVvT3nwqC+kmwcvoppz70eqa09ZGpnPcStvj0DATY+SvyzPaCVL76penA9OIrsvCZL0rxLhwM+mSJvPagXJT5HFio98nWDPXllET0dGC+85/80ve91sTtgPJY9jBdAPut48T3CEU08z7ypPnwICz6+68g9F4eUPV+Mnzyy+h4+AeEsPcztpjxDwpc+ME2pPUvsdb2kpKW9xE6rPmNuIT3QZTw+nBasve+svz0SMhM99gIePp8lzz65Fbs9NXJoPtSe7D1trRg+o+sJveuoVD5Xd9U9AmIYPu1Nqjxv4iQ+Gu1pPTAqZj4tT0s9LGS3PZTKlLypEbQ8j2k2PgGOvz0o5N88af4XvUytUj49yHS91y8APP52v7zLjkk+B1eNu5PI4jzWwCs90XO+PbC8Oz6KFeg9fBHFOzw8kT08PjW+CQ13PgUV0zs3XbK+MZoIvobERb3B7HW+E6WiPekRrz1KD4Y+G1LAPTJ+2zw8yXi+1g2tvODrxb7JCT49ByKNvNt+qL5qaVC+Jl6Gvq1AAj1HB6a7jYHHPUdznDyVJ9++C8aCPdNSiL6y5hW+vy6EOpvSPL4bOxI+Gq3zPgK3xb1Lfd2+4F22vGARBz01mzM+KWyhPCOcwLzGmNy95WiCOuXpObz+SxY+hbUvvrqXOr1Y+s29UfbsvRZIN73as8c89JwVvhe4772wJQe+2OH3Pfcn9LzcuXm+2QHCvWqeOT66Dga+k/9mvXJKWr6ccY49bhVvvr4Yhb41Fp29","rCUXPrHMPT4sr7S9tm/JPNJwNb1TW7q9aosCvvUXqbxORFm+M+jXvZLGPrwpU+E7HfE2vmZ82zznxg8+wAPLvB6o0z3D3bq8fSEivdgm1r1zECW9eHSjvKJEU76vgG2+2woavuQs+L1aLQo+12LhvVf5hbvIWS08d1MnvtI8KL5x5Ac+D4EUPtUcV73bY2S9/Nr0vacOlb3YaN+90HMlPhCRMrxgeBm9ZUyfuqzOYLtV1ci9w22bvS3qNb3ISDs+rg7avWMAYL32YB4+WIK1vbhRCD2xGCI9jWDsPUJ2jL54UDc+1U9BPVVhWL0IZEY9r7Ivvn0dz7sYm528Lld3vCuj472h4gG+lTsLPooJCz7j81s955CgPXBFSLwNGWW8sBGlvdTws7vLZEw9zxKgPfqZBD1ecHY9IbmJvvdjKzxSRQ0+WUMYPTpj2blNB0a9wZalvhqIlL2xmlG+rq0hPj6IyjuGdmM+qaUFPnyDrL0jOjg+ifiYPY8Yxzw7JuG9rqJSvE4asb0fdoO+UJEuPGypfT1ghwM9YyY1Pu+taT2o+4U9O1JTPTwuDb3eK+W9WxZFvZ5Jer2ZG+a93l+uPCQ0XrzGBE2+HWfZPjrPhLxJrAA+FqqJvixvAj5P5wi+omq+POxFg70KoPm9jx6mvU1DBT2qIJo9+f4YvbR7Dr0DEra9CgRavTIndb056Pq9xJujvHwtgr3+thO+etwbvmi8R743VQk+hI+tvSv9Vb4erLu+xDekvRKBoD52aCi+K3zGPG+yFz3MGGm9rik6vpDFnL39W4q8vWPFPfR+J75/GMS9cTXIPdm+Y71euIm957F0Pdmbsr4f56K+7tTFvFSeq74KUTg91iYdvoR7Or1F6BW+5x7RvR+9pjxWszu9Rb74vUWA1j3C1gO+v7UyvkTAHb7Odv29zasDvlYSir5kGQs9+PPlvHNUsbxd/wa+ign8vRCZszyB/ce97pZCu0VoE74BPXK9lXaiuvipTr0SieK9P3HpvBrEujwzeD6+","hEKCPJ6K/T2Dwou941TnvESP0z1XWjw97q2evc54cr2vJnC937u/PUtgF73mRPI9r4cnPqhupLzsaOY+zDowPS0jdTxUiW49bPrnvLHgw7orMSK9w5aAPa1QnT0zmoc9+o0avbXKqbz/M04+v0QKPhuxmrxAzdw9bByFvJ0ZOD6iMy+8W/y7vQS1Gj5Gl+w92TmMvbOlvrzvipA9jkekPhqZNz6eeeU9aNnsPU1iKD72zdi86HeePZJ/IL+xEhq9tOrXvbCrOr0wDuA9mMGXPTJ9ibuW8Oo9keKbPQF7Aj0CHDM9u1StO4xWuD3JfYI9hZCmPAnLmT1qo/A9R2n6PQO42j37kSs9uaIPPjvojD0F5Ls9gGGAvLocQT20X8i97oiXPYxMQz0wJbo9gldxPS1pWT4rejk96gszPkLrhz03/QK9jSSyPqqe1D3MXRA+GUY2vfrJiTwtTWQ9Jl/GPVpGIz0qyIk9xOeKPkcJrT1p2ks+RjM+vX1vBTxFFYU+7ZCqu1cINj4TlOg9K9MsPfRJEzw1fT29uAOpPizCqT0Kzo495ZhuPqHaJD4ZyJM9BsfGPToELj2iEyQ+DDpCPciAdD4eGz8+GrzNve3avD26CAw+pVbMPCTcY7xK/6k+nVpMPnnoIz6oBIM9oMaRPd7rFz4r8Qy9+7WSPZwFez1XKiw9aw6aPHDXDT7HXAI95DfQvUBAez2Imgg+OXPmPK0yOD2yQyi+w3lNPLqO6bzrePC9H4MAPY4nJz6JR3G9KTbLOm3ESj6X+ZY+K5QRvcXxkT1CUQw+lW3IPfO8GL293oQ+VlUQvZTgGb3mfBM9gmH6ve3kOLyeIj69uzmVPIBDVL5zEOE9cmknPobHjj0E6hg+vYbgvVk6jr0KpN68Fw1tvRDMKD3WQJC9hCh5PSroGr3rb8M8A74NvqM6hT0AiRq9YcePvUY0jD1L8Ik9JPIVvg/JEz5IQKQ9ONKGPRd1i736DLY+MPOwPe4GET0LG6m9QYrwPM9k0DxRAic8","eZjCvESQ9j01OKE9UXbQPWrQA77ON649CevhPRbkk738FDs+Ff83Pv+Wi7x/gD0+vNsWPivp4z2/EqC7XExIPQ/ycT6nZCQ95nIoPuyr7j7ZEo+9S4whPb/MVrv4+EM+eoPRvH6s0z5PHLM89y7qvbPmbD7YukU9Vn0YPQB+DL0fuHM9pwpzPDMOZj3XAyw9lpJEPq2Scbx8r769EoRGPfH/jT4fbZ46vVHNPrQ+RD4uzSI8RHrkPhQ3AL6qlYE+8WzUvD8fM76TAeQ5jQkbPVegEz66q5g8INKjPO1Pjj7ee8W8CQ0mvWghsbwhlxQ9T35+PujZuT1Tgqm8HSiNPpO3Lb3hsY4+aFcmPaQHJD5zPA8+31C5vaMHIbzCkiq9U7LDPPlSCz4NxSk+Ycx3PULlvj2RyD07dSxGPr3OBz9fDHc+2mDcPVBEsLyg9QQ+Wox5PUjvfT0hjAU+cTe6OzoV2T28Xa49Hg0uPs7fJD7fiiM+U+lMPtE5nLyuy7M+uowoPrtg5z0ndmU+2ts2Ps2Nhz5zTTO+OBpAPLfk37wVQ1s+AOtdPjnq4j7+0HG+pTWJPZgFmz5GgeA+SNqWvfP7Vz6SniA+S/TdPTPenr2bP7A9e/IjvRptRL6k3sm9ThS4PtfB5T2G5hE+g9TcPs76gD6jxXw9DzCNPlYXNj7jeQE9ADFMPiVHlToYIlw+Iz2MPRRKXzrFiGQ8lSyBPgDDxj0kOH0+mSUEPoGVVz6oNTQ+C9CQPc+bfD6I9wU8ovcsP4X0kD2uDGo+B+Q7PnbkFj7eRgw+IT2oPde/GT776cK8tco4PtwtNz4/8W0+WI4cPrjfYj5UTiW+cSRrvlN10D6vmps+ZtQMPnNrmjmzEKA9ZoPwvEODBD7Xz/A9BJb/PI0rUr2Nlyo9zhvavC/9uz0Lrwk+rcCEvcwJRT4GDgW9/TkJPvU7L70gcBs+AtsgvuaeYT0DFY68UMNoPs1HYj87bBy7PfBVPB5+R73UxVw+2d4dPrHkbzspD2k+","9ZEfvGuejL3e6IQ9lDCivXthdT4jtxe+PYZNvA9Khz3hwIo9KLrDvUJvAT487MC9C55mPbBIJz1Yphm9/eAJPb5hkD2QgAk9uBQavvxznjs6dVM+WxSuPHCwDD5G4mm+UjWOPenW4Ls6W4K9nnoYPH7Dn7wRWAW+CfggPbuiMjyrb6a90cDVPDtDZT1Pg9Q9Apz6vdFE+z1LbtA71Kq3vc54mjsXLtq9msWjPYpyzj0WPSq+rE7CPRQGHr4SIQk8HdOuPT61vT24Vu29sKkJPvf/SD0M1h0+hEHdPOnjcr2ZsyQ+m1LtPVyeCL6/Zfa9aFWMPf4NIT0e2m08c+kJPbeOLj4zyM89er4SPg4gET5YGRG9UW0XPH7Rwr3jPII9dOitPbj+5D5q7KM8capFPkvVFT6XtMC9oyeqPdjYEz6+ZzM+7+OfvUQqCD43v5U+IREKvnHJcL0AiLa9O0cIPloH97u/RqQ+vURfvVqUn7yLaUm8BF3yPeyunr3c3SG9bUQCP4D8/Dylydo9ire3veVTCj6tWr09gmSAPqhqj7z0xCY+ZmEIPpQKmj5aPhQ+RMmGPth4Wj6PdMg8MXImPmQxqj3eF4Q+SgTHPbO59D2Fq1++HKCBPHof5LwvEac+w4UKvtLeDT4REBk+8dXSPXhZbD4IEGI9CP+ivB4gYz5XlNe9YpMQvhK/Uz5yN8w+StoEvOKyhbyDily9qy+fvboMDz6YiCW+7LIcvrkzTL0mtIG+ubsevRlH773DCuC9D89VvqbcDb6BCNy8c35fvd6zFDsN1B2+xFeKvN8O8DyjBLy9ldApvizUg70Uoii9dc9RvopniL04Pf29vCOivm2ikr3thR295Ql4vfNlir0X0Rq9ukEUPFFCtb6lYCC+KaCEvlwX372tDMO+230mvdKtXL0xEIk910ZBvo6CiL4ApYu92o5Ovj7pML3RZfy8tveFvXAoZL4Cz3y98c7tvXkWsbtsIia+xJYJvuJDhD1IMqu+ON/2vS4hMr7Qbjq+","lsEpPW/oob0nnRO+tIW+vNfJ571IEcM9j0fQvX0tAb7CYoM9N4MSPUdVAry+XMW9yicMveb8Rb0atI2+cu4mvtbtrb1Ctem+Z+3kvEBE6bsEXui8cg2LvQTg172ssmy+AEeSvfSTp72Oidk9H1ZRvRoiZj3jqbC9AbZwPEXFLr3936I9b/yxvrj/Cb0w5389oJobvVlGBb1RQW893Ntwu3cLc76jlfK9zqYHvuBjBz3HFyy+xisNvlE4ELxhXHK+tLVSvs4cYr7cXz29jh4ZvkC6IL2LbIS9AYJKvRwXs74/hW29aAIRve5Uxb1gUEq6HZq/vWX3Qb4jFwu+CvjNvRHJG71vlew9uCUIPRxpWrwtJLE8PAgcPttnkjz0Eq09tbZJva8DK709URa+QlXdPA/u/z2KLBK9EYdPvcB9Oj2JLQ+9eDXMvbuPML7DSs297udGvnpke767ET68/GnyPJxYcLyCen28RvSPvRVicb6EDxE+zt2bvEHCCDz9o568SgiwvShdCLwYMKW+4HmvPb3wALxgHiy9Q4B2PicCFz2lYnM9QwsFPpRgP72OqQQ+sRcJPczj3L0ANcQ9ccPNvXiK7byqEhg9TE6/vdYFAD67DiA+DECEvVa7yz0+yYq9IMFfvSzzfDw5N2E8yJ84u0AlFb1dgjy9WkTMPblZyz1UWRa+Vt5NvZ9xVz0OcG89QFLWvTbZhr2UVSY8c4oEvb1YlD08w+e9NAVuvTR8jL4D/La9ym8UPibZ2TzkCdO82x2kvQeCXD2+skO+epJUvkJKTj3PuCa9VqMLvSHLPL78H5g9AmLBvrtWijx3J5g8DGmrvHnfs7zZUau9TlZ9vXOmGb7eHRy+zRkGvsTw1DvJIhO+JQ6tvJf6+r22ToY8axiWvqUPBb5bEPi9/7InvtQEgr1scR28mCrmvF2d+b6ebDu993sqvb3ebj0q1zy+HGsfve2cCb3CqKo8jJChvkKS2b29vYM9e/uLvGDJAr0FjZO+OSF6PagfND7yZUK+","iGW5PNH/Ib3AKXK9Z5ufvspf4bzgORy+00W/vc79nL7f87O9lm2ZvTcyg7y+Y9o9XCI9vhHCqr3eiNm+cxUKvifv6bwsXHg86NmXvuq+Cr4onQw9XJ/5vYAUib5D1G48OlnuvDTEHL58yRC+B1CfvNQskj3iVUi9NKDsPT0JvLvuRLs8lkwZvPD0H76YDZK+tH6ivlPqHb4fzqS9jJb3vZyLMj7NP7a9guhJvul9mr6qHR++D/u9vmDivL0IJ5u9RpI0PQCdgD09g8G9/GlePT1f573fVb87wP7/PYYY8b3MrBQ+74d5O3XfIr7hJsi+75yVvUDGxL2bD+K9Y/u8PTIq5b1XkP29xAPIPMbNRr7cho+824r2vGa7B76jfJs8JFuSvEfzpb0mNlQ7UUIVvvBIjL2jqsm8j/ilvlpEjL2AnRA9kuGNvbte27ym9UO9Luy+vD4C7r1ik2i8A9yDvQvdC74BqCq9q59tvtI+or0kkwa9fW3ZuskF3b2zaCy++SL0vTPwjr7o5R++e/U+vuRlPL7Tv6e9wC7SvXzM7r0tgeS8AqPjvX32q73JfwU9IubIuwWkoryODA89KlhdPcn6zr1YeRa+w8rWvYpHT74xXHa93+ufvPNvJT6AiKG+VWCrPlrrhj6RTkC9rKTbvJIanz27Wqy9a6gSvZ9EIT1fqfq93W/FvD5klz1mzUy8kYW0PEoIVz0P8Vc8FjnSPnF+d7t/UsE98bIiva87ML09qrQ91K9oPVCgoL3AVQU+xWQWvT1RIbzk0KO9rHFmPXSGlr71SK08c+elvYz14j0GN7S9cMiZPVEmXL21DtO9eg8sPUyPRj13Xrg70ykKPmSwt7yu+4+9hxPpvN8P3bzaK+e8WhvnvegsIj6TTac9KNqAPSAC+LyiFAe9NuOTvu7tZj3M8QK+FB5EPKNenr3aqjW+dXbWPCXl2zzbpVg7FBpEPtQWB74yT5s9tQrqvJX2Kb3wukC+8K7LPb4zd77TP7Q8j2jeux2Ixb19XOG9","Luo/vnnrCb4I1xS+2plCvnt1KD7Khw+9ki5YPdAMmDz6yrY9YVsUPXduK77Oxoq+eppFuW0npD3SR2G+4Z8rvZR7m77bmRk9YAnjvZPSWb43USc935RlvIWQf7zihHO+6rGAPZnzmr6gcCG+2jqfveshz708M6286vGauymJfLwQiAm+RpMPvjQfCr0erxq8eeIGvokRPj0b3s29HcxyvW7Ibr2OQCC+iJD5ve+2571jkMc9Qj2DPReODz4a5qS9rUGMvv7ZST0PwoK8nncGvvKKbzy6sFA9IENnvKcqxr43dv28bZwIvtdQWT52hGa8A7HHvbg/gL37yyS8cyMCPrjqJD4vxTS9QB1APr1BOr7c0c69U7xNvhtlxz0K5YW+YlwavuePtrwaFdi8tysXPmE/kL0N8uG8pmcJv1wSSL5nb6c9hxdGPSTQZr2OD+c9kPhdPk/NIL2g0kK+lI6uPa4Msz26zoS+aKduvcFZcj2CPLe8LbnMPSVu+L0mElO+A8qKPFm9n71n3NW+rj6gvp6zhr7eoN+9zPdXvW+Z072c2C09mjRnvb2yor5DAIk9fQhxPd/oq77ktgY9tEyWvvR7nL2pXby8FfzyvYABvDwEFRM+mp6FvYyH9b3A1Bs8x/YsPtcYNz7KhOi963FivtR6hb5sjZq9k6yVPSDAmb4owm09fjSNu2rjDb5JARu+wPGtvZWBtD0n9Ji8y3S6vcFCVz5ncP089RD2vZC1FzzI+gw+/xKAPS1Dm779sKa806bZvLVHHr7abzU+ZBMEvkJa/T19gR68SMikPZqCrT0Fc/o8NmUOPhGrYb6brCM9WIpevWvhhT1QTA++msdcPjbnQb5QVrq+kRzjvm6IZD6dwJ+8GImxvB6pvr2yMMg9EUEHPbyc/7tM6pG90rznPeTzwT2/mNc9MPHuPTmnJb6YlSK+K2lJviLACr6ueQS+Wk0EOlai0T2YMpg942OdvulURj49qHw+y3qYusEEqTxl6yA+UPtCvpXoHz5VU3i+","MKfaPcaMvDs9kwI+HS6bPBgDCD0BchW+aiktPsOUtLxFRI09k3rUvRk4nD3E+o+8viYivDZvuT1hx34+ymqOvCkJPb1DMrA9IIcQvu4mAr5a4kW+OpN9vRg8eD2rwO290BsOvlRjcz08w2885Wx7PWd6Db62p6a8x1w2vreMPL5TtFQ8pq7sveD41r0aHKu9QPn8PMgdpjq0v5W9K0yAPfzLGr09l0S9KgaSPvo3p76izpC9jHgPu0CzIr75Vgi9gO5TPI3dDz5IxCo+u62bPMwxQD4DSGG8lM4svtJZoTuFJ5y8LF0sPVdRuT3LCRG+RIrlvDQor7w93g8+qbcEvsXKrD2BJRG5FmyDvB8IGL64hZo+5ZNkPWBMzTzP/CY9yI0/vQejo7qhnpY7Iksovglu8r12WF09jAtcvczMnL0wTSy+toyzvTyujL6gRbW+EIpnPhgGiD3usk09YLp6vWcSTT6g8qK9SCskvjM3fr4Oiba8piVWvW14p72ogm2+44/DvWD4Or1HIr+7zGGrPHkDnb6AMze+TqQpvpPGnz0PqF49Z9CVvk9iD74Vs+W9vNIDvoMRVjwk3kY+swqFvtpIqz1glCQ+wqeIvCAagr4htx0+2DxVuywnjj1nqHO9pSLCvVNCjr3p/gM+Ct+9uzNCn7662hc+eDAmPlVAAjwmAw++W9nBvVa2+j2UqZk9c17ivfc6WD0XNx49YIAFvP4jWT0aSty+7y0QvhThHj3w+Di9GvsTvQ+Nxr47OCi+1IwNPVrjFr6e7SI+gm7Vvcj8L77b1XM9woGPPO28AL3FSWw+JCsPvmQmvz1nN5e9xNzLvZDAi740p6y9JYKmvjnmkr1oXGG+HhfIvVDBXb62zEq+kjlYvtFRkr63zKe8+z6ovhtqCr67HFi+5JJUvmT5r72eo7m9xED8vcHQ970qUBe+ur60PdhnC75D+oY+i0NtvouJUTz+hBO9NMZIvotNxj0oUEW9BWiuvse5hb6mpAa+F80vPjvdubyUAag9","+QK0vYvkuzzVKui9xiCuvfCE/L0vtUI9ydu6vO0fTz3yksg9zTyMvWkS2D3rfeW9VP68PdfPZL1Cl9C+ATi4PXQxg73KIiy+AQOdPYlPx7xRUoA8c7sUPsaWab1CFJW9NZe5Pa/Go73kJDK+dHNFPjGB2r2/KCi9WgAKvuO60r0fEGA+7aLyvmRavr2vSCW+mvT0ut64Gz0/TCk9wey9PVXPWb2zU6k9OZePPcCRJr1PPRS+4woqPlRXmT3O0wm9rHTCvReT773qbka9DHI/vs00Rr5oQj0++WX7Pbn55L3u9cs97U1JPXW5j74hkD49CMEVvvmyvbxF+8c7inL2uhUrmzw2OQ4+2Kn2vHqGGr6I3S2+DY9Evc3spL3/kim9sIPLvLF3HT4XGPC8FMPPvLfdMD26MqU+YSv6PNXBLz4PBy69991UvfF6hj5I6h++O/gOvmQwxb3AIae8KzGXvf2x/j0NgJa9JGMfvk19Jb4RUrA8yzc2PmgUND3re0a9urc0vlGpYT7/Kwa+LtHSPb4fOj5p5v68THEwPRvRqzrTLb48Sl4fPvya+LtR3x69bSooPWmAsD0+HxK9AQCDvR68Ur7F2aY9vtMuPJLyTr4D0wu96kOqPD5Bq7wAKr49n4BgvXGHtjwAH14+PEz4vXRleLw16oK+12EGvnW0Dj7XRni+35U6OmNZbj46qBq9jAziPWqtfT2yg528GLFjvm3qub5KWIE9khsdvevlUb7Ljmq+PiYLPVoYoz2SqLo99zcAvoJRoD14UlS+Pwxkvhl48j0FHZQ9eek2PoPIsr5Eby0+ot2mvsYr4z7tuaG9PS7nPTimzr3fyzS+2IQ+vtrfmL6sY7o+t/OjvSCvp7wCmeG9NMeEvvHQnL1oIoO+AAwvvvFq5L3E3iu8ZIFCvqGwEL5kwyO+1q5Rveru5L6Q3Am+tkM5PIBiID1CP46+kJLCvVr8wz3wzlY+mU3tvYRH+T4xh4W+JQgQPS10uL5zZy49QWiSvg+YoTykOsK8","eG+2PGcm2r192fG9FDxePiHbbrzJVdg+W5gAvs0cv7yYCLW9S4CTPrsfDz4bn7w7luj0PQ6KJr0Pp3o+nzyPPbj0/r23k9U8CdpAvtwBLr3imyy+o7sgPunNnz5pkR4+DbaNvWRgYT4biB290YHVvdYpiD5f5OM+qOX/PeGTnj7+SsU+wj8bPiKaSz73xQo9FSK2PoJhb73wrsc8YfRbvbIoIT26upg9jJVaPhPPOj7pCiY9IwW7PjOg+TwQFsY9CyN0Pa12L72yc/Y945OzPaJfST2aw+Q96KtAPumRML5agqi6xHT4vfI/YT63/Cq8EEUmPialfTzMT+w9P8QkPhlZ070r0cY9Jk8AvBzIfT6j5Kw+0gDtPTkfLL3/MP+93qViPYNNo7xPq4c9yJhbPlGP7Dvvgj68ZzjFPnnjEb4GIrI+q69KPuXacT4Sg3G9zPnCOdhfJT6bYgu8zlu3vHu7uDwm1bI8AruXPXM33j0wFR69fF20vSKrGD6xVlI8UdWoPWBZyz7110k+gcXiPE72ID5n0Ji9B7OjPWn1i74QMuQ82rRZPRZwqD2+4a49eBfivbIeB75Ezgw9D7/4PZXqsT2fSXQ8PbUdPkRAIzwtxNY91gKZPGe31T29M4c+E+mCvYTWTL1P20E+XjR4PWUSMz6Cnm89YhlXPSfaLT7pMaE9JTuJvv4Klb1fHIc+OWPEPXDlnL1vFUS+Nr/Bu2EdN7zYNwK+AkouPRXORjwRra69RWkMPjeFm70n7749FTPgPRVqHL66vXO9mNjKu+rHkj6ayiY+DpCfvbmBSTokUtU9ouUsPWPYlrzVraU8/FGzvJzxiD3iAxO+aCf/PY/dtL1ctTG+N8IZvZMlHL2pacG74D2kvVYTUj3lQGo++W7ivSbA77xBqvY77/AVve6omTtzoLQ9hlojvoY8r71Jqau8znEHPnzENL5MjV28CJRevTzfqz1g7n49dScUPiTYNT4NpHG9qUdOvVGfHL4Y1ES9AlodPPlwBL6O9x09","DnCDPQxusTz48WS6XU6vPVZRFr0OphQ+TymHvVZBDD4JyNO9y15nPpexVbzdF88+ZpTjPSECoDxnIae9yqwGu8IBSj2n9z893YMrvYLojD6GQao8XibJPUO1ZjtJzUI+2WLWvVJJpz4lbD2+rfq7POzkNT44GZa9oFPWvAo9fz6cXUA9N/eoOyqxST5njYe8uCZ9Pez7DrxVxwu8qUYxPnI3nT1/ZOE94cIlPB321D1PQxA+oBOgPoKkyjuNns89YWsUvs2SO74cVjw9TY4LPqqe+zwnOPm8snEYPQ6IPD7VKfi9TOB5PlgYzb3J+Zg9CNOkPpE8oj0rgOa8bw06Pn3WCDzhDS4+XExmvb5PBz7yNcI9cbo2Pj7/jz2vsc+8ZXsSvbD0bbxdq7U9tqnbvCUq0T7Cza29fE8SP+vsHD4qOBW9PwEBPiAOgD43yrY9lnI4PgETGz0//lA+QVievAR1Ej5lkSQ+8ofrPQPVTj2PlDg8Vir2vN+gPT7ppYs+v7WcPNJgkjxFLDo+vHwKPqI8AD6YaoM9xmyVPguSjj0M7zs+28czPhz8oD4Xc44+HvbAPX/36z7Z3Fe+wqbAPLgWET6b6uQ7BFfoPXKWdL10z329DV67PBt7Xz04lhO9GooUvg+Jy72nU0w+Qt7JPluicD58oO09TNY7PgGtnL0bJho9Uq/KPb1Ruz3hq2M+sz/nu9DI1jwEUYU9V9ruvZtOWz2Z4DA9wddZvciovT3P2iQ9X/+vPJ468D5SAF+9mdjYPA9d2D7v/MI9LLeHvVfkET5SbyM+ii0HvIU7yT17Cfs9UGn2PSWcpT2v5iE+V4zgvd7ggj3h7hW9oXNgPfIjhr1PPrU+/AaQPXqugL3/whU+NvojPkrauz3OF9s9fqGNPQi+IT6pKKo9kbcSvMJNKD14+Im5FImRvfDC+D0clz8+bINcPlPubz3Q8uk9lDySvOK+bbw5cdU8QwWCPvZ71b4QK1y+sHiAPs4aPj2INio+aSAKPnStRD7cST8+","JzoWvVwIor3uy2U9mlRoveLJyD2Xj5w9DaxZPWv7cb4b9189mPmuvH2fFj3R4ZG6I2IyvDYKXr1spJK9I39SvmV/AjxfSJ49qKJaO3TXHL6ZOY8+QzjAPXZhY71X/f69w3+9PW2GyTqmYPA8wNcxPZatNj22S4U8aFKyPBBSw7uType843Mqvb7KnD27hZI9Gq0oPRtp6L1b/Zm9pL1RvWiaOj2B8au8VcWhPRyoXD7Ggdy9USmpvYcbbj127PC9KnqrPQmzjDy6G4S+/WFBvWmf9b0a27E95n+mvTnjkT2CPj09GuXpPTKmxb2PX50+a/+uPFIhlz0JXMI9lhRHvRn+sj1NshU+Q763PSrIRz5P2Qe+FhMKPQwOfj1mQ9S9ADDVPWK8zTzqwim8W1abPsRGVT1hn+c8lNaBPdXZ/7yjCbQ+qRKwPU0u2DxRC1M+gsxJPf07Hr0I12I9KtHvPQFpQTvotGg+SvIOPiWtGD48RB8+tnaDPHaNFbyqlQy8+bWhO6ynFz7JBP08YfNFPQZ2oj35kCG+JfhQPhfkKr0N7q09oYb5PSVGfT4kYMI95zbNvAsV673Ev6y9hdBRPo1YUT5UTXk8NcGDPSGCdD6nDxg9yi+ZvRo3HL3GfMo+nodxPT6rSTyI8du83TWgPQhkQz5JyPg8s0+LPblgLD6KByi948tWvjn6Iz5xBGu+PVRGvrA98L1X4429cDUBvndZj7woH+W9Hf4evZqWpTxiz2G+3ycLvIayDL7J9b69Mhcmvqz2ur0sBvK9ZCjSPZj0fbyof5+9JtUivrt+7j2guga+vuhjvlGZV754y4s8N6fnvIv2kL2Dm9G9dXorvsYJDb5Sepq8ZAKcvQ15Db3B7Ty+0qk1venDZr6/fwG+lW3SvY3fU70uSXa+HFlgvaWGEj2hNlW+UU/Yu1fzbT0n1Ei9yYEQvsoQyL2T8Gu8oQJJvjA2xby0Gza+4F6gPVwihj2ibBy90CQnvjB9fL6YdAq+sfelvTmYrr29HR09","+akyvh2lN74ACb29XmiyveLfID2+Msm9A0vEO0dSFr79dqE8IjBdvfUqlbzNm128zrUXvb1nfTyszsq+BOwgvoSQ7b2y75O+W2dnPYaLmbr0U8A8OgAEPR4NQr4C4W6+ZXXIvVgH/b35guu9yYVgvkK5Eb2YUN49wURsvQBEgDzg+YW+6xkCv0gWFb7GxZ29f2zGvV9x8LwavM69QS1jOwVqAL4ah1u99O2DvldbY702sqI9AqjmvMpBAj7mtYi+jZUCviSKQL7PlSm9s1aAvaPURr2cu1Q9d1EMvcUs3r5Z2qU9zYPyPKqd07wGW7s7/ICMPNdyaL5GVNy7w3Wkvf5dpryHqP09iFkbPODVwDxjRqy96NN2PVzrqrzs7zA+cIwlPQVc4TzBy0e+NpTmO3em8TyDbL875FA9vraIAT5pbks9Q8mUvSJjGD1LNLc8sle4vT7JDr4EKxA7d7CyPR3z2b1wP7a9WaHZPFXrMr7Xo6m8kxM5PJl19byuJpE9GcQSPor7mL07E6y92cvBPUZkc70fcni9GnFdPb3zND3bYbq8bIWQPb69bz0vaxu+qAGLvJTf+j2oftu8bMBlvY+h4Lxr7XS9lr8GPhncIT0QFrQ9FBQdvtaFKL2uboK8h69avt6OBL2HA789ADbwvU9gYz0yASU9obGBvQ/eIz0s0Aq+uGbYvbpJa72CERO9K1QDPnbYDj6X0Fa9IQ7yPc5YJr0Z0dC91EMvu1VEoL7Arz++sSxMvYyBiL3c07m9iWrwvS58bz07K3q9mskXvirtOD3lUMW9OsIiPddAXr6Iq4w9aa+lvkwOpL0P1ZW9L5IsvbWzKb6wFQe9DXgePQoiWb6NLSG9fAxavcWRUbubHj+9Y4pxvUoKWTw3fp6925hnvT5D371HoIa+lJs/vVCqUb5xMBS+9xWjvRzO+b42A4C+Gk64vebb3jpkjK6+vYOXuzffyj0PoEu9lcirvi2NJTur9jA9Fd9MPBBiV735S0C+LVz5vWEZoL2cgxC+","VWQ5Pr4YJj7SEXW9hCxTPtSeXTxk5rK8bYbhPZ8fHr5d9R49jTdUPmMOsL1Fda49SG2DPo94NbxyOoc+qbM6PgeACj3Z+gs++Ny2PcsF2LyILKg8VOxHvdCYMj7I75E9DD62PUlQqD0XCwA+dL/BPVkLrLwz/Cc9bWr9PuXzgz7t3629eB30OyIIcD5sx2a98BIEPq4jMD0PrK886lgCPOS66j1itPM9qhAvPrPud7xCyoy8VADjvTzR6rz8Bqg9L+WAvTG15T3KZUM+wGz7vS5mvrxO74o9y01yPWBaELuLF5U9XnKQuZzraD5B0JY+vgOSPhCKoD07xRK8m7+6PW45/Lw0owC9SWjnPYsWID45Zce77gCiPS32Fz7SaR09kYABPSTNRD2+ani9Xx1cPeSb8zzV8H09LmnIPpbgAz7xFKo9jy28PgHJpTwoaOq7HiQHPQBHhT37vBg+iSaxPTj7rz2Bz7w9q/8NPuQKBj5RyW09r0zpPUGlcj1f33o8sQH4u+Rk0T6GjvU9a5AiO0Vlrj1rVNK8iqvHPYBpqj2gmGg9n1LIPbkk5j0DWYI99Hv4PPlmJz01AP49kEMPPnwUNj7baXY+hm8wvdCRXT3JngU8vlECPgoGULwZ+xg/Sv6VPf9U9rxZR/87DbpXPTIq/zwYzNQ9TQgQvBDBVj2iawk82I7XvShxED2ff608J7+RPW2Wy7zCgYg8OZk7vUSYbL2F7h2+YdwQPr7AKLqXd/c6aFYbvjduOD7MqqI89QybPf+SIj7d+I08mNihvTvujjxjKQ8+0TUTPv+ufTz/6h0+xldgvYsihz0Yi669LdL7vY3Wob2zd568LZKcPSPFC7uKpPA78UJZPgGnabw75ew7p3vOvQHcB761hQq8Rp+LPefRNLwFmLQ5EOoAPhndB73Mdzk+Cys0vgpRjDzh5HS9vTq3vdatAb4qkCS9uQp8vTXacD2wxjS9sECRPDqIGLxeEmO9Y8a1PLQxBz1V35K9do5CPM2iYb0gJ+q9","9LGjPU/9qLlZJXe94485PseDBT3NlGg9IrbTvDASkj1GjG+7chetvFISfD0XJvI9HjfVPecdBj0/HA29y9bKPe2bNz4Scpm95sQLPa2jjz6IuF29oIaQOkg62Lx/j4c+3FknvRvElT5C2qi9UkZmvWIyBT0DB4s9keQ5PZmNoD2L5cQ84KKfPeyuKjw9U9g53XXiPQeMDLtc9Fg9CIh0PXC6yT2v09w9TTt4Pn5zDj62FRg8f56YPaO6lD1eBfw+lzJMPmmbpz2kEa290QlZPqgBM76XqUw9EMTdPfdJWT4LoIe9fhxLPi5Pqr1Ex5E9ves8PuuChj3IAvu954iqPk+9Q77BlDo9ByeUPR+zjr1B57m9o3yuvb8Wb72SGbi97AI3vSQcM769BYO+Z0iXvHpzQ750iP69D2fKvo6DeL2DOOg8nDpCvqRJYjz/Dhw+P9M5vtYIGL3d1QQ9zSqBPEY4Sb0z2Vq++UoZPBJFvb1ZWFO+bVsQvpj6lL46Aii+5JU7vbT3JT1T+LS9Ahkpvf8FmL6Njpo9irA1vrJCK75Zswy+Xy6HvSf9h74n/zi+YTlNveU6Br482ss9G4ESvhD8kL1GGNY87P+svWLIoT0jnpu9+aqcvcyBkTzB89U9OhnHu5oBnT1kB0e+prYnvmPPeb1b3oO9FD+MvIpNU7znwgS9MVgvvTUEZjyVYH+9CMYPvlU8GD5mh0G+nL88PIN9tr00ORy+vmKgvTDiiLytmqG92ynavA8hDb/MhOO9cKSAvVW4h74M04G7NiUKvh0OIT14uZy9EDM4vh/Ecr0g+dC9CgrrvXfdvb3bV0o9TItAPT9KhzssP0O+NfsHPTR67r2P9Nu+LLMRvVOBUjvlBEi9x8J6vfBUUr4SXN88d8CUvaqFGr5NJSq+Acheve2PwL2ux+A9ykmPvdCrwr2RxY++4/wovq9Ihj2q0RC+p1jaPCDUyrw+Ukc9JIeTvTE+Eb62PMK5x5MEvk4MFr7R8YW9q0BAvvIWeLsWuFa+","zWdWO6qXuTzdFcA8HQp4PXqNiL06OfW9iXUzPNvezD2L+pI9kQDXPGkx9jujhIA9TCxnvdDTGD6mgxm+kQBzPSp0Qrx+8FG7o+68vRi6dL3Mfgy+g9knvjQUDD4IQs48wvGGvaDvUjypl/i9yqWEvbXnazzOJra74YrnPGMCHT1lx1a7SZlMPWXYyb3P+K07lgnRvMUXob2qoEg9+N04PUJAnTxUiTS9uwykPBP4Fr3+vU4+V2PePQ/8xbyUFaa9RGoBPDxYFD6pz4q9k7wjvcPNhz0iMwg8txAMvOi4qD2hdtK9ZH6MPRW6JT608IM98iMbvUyIYzt2f+m9KD5NPE/81L01f/G8upE8veE+MT3dEmA9r/eLPeheRr2eQMe9K9xJvBDLcL5oWPi9+sVSvg4DA74T8ke8sqHIvZR+I74TC3K+6wMPPfe+rr2PUd6+46c7vSNf9rxknY09V/28vqGALTsZh4K+FZHhO+QVDT0Xf6k9YUlAvKSBkr0UycC9wL1JvuM+RL0lw2y8LC5mvIkaXb7KGLm9BXswvfECwLkhSIO99PEDvXM7YL6TB/C9nlKguy2HP74wo5U8wTQfv2QPoL1mGVo9Y2uXve3Tw74HVBm9I1ilvXL11L3TjsO+O/MGvfYrOD63bQi96S8Mvd3Xer5CUJ+9H4AKPpCOOb4AbO49uwCHvqQ2mD5m8+m+n2ubPrtK0b6KRQC+RAIjP7kOTL35XB88kGOhPWJ/Hr7xo8K9WqoLvjqK+T7Y3Cc8IXzKPFe9cz185329QqwGPv0dPj4iplC9m7dHPNDdDb1/UhS9DGMJPvhx4T080cW9jAVHvq8OZr7n61s+XE9mvhNmIr+Nf6k9d9YJvNww0b5j4AE/3MpFvTIyhz6hDDA+XqtIPm8E37y1Kfm+Um6nPnh43T5XuCg+yGUCPmpwij4RbMY9UK7qvb3mhD5fkyu+pDEPPxjdvD1rd929Dv0PPkPA6z6aT7Q9twR9Pj3TsL5ABOS9lITuvcbaB7xKtK2+","63/aPVGA7j7HvyG706WaPEGQMr49Lg2+fFgyvuE/+j6zipa976bMPqIorr3NydO9lytqPPqf8jwFN1u+JI69vkvm7T7Scp+9KQUIvph//L1eHoS8xq8EPjnY6L1CVqM+yM2zvSfapb1JYf2+qUywvVq+DL7NVB4/dWgeviy2nj5RvHg8iDj+vp81Iz4FTo29PymmPbRJv72t9He+Heh8Pm9rk74DGAA++ZU1vNLU6bvlAAE/mMj/PVJHzz7SNJG+w48Jv1MmJL1auwM/5wADPoNTzD1MONa8AU6PvTSNFj4f5/k+dowBP8AAi7y4NKo9JSUFvhsuir5BBLQ9hOACvvvTsbxPGRu9F2KsPfTF/70iQJu9eo+NPcwUl72rz/C+hi3svdqXyb7HYYu8vQeIPe/Q5TxntT498hBCPrZlcT1cvoa7Q/pRvrvnhD1sWxm+TQ21Oxrqwb1qLiu9OMSbvDUOND4MWxY+iY6HPpyzmjtMf88+zT+wvvJ3/L4/VK+8d8YDvkk0ir3A6Yk8Fw2MPtxt7Dr6lAs+dIvEvs2gbb4A0t49W1L1vpJRFL2hcFC9ESuFvhM0NTwwLIq++cndPcIqoD02Tue8XE5jvgs4rL0fBd09hz9uPibPxz0onAC+lWuxPn77JD6sqgw906DZve32Pr6enck9XsIbPi+7Ob6ymgq9s1c8PNoEJr7l/Sc+mJ3cvMAN/7vMvQi9MfygPlPa5j0U1D8/FNkiP5hqtTp5DKE7AXAEvTR+uTwxrGm8x5I+PoAmnz6qclC8ec/BvMIi1j26gac9jOLwveUI6L7gpZg9V+x4vf5Uy76PoAm+h6AdvYj9Dr/S3UI911jWvWZRoT4J1RK9yl8WPjSXK74etgQ9HjjuvNEZBT1ht2M+pqCiPSi9577HdqS+MBW1Pm6q47zNunI+LYbdPZ1Cgr71Pio+bFuuPi6QrL1H/N89Qkk4vvtZybxY7Mm9CnaVPkwtRb6RZK68mEdhPVqZPz1u7+m+iZCXvgaUWb07tCy/","M/iSPdcwRL2khJi9COQ9Pkx2gDyxsc89hzo3PSuJp704n7s+XAdoPpNERj53gSq90izoPf4Q57wCnLC9jXkuPfLxqj78axy+JgdePqbyXL3CV0k9c1ywPftfzz0tHSK+/uH8PSCukL3na9i81bsLPk7t4LuSlJU+rOyNPsNMqT7nmG0+BEz6vagiZT33aqo8KaePvAlmhj6i5/e98Yq/vUIBxT2Vs1Y9m+VxPiz6fL05efA+FVgcPiVMmD73SFA9UUPCvSd0CbxgDpI9jPfnu55QQjzx0bY8XQW2PQR76T35OJs+P/kwPa9uOr3kPjK+jc+LPdbGhL3MnKm9MauxvfTiJz6r0zW+yI+wvJW6uD5lpT4+Jgk5Pr5fiT3eeou86XUVPngrh70x6Zc9y1SevFrWpD1lJqS8auGdPkL7Zj0alAk+aEvMPURQrLwD6AG9p1bavWQg6jwLRvY90QZoPpwsdT0gzmQ8ljisPQrAnD0OaYe+UZTAvVxpCD29pPS8QAedPt3S9j5LGRs9EHFgvtHwtj4emgw+Y8SHPnY+/b2Qum09KxfrPOjxOD57wnC9U2ioPTDgOLx6HQk8qW5qPl0+Yz66tWU91OfSPMvfWz6I0g++dvByPfHO2zziXMQ70RNgPhTJgj5UC5W9zbpnvK0Yoz3+IG88BtflPYnKwbxvAAq9c9SEPTU5gT2Zgh+99qe1PfZUK76FIbW9ty2uPZNObTwLCyc9gDMJPvJWOTyOtzE9xO6BPevCdD6HAta9R2IDPRpZDb3WMc06TjhMPfa8i7yJboc99l9pPURH0T20lMo9szomvtFhlb1zMJe6trN8PQ7pMr2JzmA9RlauPVOUgbz/9i8+olKJPZYF1zyYnGK+E66GvgBGaz0XeQI+ve/9vX556j2Xpjy9C8onvQumHL0zCrO9rnv4PW5ckT2LgJ4+w0zwPS8zzD073jo8vBr+PcN2gr1t8Ee+r/d8vFpb2z1uxxO9s/aDPD0yF70zLV2+amOIPbAZvrvNGRo9","RJ5BPhWjMT691P49xZKePunyfjwwNgk+7f4RvLy5lD7MgjQ+0i0CPmbgEz2IGzQ+A1zCPRNzzD3EiGe8DbvvPa4cWj7wHYU+x5CAPulgZT1wrLE9Qw4xvd+Fwj1YrEE+Kij9vCAx1T4lEGe+oK92OqrVs7o9dvY9je2rPs5kED6j6pQ+HEDJvdHaoz2Msq89rcsQPneuqT7tbzi9Vk2oPoU/+L0+5Io9hlHEPRDghD3IJoY+HJ1zPhGqyD5uhfI+KDb0PW7TV71qJ9271RmvPocUCT7ZUSi90uXdPdeHDb7Anwq+y6u0PfRnaD4GJSw+g4/BvaQ5hb1Dplq90QkGPtHbtDucsy8+RSbsPugWrjktCkI+7+tQPUm8/D3r6sO842EfPkj0ObxsOOk9lQnGvSI5Sj56fwi+tYX6PNguUDxCSFQ+pL86u3nreT0sbZI9l0c1vu8E1j0nvr095i3vvU6CgT6YSgu+P40gPfsIAb66MK+9oFXMPVsYYz7A8QM/+iiMvA9e9z0gKCM+mS7VvCEABD25EuC8AcyXPpzepDxdKh4+KgHTvK8Rszx9sd88YDM0PlDZkT3Xs3A+7/1+vVxDWT3KpPU9C8G5PaJPLL3JoAu++SMkPt3tqbyPhC48CHr/PZDoVj5dvOI9QTCrPZ+5NT0rUNq8pBcxPYgQS77anDE+iqzFu0T4VT5ddT0+QNCKPHdiLj5lF0g++9kLvr48VD1Gbxw+Ds8ePewNk73BEkc+f6mYu0TQsD6A78g8nLsWPgmv2T3GTbG922k4Pv9OIz0YiE09XUxKPvh80j012/s90L2aPiX2iz3s3h+9B/chPi/bIT7IIU872cQiPdTfKD0FNs0+l5xcPlzpCj1++RQ+IuyFvGWuqz5wqSa9SNxQPkdWIz4yjUc+tw9JvR2xnz0FZiu+LD0VOlTDYj6hO1c9NW2MPdXbgD3nh4s93qaZPa8u+z0kxhQ++NK/PkJxFT7xxgg8idxdPsLBED4lc0c+rgX/vb/f8TzU/2Y9","VctGvSPwJL6FwbK9EgaHuya/HT2oH4u87Pu5PcTtQb0LkKa9MTwKPSfqFT7yYYA9PAAwPoV/CrxWmvs8Fkd3PCCkmT0GlwA9M5vcutivrbxc0cI9W6lBPdW0yDyj+S49FBjWPZvlpj2QNI+9KqVkvY5ptzyz3Bc8kZHEvRnkJ70nTvU8Nl6ZPA8Psrzdy+49QsqFvdQzLz2D95C98btTvcJdJ708I649puDEO+G/I70MdIg9RxR0vqojwr0XN/M8OUOcPYc5nj6fxa68cbScPTgGmDzD2Bk+3MebvTT1Fb64AnE92tKWvTkBj70bWxs9kpbdvMeFo71vZwq9ZPqjvX+LGz4cEpo9vSuUPmU8YzzdUbw9hjMnPYOnHzzFnkm9b6i9PcBFfT6lRqa8qZ6ZPVnddj6nYji9RMIAveT/fjy6FeE+uv45Pb5cRT7WWJc+UiXaPGZ0DL6ybBy+Rv6KvRIA/Tpv+64+Hg+rvOABDT7oIgo+KwmJPbIxKD6lKcw83Y+vPtpOOD5WrqQ9B2Savf/N2D2S/yc+gqeJPR7JO7z0kgU+Oys0vaLSlD5L0p49yeCGPVuHDj5oMbw82aXQPoKlLT5z+Si+WbMQviyq9z3Hw/69UOCUvZz7ijuz9cQ+N5m5umYW5j2zXYU9y9YyvZczZT6dHoM9FjfHvQfsBT4Jw4k9gvtKvYai973nOTW+hLHKvRMwvb0PJY87goBVvl9LB7464li+yV0wvTlK472h+0O+4dJbvnG1Ar/fjGm+KWnKvcAOBL5w55K+mKzWPeLyzjw04Py9FwKAvpeJmT2z8Bu+1faOvjLugb41rtu9P4sevmiwjb7/5M69nZtbvuL4T766h9O9fD26vZyZZb63uVW+jZ3fvZExMr5aPUu+fn4lvrXKQr3a7Ae/Psm6vnI9k72IeOW+HPMQvrrEyL3zIBC8mL2VPch8fL5Msye9UOO1vGubJz0/poO+s4SlPV8mqL0hm0q8JzS7vdZiVb7tjnq+eIAgvtX3ur7MqqI9","cMEXvsIoDz1qXGC+KDyevrCZRD0ecKe9W0NyvlMbp72tqIC+mH7svZ4krTumLhW9SLYDvQ4QXz0zl+O+UoYVvqf7l74omUC+v+83vZXoJr5/KEE8HLHbvV4UZ72axa29XAPXvebBvbw5+gW+46sJvbWlyLwt6wk+SUtLvWVuYj1XdAG9CM3ovoy+G75VbpY9rG9rvrfRtbymaU89WXorvZeT/ruxO4y6KhZtPf2Cgz2C3ZK8BoRhvebHtzxyhHC+mCg3vg0I4b0iv5090HrZvbGxRb0dCRG+bHhnPZVbnr4ZAxy+FwVKPaM8u70oRgi+QAqVvZQFRL3oM0S+arwUvqT/z72XqXE9JhWDPUi8or2K3646Tzxmu8REvL2bIYs9xwh4O1nMqzzuWQu+gTyau9WZGD5bhBA9nUcavRDJvj0m9Y69uLCKvdmfmz1C3I69zXBsvqu8mb1cGN49ZPRnPg1HOb42lwc+ChsmvcyElT1riu6708CMPe17Jb052yk8DVGXPK1sjD2Jioe9vLSpPXPPiL2R7nI9M08WPvvUEj3tfNu7X4T5O00YMr0ofSW+kWbXPUCxpb3Trae9VmT6PEgqDbykGqE954NDPgiVv71yuwy8kmAjvqathz3uScc9sNsTvgOsD740ddo9t8Q8vmckDD5eIIq9c98xvlgCJL0VeQ++MBlKvVBjmL1K1Bm+/mWDPTrYr73DuDi+k2LdvcEoPr5Ldx+8N3cZvd3hqr49mHO+h6/bvbamvL2XTJW94uWSvtNCnD1Th8y9L9tHvuUIgr2VBP29jzrrvb5mDL6mGZm923EQvzZ7Eb4g+ZC9DlAzvWHAOb6lShW+9lEivWACrb6Ox+C9Aq8tvZZTg732XzW+xka+vcNwz71lPYC9JpwpPS9MEr3Qek49Ov8Jvq5/fD2IrX48YtdZvJn6x73bgja+tiBnvseDS72btlW+SR0EvMiwL72K12A8zgOkvuTnyz30NA69d3kYvn09wL25x7e8J2y7vYwrbr1Bvf+9","g+MBPigSPD5h/cm9KJ8zPoSupT6GAmY8CcuXvvMbqD4EeO49Hk7aPjgulz7KHRA9zybhPQDiBz0A+7c+1/GtPBsfYr30/qw9jimoPqyCfT28AHm9sXckPU+tvT2Yzxu+UIwMv2XPCz4uMLO9P0sPPuHFnj3fkqy+cDBXPoXwBD3hD7A85JQ7vpdOmT4WXOg9xB20PZzhML414gA+f5cZvQHkzr2j7Dc+YasdPu9Zm71sqnW+chf9PKh9670DAA0+UwcZvjemoLsQ2lw9vJRAvh4Pgb1YhCQ+Oee1vTpbJb7kjYc+gftPvnbe0z1kuLi+7Zm0PfKSoL0wZwa9GL0NvpvYib3W3AC9+d+Uvag7IT7lE2E+mpcEvj+ygL5PwDU+fZvrvfShA74uaIQ+kZVIvBTvJL7I7Ak+ea6RPq/tA700Lre9/yvmPVtwnT69J2A9CssLPrV8ubz9tqu8qXI5PbrFor7v0DM++cMSvMG9oD15qIA9EG7kPZZsi75zzuy+A98uPop2CD2xBLY+Nxb6vWRR9r01Gim9boe4PlbGkj3I+6W8JJJuPTc9Or66/F09juvYuimSrTssz9e9ZViAPkWApjzLEja9wwW2Pc6tMrzEaLK9rPVMPPx7iL1CaRw+GyKVPVT5/D2POBo+Y44SvrZtuL6BOb09qilbvtlulT7pt0U9iJH5PXi6RLsWOPY8IwxjPpbAdL0NE5m+SaLxvdpog75h0bE+s+wdProNtD2cDO69Yw8IPQnBTLzlJx++dK2+OmqRmj0hWg69Fv2hvd5LfD1Dgq28PpADvbOVkj1Wd9a9P1+FvP2+DT45JLA8g2gZvqU4CD3vGL89qLMbPmgMEb7A6LS+ejbqPfFMPD41RRa9EgOMPjUjtLzCwEI8YiiJPodyTT0x7Ak8oDY8Pke1DrmcO9M8s0riveDQGT0wkk++h7HkvdLoCj7bdOo9qGOovW+ayrxvUDg9466ZvElodD2O1IK9vGBcvvq9nD27xfg80w3xPeQxD76IIJK+","ZrmQvX01Q72LJGY9pLY1vvONQL3NJQA+3zlwPo6Tlr608Ie+PbF7PhCKJr57Oeo9wB8OvszLQ76bbKs9IEmIvbPfXryeCOA8Jd37Pd6oYz7ZHqe9HXUwPscfaz78ifs9OH35OzIXcj6a0x6+ieEKPjPoBD6TpMi+/JAgvm9IML61Mta9M/NQPKh1Qr7g8CO+AE4Xvt4Y4b1H6589kMYavm2vsLzAW809acxfvtcVyL3K04i8zgEqvo7d0L36/Jk+G8eIPq/ugb41XKk8gXi5PTLW1DwWECg+/eN8vYT5vD1wmjY96F0OPtKtH71GmaK+Ls3GvS1aoDq2kBi+uQqBPQdRur1xb7y9UM/ovOVTmL6pLUe9kMIXviPWM732BwK+wHfRvUjMA75kLSG+DXQ6vfhAk7691dC90D7Vvri90L3rrYC9IGMDvu4xZL6wmRM9YAoRvvl/Ib7C6Q++wNWOPV5ZG71eogm+q4amvQ/WIr6gUk++8sHCvf7+ir7Zpjy8812UvcOk0r1xvSi+cMoFvigzgb4al9C9cLQ0vndEkL0r1vu8vstcvv1o4r4gakO+BirdvU+5RT1WFdc8Y60rPdCPLT2pCRW96J8PvrsMDTzNotq86Kv/vJ3rWL7SSsw9qvhPvfL0sz3t+1++2AS3vZ5xfb4/RoK9Io2GvsAxEj5m4jC+Sv6BvFAyB75z7mO+c2wMvEHAFL0jUty9cCmJvWGhrr0tyaa9INlFveBm4L3h0EC9MAgDvl8A+r4W6eG91Z+3PASkcr5GJd899GnivSHYl721yYq9/6AGvihxI76Xntq9b19KvaMHir7so+U7ZjsJvEFFGr3WQY29P3elvWuIgrxH2La+Q3wRvhLHAb0JLA+9GWQivVuLBL7Esny9/Mv3vQ44Fr4ugIc+XNpCvhoqqL3wP4o9a6EYPNR/g76lHPK9NUqyvQ81G76E+M29JCzvvKRQ573fkjw87m8ovtXEwr1i5XG92WTPu0I5Fj2woeS8Q7NwvlywL77FWjS9","NI9gO8dbkD19IjM9YU2CvDqVz737P4495eRyvdHfdD6Occq9WWXAPQBVB70SRn49a9yFPc4Crj0jU0S9ofRzPY6xOj7qkAe+SMvOvZZ9G7xCl1q+djxUvkVgzL1HcU09ZtKuPZSGLD55HRS9E3lIuuk4gTvTY/i7oMWjvab0VbyQa4q9ShgEPTjbb7tDZSY9M6czvVVrij0V5Bw7ueOYPQQihz3n0QM8kuxXvP6Wcb1psBm9q2phvcB3yj0jepU84O4yPREbjT0uM888+tblvNR1G72qXzS9bBnZPcrlq7xH9ey7KHC+vRGuhrzbu868f/B2vaBqH7t4r1m+D0SJvWFnJr753uC9ILpfvk/MHr7StOS8EhqYvR/Ma7yp9MI8LY63vfXUDD7Zndu8SShlvh/oVL6GJRa+eB6+O+iHUbyFPmG+yL/JvWvi1LxxkIK+DdvPPeTPUD3/xue9coBGvv3CxD1Tqb6+dhMYPRb7u72TGDm+TcICvkw9njyEPr+9gxf+vslQEL4V8Y29+XFUO76qD77ccqC9KFffPWUFkr2B2w++q6ROvUmbQz3VpYm+LLTgvUtK4ryXFRk8Ks2GvsJr0bzhWmG9kXwdPSGZnL6T3wS+UXE2Pf0OsT3rQce+GxUkvsg35DtfGDa92lJOPAyCnbw7vSG+axNSPUi6Fb6jXYw8Zc0LPjwIvj3e0qI+uw4OPVM/k739jRw+QA0rPtFNzz3fFyI+YcjmPVQLtrxhWfM9XxW4PC6lzD5a8xs+CoD9PViwSz6SEyI+CGNFvbfCpz2zSO28sNv7PUKJK72i3pm885uuPSFwoz1r+ss9OLC8vHS0jj1bKPQ91g21PcpuH71Ey8E9yAg1vUU5WD4VYZE+OIJlPUXXGD7NFvw9XnkhPud+nz3B7h8+14iuPtb1+T3dgQ4+KRKrPNNVjb1FNUg9LuZ6Pc0y4T1Dvym9w9BxPNmt8j3s4XE+VRnRPIfgrbylqAM+LckdPkwjOD4fS08+6CTRPeNvRz3KWjq+","CDy1PKYZvryL4EI9tT5EPjuu0Dtlczk9vRbdPRxqKbw9EQI+mn5mPB6zpD3zLPE9OZ74PfL6lzzeNDM+H0VivaB/vj1kDYY+Gnj0OpsGSz4E3Gq8FVxyPEVzRT2yVxA+CXn6vAwieDxJOgM8M0UnPeKDvD2M+QG+trKhPeN+3D2k6+O9SIIVPxA6Gj4wZFi8WKloPYDZ4T0u1+s9WXuAvGUtJDyr4NU9lL6pPVUTJj21cGg8OE1MPBtNIT0XCWC+vO8tPjjLXD7UUS09kqfevAZTKjvjxUg9yDiwPReSuz5uYCg+hNJ3vFd8vD1Wh5g9iM6vPVUlwD2GWLA98vODvbE+zT31XNu9PacVPcL+Ib0zpVC8MAwyvfrg3rxxXY89+egtPL1qDz2abwW9xNCkPOiowb25uMu8VGfsPV5IFj37IVU99jwMPZfbkrp6jda8j8akPhxVXL1qJF89gGRpPIo4uz1RLow8RgFoPLoymz2qETa9grtGPTFGDD7Twb29yDIGvUdo9r2uxAQ+G1AVvU4pmT35dsY7tZgovVNWsb3DqQU84q1HvKqcAr1bQqk941c3PY4PeTw3qt+9SK3yvZrb/7pqIIy8zCmDvRjUFD1advO9gUgLPsWcpjwiGuU7KPWiPM2SWj0w0gm8PooqPmB8nLxtz2q9H99IPPA9VD2Xaho9QB1qOwQvHD0wX+g8NQsgvuucCj7KTga+y9OzPLhKhTxfFTA9LtvGPC67dj4YUJ49G7ciPS+kGz4lMns9phVzPkOqor2CIIq9ObYpPiNaEb3pAjI9cTUDvIoyUz6Svco9a5bMPtOT6zwWtP89uyjoPbfgcbwCghA+aBNKPgdChj5KbJc9KUzyPBMFM71RJII9avOYPWLaCb0uauC8bSHnPa6sCD4F7X09rnoYPllVLj5Te6Q9nOSUvUHdkT6kx346WUGOO4loWry9pQw+ri12PaGx1714gz69hE9WPlmLaj0qJak9GBDHvbgqET4W21q7d35NPpqC6Ly+e466","k0FevYtBAr2Wiye92VsxPisly7w7KQY+UV/Luf6HAj1zrW+9Thlvvm/cT74kOw6+2M+5vTxipT2+7Mw9RhjZvXlFzL7S32Y8/u31vI1ZKz5qFga9C10kOs67j7xj/fU9cigevqScNr2AZcG+2L09ve8vir784ri9DOuBvmGTJ77BIjm+eqsvvqEVYL5w4D6+TTvwu3umJDzZbKS9idsavOrbg75CBs29lkrovnYTqr21n4I9EuNIPZXklr4xd8G9OxlvPsahA71UI9W99nFoPcKOrL3j91+9YM0MPYSIor2GVV28u+6wvWC7Qb49gNg8/Ck4vs+1P75JbCq+4fWXvfhs/L3Iybi8rVk0vXUfi77/U7g7p5+0vcvzrbwRYdY9qkd5vfUqEr4Yzny+yZoavoL6MD2alSM7X6SnvhDrTL09dJC9Iy7wvXvBHb11HN09XYNhPJnZhLsN+2a+9/khvbar7L1Maj6+mnCmvYktBDs2UQg+3GTvPdkYAD3LeZg9AxXUvQrA1L4KOr69QNoavc0zgL6DpKe9N1JyvqNGGLwIfnO9tbAYvMutyz2eftm8R0T/vK2Y9Dq0lNG9tN6UvkgC6L6kqrg9CZI4vQcdf72GKlA7wD3Cvf7xdj21lze+H9q6vNajrb1xPtO9qMWQvMt9w72n0CS9UK1GvmPUz71wtkA8D1mkO9quET1gnnG9g9BLvOeErz007xy+OXEhvdePs7xaUAY9sRRcvDTcD7wM/549GoJ6PaIflr4cPQC8devQPPZ5E77B2hg9F/FgveR5Kr0a+k6+01jEvekC6zwgz0E9le+hPU7CJb5DwRW+nLiDPT+gYb3yCHk9cQ6HvSwV17wtvUg+/1xCvXYs6LxijyW9yGXZPKhUOD3AhIk9CpxOvXT0qL3d/8o7bAFAvY6UWb1dqjg9T+6fO5Hq6rx9fTs9X/Ybvp1Xjz2BhmW8v/M8PrG5QDzvwQI+nni/vWYlpb22q8Y94lLrPfDrnDzSxxw8hBoPPg8Fgz1W0qS8","Vkz1vbuA2TyJM1O9hlI4vh5MsbsBM2I9REyGPG5xcr2aEr69MDCjvRpvoryM8vy9F4RxvkSH9b1n2/U8fVSVPH//h72Uw048VGMZvqRsvb3O0zA9meGtPHdXGztp8zK+cK/avfe4Zb5hPEI+4An3vFq457zy9ti88t/xvWu9hDtUiVy+alpxvIJ12L1tzBS+jF7ivd16KL6uc5S922smvplhc74CKdc8uGGHvvu8672y2V++dDDCvZHRg71yMfq+3M8xvkRJLT2RwwG9vkt5vhXRKLqtsAM9JHjvu7MasL4fnps7vGWRPQ4SwT02OrG9aHfmvK40NL6ODHI+HgI3vvRF/rxDqUs85uCGvtBEur1n0y09rqfOPQHdNz20YKw8yfh1PazrXD6VoXM+f+GEPjEHHD7/j5E8iLEKPwU+TL0cOSi+zDUYPh06pT2aQJS+HtcpvpeEuLyubgk+IVA9vLI40j4wt+U9icF6vj45cb5x+hw+7jWjPk2ggT6/Ifk+/dIGvtK4gL5Ytgq9fal4u+PPsjy6g2o+YUo5Pmwijr2QqeW8NwNavlZ8Tj6laE895/9NOyxFrr7kQcS8GouxvHgD1T3KEaU+PSm6vmqRJD6mZhC+J8bPPYuvoL0MYOG9U7C+PgP/jj1Ah8E9mL52OseHIT7vwuO9of0UPgmVID3Qq7o+eMbRvQhScT6zz+A9NsmwPsMPAT4vvfo9S3lrPpyqWz5x/Ua9LxUvPrvLCT3yNnY9JhmSPA/rtT4pp9o+VkKHPtVasLzfeN69YRUeP7/Z3L0T2eu8FupxPYrRyD2Amzc+RiGMPu7Kn75d7xq+uKUxPv5nNz1yVpg9pYU7vjooF779HxU+VwyfPVtxEb55q5O+Gr3UPuBHJD7NANY80HojPlFGgb0sQdG+VnsGvTC9Tj7bZHY+ZuM0Pbl6UT7oRo29s2AhvkXA370F+rm+E2BXPb2+OTzsKfg9mnOFPXikaj7eGCI+cj7nvKVO1ry5Sxw/+n/xu6Mk3T2olxE9","0wTPvuHkvT2maqo+EQ6sPUFgKj5nLSe+pkWqPUSvAb9m42G9SzjHvSZ7uD6wdQm/UVtAPtA8Er6OnVC+7WgrvzQhqT5x/Bq+sqvFvsO+rT3NZVS9MyUtPJo5rbybRva7NC71PTmT6j2XBKW+JsIFvV8YAj6UagW+opGdvS/8tT20LSw+smoHvjSKOj0TzqQ9doV8vjt99T1jTpy+Nq4/vdfFgbvvqMG9T95Vvr69D76A2QO+7p1aPlZ6Gr5F+RQ+Cpy+PXi6gz5bbb89VwlHvtRJcbqeP24+NDMXvu2FHD94eBE/MrotPRT/nj1R1Ri+bLnnPsqZVj3W2xw99yERvgxGBr6Yboq92z3aPd8hkz5lrnA+Q05/PYEvKj2HG+o+7E6SPqYYLj6TrnG+DVYKPs6i3T5sj0I960WJulzduT7PmPs+jmKFvILPWD59d3o+ufOJvjU4176kk46+DjAxPC1JIL7vYK0+3Z89vutsnb3o2+a9muwGPlPU9D4SP/k+26/NPssFK7vwaoc+mBI7vl5KrbwF/DQ/zG1bPq+mQL4Ntbw+TaIWvY0NvT0MApM+LxI5PXihzj1ziB8/K/3rPXeqZb3kWgK9gzbcvii8KD3/mOa9aITBvlDtaj2eN9o9sCpZvZQ2wD6kZkM+ipqzPpyo/T2Z8pQ9r04EvvxWybufTYM9d473uzX6SL51kSI+jGw3PoYnGj50iqO8IvDGPTQmQz5QDKQ9pVzEPTEfE7wf55Y9FTUmPVWKVj7LbN69x9wVPjCPZT1CRUQ+cz1svZeLWzsvOYk93WrWvCwpSr66mlk+SuvLPXLyiz2kYKM8r+WEPj56Lj4zDN8+CH9jPsBkDz2OxHU9DeQqvvPpoj04boE+pEsUPtnVyz3dh3Q9W7sEPvVHKDxmk04+/8r3PTaJ/T3xiaw+VB2rPF9ECz2A0N07KEuCuzXSGT4F3x6+NdDJPch4tT0wRCG+DJVkPtcjhD6RTZA9f1oTPb0Vnj61VKm9U8erPcZ4Jz5Pw347","MWMpPuN+5DyCvLo8r19lvqgeWT6tCGs+X90oPR0vtLx5npQ+tB6FvZyu3z1H5G09ciyCPc9mwrx4YdE+9sE4PuLzOj7/sNo9yXpgvhefdDwWYbe9mTJNvTEHEj5zQNw9PnHsvEXfYT7oOmO9WhwSvooAYL37hIq8+P/rvIWfOL7UB6s9v/PtPjCrQb5hef656UgrvmYCDD4bYbi9O1qTPWfmXz2jBtM9hvB8PqsCh736GRu+FwAovq2Xtb2rIWI+BLoHPpxoXj4PeDE+XrtbPqI/Mr1d0Ck+gI4UPitZBzxdfew8/YODvFDUY7wjcMY8NYWmPu5IZD56b5w+TRg+PmDWMb6FmQK+JpUFvuQfxDydrXa9+6j9vJ21CL0Asim952o5voH6Zrwm5KU9qC9bPaCkVD3098y9uAtHvapbU73xYRm+A42nvXipwL3vgfk8trMyPY11WD5q6Hg9TPcZvtbUiT3xRh29TOIYPn+qij2Tvii+hja3vMJH170LNxs+GtgzPqxsf734tdm9q+KQPBj6GL6DeP+83JWLPK10rz2Pgl+90Ou8O4YOPLv9+9U9fNaBPKpce72UPhI+h7tGvbaJvD3r9PQ9RccTPJaVXzy/so09YgC9vaJPL75wq7Q8AnaOPhTdo73Bz3u9LIjWPbowcr0szpa+JLCQPla5tD22o80+MnMHPWP+Fz5wfd89vgyGPRQIQL3lJuI9WQN7PnXWsT4arUU+i19oPFTi4z2AG6E97gGoPMW4EL1D8pA93ZW1PvAURLskwkM+1gVQPnibyD314gS+JXqSvbJUHr1nCaY7UUqCPraIEz5w7Sg+01ptvY9jpj6+bK4+liefPsSVFD7MG8c7IghWu71ZFT0GN8S6TNWtPnCE+b0PLeE9MDEBPhugeL4Wrvg8vyYivVH2Aj62xqM8EROgPsoNBT8zMvM9MkKlPfrZmT3xWv4+6l6GPdq7DL4/fOE8qjooPBjz7bw7BOw8nTwNPr4Ktz5heEK+Wb3VPprckD09cjC6","FuumvVgHJz4EDL08cKQRPovfAz7TxvU9mg4MPQog6TwaMTo87Q5HPqtZpT0kxZy9R7SKPaOXQT0sW1c9zvdFPrkxBD7lj1E+flxTPqK9AT0Kt8Y8BZUyPkEQ9D0d+dO9mrMkvRb0gT4beGi9uyfOPd0/nj1GWBU+1XJJPqUcgj4seTY9tpt1vVgZDz66SgQ+uRpGPRy9Qb0GsDQ+/jCIPZJ4Cj53xBA9ga81PknTED6Pr9o9DwDuPVRFJT5kTJe8+u0/PtP3iz37A4Y9XBXrvdUWrjxjtcg9q2MVPTZQ1r04fSI+tpjxPRxAjD4oVS0+kqTcPXtK5D0rvMo9yfPmPe5mAj7enRs9NHNYPpKjRz51Uhs+rE+yPTES5D1/DoI9Iq4UvEmzMz5mvKI9r6JIPml9ITsNaeg90DLWPtEf6Tp1zVs7/SZHPun70r3t3WY9GJ+EvDoQyz3+kHM8i60EPv1Xaz0pUBM+Nh4NPpU6vD1T+/o9I9yUOyDUIDxG0RA+xOU4PZUNoj4npAI+GDqePGVILz7ZnxE9Oq0GPrmwGLvr0Zs9A8vBPYVkrLsAjLC9O3i4PTaujL22Gk49wql3Pru4dbwRPiE+1uYvPf/slD138Wq9Dz/hPcJ6bjpmTcI+Zrw6PuBZQDwSeMQ8cK6KvACrK72s5VO8Gx6hPFJHyT3o1ww+jXazPObo6D1cGei82ySgPQ+bu703rqc95bomPTCZGT2uyQM8ilsrPmWiPrtlBsI71dzOvWGGBD4aBRi9ECsePkMNzj3nWCw+/LSSvUaAJD5tJV4+G7qrvacMuLzad+G8y8KDO91pQrxpHVU9VUKYPIeRJT7217c8l9u7vRBZE71XYPY8a9A5PkxrQL3hTUG8+kYfPZJxKL2MSNq9KtCRvX6+pL0DLi29wD4XPODZ/DwSCbe9YVBHPd5pcjnJomw9ohqDPDKDdr5Wiac8fLOvvN/kYz3drP29eEipPMZ4pDqLM4+9VEZBvShgX7yOc4k9hZCjvdOc4bx/rUu9","Yd0KPnHIRD1ebJc7TLAePpdh2r3jUce9MCDxvDIerTwngq89/JanvDJzqrwpo8c+pge9PcQ3hb3c63K8nPMnPXn4zj0CsbG93oaXPeqskz73tJY9ndbMPBvU2rx79VQ9lf2GvN0fvT4uWkm9sCtlvSVUHjyN7k49asIaPpH1OT4f2o4+fdsAPjycVT3va4Q8shW5PSAiyD1GTIU9MEY2vahVKT4H7Fc9Hyt6PWAoCz2WIes94id4PdI02LzBG2g+hIXcPKfCG76/Tni95AO7PmjrpbzYQA68lmXVPIYAmD5hXw6+AGygPTehuLyUBwk+qMVVPs3r2j2TWIY7/Bo1PvJBUr6I96C9VwMLPsRhH7weX3u9bXEQvGuYRb1sAiS+KMlQPuwK8b0PgAO+TGz0vEY1Or4k9Lq9e/9NvrrsAL4XDhG9gD0CvqSfY7y9fkM975a9PGX2L73yLdK9UzKkPUeZ+L3gTOu9Cr2Fvd7pG72yHbu9gDn+vbqf0766qya+rtcTvkxF3jydMgi+KULDvVnL1L1cyfS905i6vduzjj2MDV2+cSsWvqGXjL4HJPw8doolPdjTIz69cSW+vLyjvsXlZz1swIG+1CQsvqnrpz2UtXQ9yAYtvpSLmL3+Hxe9L0O9vc1xFb5epyK+tn7xPFFOV77aHLW8RFsKvrGNbr7hbUc9O+ftvSiUML7sXRq8VMsgPUUe9726i829ubDfvBfA2zyFeLg9pQ7EvdbbU72r3vC9He/gvWNvCr9iIV29PdfavQdYgL72rJO9V8POvPVObr0MIge+iUlivh98AL43Y1u+8r+bvdLYVb7yk6O9Eg/IvUl1Sz3VahU9rQeKvYrI9bx5WAS/1huhvQzl9rwmhgy9TgyQvQqeRb76Dgo9KmonPNDyZ76Olx++EiSdPSyXQb74VKy97gAnvdgUbL7ZhAi+BtxVvrfQWLwDhrG802hRvsSZJz1s//S8NF5QvoWltL3X9Ua+wj2EO8aZgz2MhSy+//CQvWZ2m70fg8y9","HtUzvlpEAj68wSs8FDH4PMN1hL1qGFM84qMavmmRxTvMl+264ugqvJ6ApDtYl+29kUP4vSG3MTw7n/W9DmCOu57ko7xbzuG9sL4YviQNXr08PIG8G+9evpmDtr0Hzlm8ErWfu9txXzyyTA28Na6evUiriLzYarW9JSyxvVExMTyrAg8976pivWoWvr3k15G92zZovSHHhr1bMgo9GQaMPZHUqj3cqHC9yJuvvMhTzr0C8KI9/MO0vMDZgT2YhTe9r2uCveBGUb1iAyQ9vahyPYQohz0Bpsi9Z5vyPTQ8fjxbvpm9EdDWvCQ9KD1bh0E83yJmPc9cqTvFKES+VPZtvJ8s3b1nw488zoy+vSD8Gb4Sjpc8vz8KvVRxfDsXloG8ejuxPGHBBL69qec8cfZhvrqL8r2xk5Y9xsAGvPMpY73Vtiq++clJPdZsm73Brka+vqMOvWd7xL2eP4u9ZnJLvixSU7ybARa+alZovEyEUT1wf0a99nWpvEQtU735PrC9G0tmvpnnxj1aY8u93UdwvcjSKL4lSqY99TCbvao4E701WOq9ENouvr283b11kRe99St3vaBlHL1F4oa91/XbvhA4Jj2QanI9mtMfvQDTjr6Fs5a9+qnBPNCBzT0FSjq+2ZLNvN++Er3c8S88IN6UuogWkb71Bak8SKDAPfrxVL5Fc4c9UfcIPhTiELwUQYs+SIk1PiAPWT6JU6093V8RvhVkrb0nlbE9gRzyPcU45Tws7Us8iHpGPXw9lj4Q17w7hQzTPQUByD1V0yA/Lam0PRs1fD2GyPU9A6hsPsTIp70pIuo9dNcKPlfJcT4coMk9s9W3PG0rjD3F7Zi9+WuNPq5ZTD72/N66kYnIPcTHhD66ODU+RLAnPtfygD2f3Q0+PyXdPSWB/bzQ6mM+s97APUIm771qcBc/Zh1Rvoht+b1OMAU9zsdfPknqjD3PhCo9rb4EPTsCFrwiBso9KkoCvkg4gr62H7+9SSznPawBnD4xfAk+Bhx0Pt7FgT5/oM69","1kMNPjnBxzwXs+g9FeikPggjvT0hYk498LeWPcpQY71Lkwc+tBiYPeUluzzMaIQ+C9qyPaixI77Sv5Q+QMM2PtNnsr3gGXg+4GSMvH+7Wz1ykZ28PAGGPQBQxj0Maqe8nvRuvJ21mj7j8VY+3bDjPG8nC73FTKG9lcZlvuj/LD69ItA7kcfiPnxmij2llCC9kRm2PQ1jqj2W+LI9JPu3uqOw7r0izB8+ZwaWvaSzY70D1ri8BiQavnRHpL7VuYE+Zoa4PftZYT5sr7Q9hV/tPaGqvzyKtkq9IzN8Pe456T60fYa/1TgzvjRnAz6aj+Q7/9DZPSxeMj7Dmqo+4SpUPYWzdj1LoXS9mAcbPeuX9T2xLJo8DXbdvHMhiz1qFUy+OySbvVzrtjwJxeo96qMRvJCIHT3STCC+5lSJPYosBL0L2vo9fWXFPd6/m736Y7K8hrx4PiDfkbwnApy9eyNWvsANDD55ZCE8e0W/vQ692Dx47bE9DPjTvYb8T72qT0O8OxLhPctGkL2i2Zo91mZrPCa+Yb3Oiwq+ulUWvbR/hry462O9x9L2vWLH8Ty4FYw+UxPxva8bT711O189C8rqvfqc+j0EyQ48Mwdevj8QQ721cw++YelFPvzBLTy9+2M9SXGsvCAnpz4cY+O9lChmPgs00L2AwUi6rqEcPrikDD4tZIw9G67HPNFNGz7vY4I+KOe/veZO/TzZudg74a45vRV3yDw6zZq+DJ+2PUkkdj4cnKQ+FRviPVsN2z13dEk+5Is2PTBTDL26GgE+8sNlPhYNH71jDV490DiavOr8hj4moSa+QD2fPt+F8D0QH2w+Y89dPbncIj6zzoy9xOdAPZRWIL5SzH4+pYvTPT86uz2HRdk9/1OJPawjQDyEWSk9FsbPvPUlTj3aEko+2ZWMPlnHeLzuCFM8vY4avs7NyT36tzc+jsx5PWS+Kj0sIRg++yi7PW8r5bt/wOi9jwnmPqH6vD3GzGY+i6vOOMHJjj1YpGY+aP50PnhahT1fn4e+","sEx9vQu23T3Kmyc9Z7jTvqJNFj1jGRG+hcMWvo7z7Tz5clc9gYYSvk/ke7w32cU9hP6xvdj3vL1uX8U6naP8PY2/tT6BV+G858S4vfaxSj5yLqE9Tq1Dvv0TEL7LgBM+kuYQPg28nD6Z/CQ9sG/kvcMWlr1ejB2/h/YPvzXEz72OH0q+Hjd8vc0lo7yai0q9ju8Kvtw/2DzsaJA84I3Pvs8o2L0qTPs9P8MCvrMV+72284u+N+eIPaWsnTtcp3G9UiCivE76QD1M2hi+CUSzPmoN5b6oeBS+XsAdvqgFHL7voYE+vYKGvTPDHL0lasa+i0CaPe0+uz08NQ6+NogOvA3ZBT486+m9JiIVPY2kxL5OCYm++2gPPg3W3T0lrko+0ZbLviMl4D1MjHS92A9pPTVwUD4TfiE9CsCqvTzIFj1inwA+DM6vvMhx6r6703M+d/nqPdktU75znmq8YJZrPZyvcz7JeM+9Fot5ve9PPL7SkkU9tymtPlYMyr7qrWO+GK1TPjzo/b5WIbu9I34ivXq12r7gVBc+M+guvg4stT3ctBQ8UBrYuyNxPTtDRwe9SB6xPXaZXDz5/BC+VycxvS4+oztQFtE7w5VLvnObxb22ERi95ByAve/o3jxoMhS+ED4xPtBIpL0ZL2E+UQMGPgN1N76V7K495f0mvblJbD5r+uc8gNC5PguyGz7ybGy+aiD6PfIXXb6cN3O+aYZBPalaR75yo669xZz7vfCRGb5TKlQ+EY3GvSRVub3XBhM+vZG+vTq0Yr5EPhg9AD28Pel81r1URlo9IX0jPhk8Lr5bS48+iFxqvvhvqb1nrFs9EU+6uoINWL1muQU9hb2SvJTrorw3yUS5MS1TvZRTwT1p+iS9/FCsPtsE3LwGZlC8m5C0vVO3Ej2ryvU7GkTMvdsWDz6dx48+c9/GveJmDj2Lwcy994byu99PAj6tgjC+uY+VPE38X75cN2c+H8c1Pc+HQjs27gG9MEgjPai2Bb3vgIw9ZTk1vWjocb7OHeu9","q5JSvEz2Lr5VaqO8q2/YvSI3Xz2jxWu+ueQJPiVhN76/Mom+k06+Prc3074wp3W+RWE/vn2dkr2g8By+mymcPKSApb7lXXK9NDpQvhqegb0BYYu9vuaMvR+SJr4N4pO8gQjHvTPw976ZRZk9m4dZvZm3FL0qRIa+xr/pvnpXFL7NH3I9UYZsvbEy9LyXvhs9Vo8ivjlvCb6qggq+NAn+vtPxLz1MGYC+tBu/vhpez75w90y9AVcUvuH+7L1tVHO+uXesvTRbiz7u+8I8tgfQvqFv0L0G3sS7mKJDvgwytL1VCq29zqGEvovzLD6ehQS+hAatva/U5b2HgMo9oiiZvszlATxaxqq9ETARvvia8D3xHci8LhgFPlwXhb0CI4++pKSJvcwJcb4tugc9pm/XO3R5sr3vDwQ9WVTxvfMADLx0XKu9cwJ7vdO43b14X4+8QaR4Pl5hs70WiI+9MV7DPX8SubxMp8s9RbEnvjI0iz3MNIU9Gzx4POlBZb7XBL493OgvvtpDXDxEBa69YQGaPSAue76dVO89CFYpvjqOlb4Ops89Bsc4PhNTZz6z9pe+FEjmvT59vb4XA3y7cEFXPuSMhr2g0WO8v3FwvfNdZz3O4xc9U5RLPGTPmTyOeIg9Mg21vDdVTT72ZyG9dYwdPm5qgLtvnxe9XqpgvSmgl7xnTnE9/ZMBPnroIz0tXSu+Zq6yvG83EL5TkAu+OucJPYPgAL5m5FM+QQjBvjB8jL6wfa+9hNGHPolgTD5wJLK9jXunvhov2r2uaXG+ptA9vcYOET1ZMIe9uOE4vmmHIr0HaCO+cHS0PIW+8b1y2QK9mMYyvqAR0b3Em3+9Rulnvu+0OT656JQ+ANTkPf8rwz3JFA++LTnlPVDukr0FsbQ7bY+uvECTC70qHRy+kq89vqq1AD5qKwm+ROtDOnvcF75liEm9OOCGvheOBT5j1Z29xXKBvE03NTzjkpm8tC+WvvG0AL44nbI+vyURPmrvsr2lEW6+2uAavsR8uryoZ6S+","6XinvL8hmD3haKe9hSUJPS6NOT7etgS9S+YbvQ2LFb60cqM9CFxsvJePAr3jtoG9IXh5PUlMw73uZZi9aTCcO4rQYD2eis68tVoKPhufCT79yW49irzMPTAp4TzFhTy9+lk2vuZ9uT0kgRy9HxaZPBjVeLwy7dS8BU0WvFmOFz7tVA69D9e3PRoq6L2HV0Y8tht7vUwaPb2osHE+yEZgPrvKED0qG4Y9MyEEPoPdvL1HQkm9YCl2vsQgQ73ZOZI9y0Q9PbGh4LwIliy+7Y8KvusTCbyLqkm+CeTzPV32YzzshrY9iDcJPnVzgT0htMM9wkKlvSqpNr3jwvg8Ys+avUOQITxDQe68O/Thvcat8bnc3Fg7VIf4vQdnDrxOOAc9vTSiPF9fcL5tcYo+slqTvmIoJL6/Hcc9AgFMPNMnyb0stTK+s6TzvMqd3r3Y90W+90Txu71+Hz5UUBU+u4sXvkzTHT5ZMgW+lxxvPKRI9j1Lr0C9qMUqvRAD3jwNmB++XoPRvX/yrz0yAt88BDe8PbvXPL7Ikj29sSR/PfUGpLx6uAc+lCcbvjktOD4jhWq9iHcGvjt3BD0xkGC82OMrvHd/+z3k2tI9omKgvUo01L0CQjO9YTIiPdSGzjywg+i+GaWRPFC177zgiki9QtmMPGQXmj0xrAU+EAMdPjdSdL3qt4w8t4H2vIfmmL13HtU6sfYuPjEAST6gHR495yuUPSwFMb3Rrm8+6ZMrPi+GLjxW43Q+YVmWPLXOtT5hnmg+JDegvWl/FD4omjs9OgqwvGTsRj3J3jw+fWPlPMCPlb2AuSI+2Zx+PVa/172lZaI9HH1MPk0TEj5TiTA+uiW7PFAbNj5QV1S9HkPrPAT9Mz0bTow+CytoPQDrQj7qvUG9q51QPhnHb7wKMYs+3wgQPteMKz5Z5LA8s/uoPknWPD6MBrU9XtD9PSVS6z1lTqC8MXCJPTVcJj0G9nc94OCRPUxntz29sxU+L8oDPr6JHz7nr/M8Z8TWPXB2BD3715Y9","lhYlPoD4MT7CNb49q7rQPQWVPz7yVBQ+uqLKPZ7AEj7ZsBA9r1SePSMfqz2KgFM9JccUPk3MFj0l4vc+iqCdPBhcOj6zHmo+YMkHPXZmcz3nri0+eFQePkQsLT0QNbA9yrxWPhDlyD0+ahA+yxdsPf5a7LwTGQq+5/xHPTj0Bb5dIBk+WE/vPntBFz4IZrC9TFjoPFRJnzy4X5g9uZFovX8K3j2NPIw98iaaPRYMfzyEwqk9JUHuPfdYmTcGbYk+PWSoPh+9FT4BhSA73rwFPmNl+D25XTk9SmGDO9DoLj6lm+k92sfNPZHVAz7fk5M98vvUPe+fXDxqT1c9o++aPTvgJbze/qy8vsmJPCNtuT2CDuk7gFD3vWvXaLzcBRE9Lf3qPXoTBL4WPvY90HcNPrWJqD2XmYC8D5WBPTGq0bwzwOK8PFBMPScgDb0Bnnw9xKXoPV1ETD7U9p49QSMGPnngWD7Gz/q8JgS/PCb/2T00LA6+OWOIvdkvyD1oRqA73P1qPEKCEr0SYGg+7ID8vPwTD7yHhDg9/+YmvhmyOjwVuc69/p02vt9J3jztLUM8oJmqvJcH970ZsBe9DBkNvtXNY73Pcai9eHuvPN4eWjtxh0i9EtjPvTwHcjz6/N68CwMyPdfwo707/Yu9JSSsPBVcjD1kbsW8CIAIvAsXtr1PvL89CzmJPZUqBD7v9qo9+shAPRTmqz2HjFk9CpQmvBYa1D1DcRs+9bRaPGEWeT4Mwgo+/tpGvf3OjbyMx5A9QwWGPt9beLtITOs9KH4HPh66Jr2krhW+zdYBvTcgVT48Oaq9c6vOPokbY7zPZD29ZOH7PVoZKD7veSo+xT4bPqhFuj4GL4Q6iTc3vXAgET3/9lo+/7W3PcmxxD2nbDg+P1mNParyFz6u2xg+sNgjPobB5z1GUCY+j2i9u6YH8z5++p29jpSGvUs1Bb3705A+m8AVPY9NjL0DI4G9eUAyPnEzC71eDZU9nM3nvcPT8D1m2/U9DFAjvbiF4b0IGDc9","bI1VPV3GPr4BF5+92F8MP5hgx75gX8g+Ui6fPgfKkr5382G8LBPwPpdfZr5J9JK9SaelPJUiq7wgP0G+xUqCvrRfmT3K6uY96KBJPg++ID4m1w4+Sj9IvgCQJz4OKMS9H3gqvVlyo75fiD0+rMI/Prc52b7Ev6g+V6eEvo88lr5i5DY+tZjIPQJOl72jq6Y+sbYKPumQvj6ZYRK90lGgPkl4IT69orC9GigZvoHWez0vmRQ+LQaLvRQq5z3EHBG95lO/PnZco7yshlM819TYvXkkeT5t90U9513lvR94yT79jSK/Q3Vivp+QKTyZ3hS+SgVYu1aUnj6qA/w9lFLqvaQ4iLx9aTk+ynDDvGYzw70Xbo6+HRP2O6wdxj0WVy695MITvH7RVj44To89FgiCPTZ9Cz5gyYO+CQe6vmHN0z2u2KO9KnGEvPv9kT7ZOm2+YEtQPWx7g7zln1I+NZ1ePjBUxT0wTf49jIQhvm35bb0STq28/mRjPYc9Fj1kcJE+h8qxPJYahL5D9ne9rxa3PaHRCj7UCHw+/rDhvcbEKT27LEY79G4EPn6M8T0XzBM+8D0HPmMQQL5BuTO8KlXJvW0jVT0GM7I8LFkOveRvjj0DYuk9uX/dPYTrKrwDQwS8QfxzvpNMID4bLOW970WMPXtffL2/Mwm+peSTPs1EyT2tfeE93+SYPAWbhr2jM7Y8Yty9vQ2jLz10fT8+/2nMvM36dj57dbi9P6I9viW1IT1seb+7XsaFvSIEKz4rpe09aGW4vRzLKz4SYE+9VNcPPejRk737KoO+SIkbPRhTED7sEK06+HVDPnSyYT6O8NA9P/zdvUuc/r0Ajgs+lRb4PWQQY70WvSI+BsvwvRZ/aL2bpBQ9o3BoPcoS2ry3u3W9hs7cPEmAFr4NtgS+xT7dvTbZaj32K969BjxtPViG+Tyidga9MVSAvG6BY74NGS8+NFgrvUatkjzvGcK+yA+qvXOObb6ecSC+ERcGPQdSBD1mVXK9/vyXPggXhz7lTEM+","PXsmPSKQ4j0akZ47FZTgPC30qz3C3/k8EsEjPGxHhz6B++I9W8t6PZl0wz6xfc06unsVOdzKTz4URRq+gxrvvI7SCD5wQii9425PPjpsq7ziUIY9sc+qvdM8Tb3gP7Y+eTYxPV/xBL4VREm+OJOUviannb1zrvo9H1r8Pbb8hz7T0SK9JH3zu8T4Gr15fME9ilE7PEoFkT7Lfng9LhmsPjHXTj1Xjys+xhHLPYNsCb6eDHw+O5mDPmZ0Dz+kgJ+9eZuHvZgYIz47t/89OC+NvLDx9z10lk6+OctJvP+WVz6cMZO9tpWHvR/mrzx4Cog8uneRvWdxErskym49cysyPg=="],"bias":["Wc0+PWmxFD4tMXs+acFXPgA7AD43o5I9r+eSPRJzMz7kfLc8BdgpPhk9Cj7eUXA9y8bCPk9BrT1poIw+UoL5umYWmD4LyAA+DgUwPnwiuL2gUm49iCjzPbNcHz71/968+8S/PXRJ2z1joK49yZM2PQ2R/z14G7Y+XAmPPsFfGD679oI+s7N6PWwzYz07kPM9H4RoPi3uej0XZ8E9zjF3PN9pJj5pMKk9IJCrPrhauz08qMM9WBZ+vZ9BMD5Ntew9aJxqPWoajT0CLA4+EGuLO696YD0uico93bgFPiU5db4AIrM9epHzPQJUJz6kItE9NiXKPQFLzz0JKCw+G4KnvQMnhT9M9oY/kemNP9pEmj8n+oY/6W6GP72TkD/00ok/KJqDP/eQjD8ksIY/FvyMP5SOhj+p1IY/CxS0P5pPhj88hI8/bNGiP3MrgT+eoY4/oGmHP0qsiT8024k/gyePP0yIjD+2jYo/Ee2OP3Aliz/inYY/BY2FP+Ykhz9+EIU/tLaIP/9htz/XW4o/WJmAP/M7iT9cwYY/pkaJPzF0gj+YU4U/6J+PPxE5iD+75II/Qp6JPzshiD86dIU/tH6OP3k3jT/7BZc/AaSMP4lpiz9LbIU/qPmGP+DDhj+E6o4/52iPP7NciT9DdYk/x6GHP+R3hD8A8Yc/QSiIP/GqiT8AqO08mw6Svekkr7oqvbE8pNh1PaCTorwjrVU8NGF6vWujlDnoE4W9oQk7vY109zsfGzG8T++EvM80KL2kFXs9C7dPvSqeCb0MAMA8vx4PPEftaj6mGH89Z9F1PR1s87w2wKI94BjqvLoWSD04XdM8w7KsvPXFbT0V5X28IwVSPKT7gztPnai70RM8vizGcLznfjq8WJpku8NgWT328qy9Yg3kvIwh/bwu1Ys8vrVCPcK1Mb0tYhs9MRx/vQ2/MLyTQgk9MzQbPUzHJr6GLwW8OgYpvf2Ct7thXRw9ilo0vWAMdT0mTD87KTedvd70az1amsK8btyivEIbej2DANk8","K9oIPjP/Vj2u65U9ojjqPb6hH7vrzAg9ESVEO8+cyzxMuoI9CSd9PVciFjzsyqw+/qDtPc9v3zwi91Y7BxkfPfnGcz5om6u8OCazPUMZgD6yKWU8AdGKPF/yRzrZB0Q+hCYhvApigD7eXXS87Ge4PIKWhj3ScQ493bXlPVnVRD3ZqKw+SNG3Pc8jwT1l0NU7rYfPPbuUcT1NcRo9SQ/iPCh17z2d5yE+09IDPrVtAT7GKqo9kA6dPUfvQbxCb1M+rgavPTaL3jx7SP08kFJDPgsVED1AR4S8RwuLOmf7RT6E8tO7nFELPWoko7zsYEg9XmcsPlNVHT30eXa9J1+yPQ=="]},"lstm_1":{"weights":["/FNmPu3mqT7Go5w+PGScuzK54D2UjkU+uIcfPuEaUj7R9Kg+SlUDPgKgTT4wCMo5xqa0PUtEuTyysiM+Uc7XPYvxjDxXYxw+IfGePZK1mD4nETI96kkwPvs3Wz7Afv09CN8vPhYesz5erQ6+d/kZPtmMvj2OiOk9N8wyPQc0kT4Y4mc9M8lqPIpJgT6kyTs99t82Pr9zaT61s7E+I/BfvEz8aj5ypxY8Lt4bvUTU5D1hEi0+FWIGPUgq/Tzqj947rd5BPsFGxz0GUl8+LIKjPWBGMT0mc029+zQ2PoIgez6UKOi9EVKlPd84Tj04tgI+yMqkPvv8lLtKdDw9XbZBvVyXij4wFpY+R1TKPdMFzL0E2FM+POXrPQ9WC74RzR8+iZEAPm3LLj4gdW28kCPoveUreD5os/M9CQsGPnH6HL2Bsxs+Q+luPs/7Oj2Bpgo+3xfBPQ6ZHT7tQyY+v2dcPuMrkD5ry7o+yneQvavQn72CHvE975ioPT3nQT15ekc+b106vrb7x71QyAw+gZPJvbvyCj7mUAE+yvJEvEfJjz2uK4o+8JfdPXA3U73zeq09xlnDvbzlATt2o0A+3msDPizxNj7tUng9KmHKPleFbD31Ddw9jWStPYemvj0p2b89r5kDPqqzUj7qbD09rr/QvagCCT40eRo+ULtdO+ZzSj5vdUK6911TPqUdVD3orVE8kVbSvXY5sT2IfHe9YLqfvnWXj7zRsZC9uQEePDUXDT7ORZc9JAzPveR6hLum17M9wJCfvi/KBj1wKlW+mN6vvSe4wj3dhUa9209Ivo+4o71pAo49/176PQRleL2KVhQ9dasmvglOv7zWvSg8gFsAPsYXjD2h5dc9gYd7vPuKML5ogVS+ojb/vR+17L6dQGk9iixBvegzFj3Wl+o9vcLpuy7zZ7t/bEk86L4tPg0HgL5Izku+DTyvvSd4qL3XjIQ7oS2IPXuvgbyhD8M9RuOfPFSDDr5fZfi9qINdPoYk1j3MqV8+22+fvFyYPD1q6k++","rMGkvVkpVz4NAwc+kWoovDD69b2Tip+9NYEkPq2PLr4lTiA9n76gvSlDRztyQ5A8itInPriykz3nsBI+8wcFPvYWDr44/v45JTp6vB6rsLyxXIu+URK6Pb5JKj2m4B67LnGRPW96Dj0zX5y9UxfDPZUVYj5hGnc+FBMRPVQX/zvBvhY9uU1bPW6zLzwA6ly9dNvCPXvWS72bQzY+Sn+CvfPqMj7fcDa85fyZPSKV4T3QlPw90f+jPAS4oDxodeW9uCxWvDX+/73jHB2+D+3sPczyoT04kZS9m9FuvM1a5r1ikJI9Hf16vX1uJb3qxlq+fGqDvbGo5j0vbSC9ptfJuoHD5b0wnA8+r4hxO4P80Lz+9fC93hs1vj4b7b1OO529xzwPvtfTXb02eJ29M0alvD04Lb70J1++D4K1vZ08Fz3qeyS+COpPvfcHA74xh4C9UnrmPJu8TL6xgj+9tuIcO2klLb1ovSe+kHUmvkmlIL7Q1G2+ipwhvn2eIb3+nxa+Xq2GvmCZkL56pjK+mCoRvEP/271cVE++WRCjvn97y741qE6+3yM2PZgaCb7K9+Q81r6xPWj1lL36EVO++1CRvioGP762Owm+uS/HPddTRL71Z1++39IOvWPo3DwEgzK+g+AbvTP6or28wm++VKdBPQTcQb2tTVe+bmlYvoCFrrxwq0K9bJ/QvTgE2D37i0K9uKoNvgCssDyDLAK+r7zcvYRvXb42/uw8IZ9fPR/nR70QbwO9y7SRvM5ekL3BAYs9b5b4PKfRR746UkQ9QCRQvgBigT1ocUG+zM0Pve4NojxiwqK9UbgHPXDeB7409hg9t8V5vnmHFj5ewiu+xHVZOyLHLb1ABNy9uGgDvogMdLwfmeO9iprrvVW+CDyP86a+HToyvf2TW7xkBqq9+J0bPZtcKz1Uh/M82dCBvmdNTb6J7gM9HdzHPBgmdT0MT/O9wRjWvHQKNL1Vuek8srFUvQpOYLzPJPO96EJjvr2Xzb3HeM69beFqvixQSL0y8T+9","QxrFPa6YAj6WLna9m6M5O7UQyz5W+Vy9Vci3vVZuqj0Gi4m83o7vvbbgSz2CMba9hUAsvmhCMr1El6K9M2hSPJ5lZDwgWOy9NHASvUf8Pb3lGX+9MSD4vGvOtjwfjJi8prtrPXAskz1iiRY9OF4aPlxPLz0ei8o8TXtOPWF2czuJ3VC9Y8PkPPy3Cz3LsMM8zAvBOs2EnD4TpFY9BAJfPdlxJr5Mt0y92BfTPVWXhLy3Zhg+fQjEvH8/Qr5uzAg+3agUPsBC873OUxG+P4l3u5kb4b1665M9FfOHvsvmtb12pDw+KQMwPYcBO75p9ic9525NvT5UJD5Nj6W9kA6RvCvVWz4qILe9NKsZvk7Nt71IDqY9OA/APT4zKr3qCYU95Oi5PJcilr3GcwC+vdonveCMi7uxc1W9nPK3veV6Cb5dvyc9ivr/PSPpOb0Y46o9BqbePRoWIj2Wr0o8NiXAPT2FyT37FZw8n+L9PahTsr3tzg09ShO+vRVgpr0DWV49c4YfvcI46z3BC+A9wO7QvXrC0D3Qads9Xop+vWmsAz5wVuu9S+zdPO6PZr3IlQO+vR3CPclGpr3Q5849ED3KPKSoNb07ed88gBwjvQNocr3UUOK9Rja4PGXoF73rrke9b3WxPLKJwjxc9dG9FOHOPWVMCj6pyti8R65FPJ0vtrouewi+D/jfPds9jr1Q7OY931dWPX9ATLzTCZO9bwnDPKR3yTr8oB8+YVizPbfdUT6tz3w9bkQ7PuOBfj2pTaI8EZ74vE9pML7g/VM+5oEbPlwjwz2zLs69ZiIaPfiOr7wGwis9PTw9vttIzz0t6wK8ekemPd0wKD4oNeI9GMXsvWlztT2kw7+9vIt1Pdvn9L0af5y8QmiavTTyB75oy8w95l0BPunvG73nq8+76zzDPWE2qz7aWKk9oPJGPtwmk72+u2498B0aPrIiYz4FeSC+kYm4PXYAXj0xTO88L66ZPd4dDb4HZQK+2n1nPItFCL7DGhY9gM04vf/+Az52QX4+","5cPCPaVs4z07lx89Uy0pPWIofD36kbC9RsLePYXlgT7lym89aYCtO5iLjjxyT+s9WMxDvQFsqj09fRM9RAbzvXfporyH4Mu9B5twPvd4Az5TLR8+ESa4vU284z2jXEw+ckKovD20aL6NfXY+78UCvnfmoz0rATG+chxLvR9ctb20QTA9cFSNven96j2jPHo9Q3qSPVhUG75ve+q9kKrsPUebrbwkNic8eQx0vV69t7wtyMo8FUGwvEAcdz4dTBq9AeIHvqn0rj1ys0y9QixOvqc7Dj00/PK8WhvtPNOpKz4iFBy+QR87vRk4AT58q7u940TbPeXABb1Gcmg9OagIPigTQj4Yeyg+CWLVPfz+f75VpaM+iSPnvQACfL4kLCS+sRFlPsolgj71LJo8Y3RKPrNjDr2DqkC9lsuEviMHgb6htja+0JwwvsGCjr7zgza+g0JWPTxlkD7x93O+7dpTPkKJ97ykNmg+nuVwPhNZUT4JtUm+foiPPPkXcD41R+I87/3PPcpotT5VOA2+H6lIPmpG5b3fqhY+Q1ydvoOJvTqbsRS+MXOZPC+Dzz1L47i9WRAfvkqD6j3UAiq8WMgbPq2xur39+BA+DQKnvD4PiT4AdQ0+zi4OPsV4rj0IIqg9P+FpO2eJo73aJCa+t9DYva8lxT08Q2e+dw8GvUKJ5D2qRMa8on4fvUzk/D36pFC9D6QtvmWFQD7EX+A+jk4rvXgSBz7Ngai9e+MZPgCvIr5blZi8i/kLvtXgyj1urwY++rwQPS+14z03+3Q9sr0jvgdMzL2gUaM9oXkcvVBpbT4FN4e+2MiXPFhLlL3sMwq8QVNkPnnaOb3SanM+Y2GwvusvBr7iiBG+z7gKPiITuz1wYmC9NvVTvs7PJb38BQw9SXPAvSi8Nb5/h9+9MYJ8PUIFwb3gy565CpC+vmE85Tyvz669Q6WrvX+UAL6S2NO9qJk0PUSHmj3i1lK9u9ioPZwP4T156xE8HsRqPmGbs7x74fi8tmHEPfkE5L3676K+","usgfPtp+NruaiCC+86M4PRxjXT4M0ig+d9oUvKulUr4QvQ291DEAPDkDCj4+i1E98AaJvMCydz5uK4m9Yc+ZPaWRT72XTdm8A3MlvlhLmb3FXeq9uEgvPvkPqr0q0rg8g81DPmO0kr1yBSs81YsVvqN/47xCQHw9pHoCPY47Cr6Z+Tu9qrvPPlYqM70ZcN09puaYPrDRwb3wdV2+9n01PSPdi71jVSG+nwQEuxdxcj0Dv9E9R0iUvXhYKj4xcgE+Ef9bPJ2zJL5b7E29+cadPpVHlT0NugU+u0JKvXJTiz0Yikc+Z8I6vnTy9z3k/oY9VoVDvhmEPj0zgNo9ej5VPuQf37yXcJw9Iaj4uo5Uwr3vlgc+OJkAPhCWKj7U1oi9gXItPqvWVbuF54I+xL0YPmC/iD3uZDg+CoD5vfO81L30voU9nIXfvHHbmr0uV3Y+jc9ZPsF7AD4bEoA9Jf2zPU1l+DzCBl09KeSRPUtIQjtU3bI9g/ISPmQ0BD1mTK09PjepPcwdiD7oOJQ9CV0nPnF5Qz4Jxue7mLFyvaYINj5f2os9/jM+PivfGj5pqGo+/2mePoDse7yAsRU+eQuTPlezhLzmiL89rLfYvGPZNz5brAs+UZ+nvHuINr14vwM+AbgnvHtufj20MTU9G/xnvfDdtj7LNkU+XCzLPadAGD57qco9oPZqvT4dXD15xBs+hExpvsKZKz7jOAm8lw9MPb2FYr5rjEc8ODxOPBFDvT0XO+w9Srf+PQ09zDxj1BG9xyQjvVjo9bw9nJM+R9kAu4ZwlD1xMyo+9nUDPZidrL1ViBs+oc64vcD9/L2bp4a9YvnwvaUPhr51U1y82whRPgI5qj3SZmI+8atdPvLLlb48Ow69lgs7PTGfJj5+3R8+4dehPWgvuT1C0nU9Z6qaPFGPBL9EERS9njVoPR/0V73ywzg+MSAlvQZZsz3mjcQ9XRWUvVCcCT3mlrU9exyjuAFBO73Jsx69YtEcOwIMd71oQM68TzsovtCVKT57GCa8","+Z4yvvL7b71TgwQ+YRqzPUqCmjyQKKo9YguTPv4CcD5HCpO+vUeLvsZHlz19U3M+FjZQvqkpwT3BvxY9XiGXPEGrgz5PeRg++A+8vR4XNL7s+qw8RFnPvYY4YL6dOva9WVZ7vtoxDb6dlaC+UNIlvYyDY7xespo9meutPtlxLj0mm8E7LwJRvRLX5ry3yoC93ZnFPYfvA734aoe9B5mNva6h1zznvt689GlivtCQjT6kERg9dlCNvuvTfz3ieMM+VZEyPA8spr6fh6q+1FVSvu/k1bxl/IS+z+7vvch1U74SiqY9/T7QPnfOnj0LFBy+DfcgvZ0enrwaesE8rO3rvclOvjzdp4G8RodcPQoJNj5vtyo+VCA4Ph6NEz6KGVo+1onNPrlhNT5jmoQ+RDydPLCVzjyk7N48MZAEPuhzTbw9DbU+Z14BvsvAcz4tC/U928n/O0FSPj57PvY9QEJ+PlX6yDzthFM+dcN5PSf4bz7wSGk+XlJUOrcyubx3PCS9PLvLPS35BL4MDms9P4EwPkmjG7sKdoM+PMxcPvCvXj6SZqg9n0M8Ph3WBzzlyEk+rmyBPW6Z1DwUa3w9Rxv7PIL6DD5YH109NpErPuIfAz7nFC4905tuPTPcJj3sQn0+JTEYPdN3Nrz5Jhw+JHY2vPQSCD6pnYw+pYnPPZeIAT2iyJY9Ll4kvQsjYj2xxxg+iuCQPo5I5j3qCuy7IGEIPic4kj0rO1A+RJD3PWCDSj5NBzQ9CoX0O9eQwD3GMJe+qAYevBy2mr2f0/49WL5kPlCVPz6IBMU9Q6liPKBZqj28NGG9traGPcFxeT2Xc709lD8dPnH/VD4wM0s+47BQPTDEMD2+Y9k+5orGPdOX2DzguVU8/bqYPVp4lDzcDr09RfhgPawWGz35GEc+z2v4PfDtoT5Eu8S9BENtPcobyTzZNJI8+DeIPfqhQb0v/EY+ITBXPjRJtT1Jr8y9MRuQPgvnAz3AKSQ+1wlePk/qBD7yB6091/dQPtHwNDy5uQM9","5m0VvI2Q1j0eC+A8NiSVvTmG+bvl58C9h0SjvZYBSb5Ej8e9AErYPcqDvT2AA8E9a6OjvWcv0D1ZcZo9u8eLvdT6T70mqrs9ncKFvQVmRr3Wtiy8t/KLvmIs2r3dhKk8mj0KPGk64zvOAYg75lC+PVjTyjwfv0a+r4xvvZ3O9j2mgrI+xGjuvMRIIj3tZ4E9trPbva7AKb4JPCG+hK0ovSyiLL7H7D69kyaLPYPH9LzZ6Og90VfVvU6rjz4afYm+TCenvcAMDr1m4YW92qfhvLhCB76STAY+kUENvZHFmz4AxBm+TxQ7OlqUUD6VOM49HmfCPfJvc75NpAg9HIwIvsyrdj21Vw87fs4RPWGR9b3Lf8+9yf2gPsFObj7RE8Y+eA7MPTUI573r7Ue+N9NBPhi4E77CjlE+aaFjPrg+jDsFlfk9a7PMPcjXErz39rI9O06EO848eD3p5Ue9nBZJPXqo9L2ZIha9Ai6WvnaTJT3Gmqs9mOtRPQD1ED/iF8q9lk7YPSVP7z1PZ5S+PYfKPHNg2z3hI549dZJTPvk7Nz6X0uI9KAmJPpAVkT3OdE4+Q3isPShx1L0MuKg+1jutPRlqaz4yAyG+VNxQvIzy1b0jWjc+qvzvvfP0kD0QfzU9eErHPsdKsz2G2ZK9QZKHva1kr7kEXkI+Yvh4PZDKJb7dmyI8/jK4uidLML0nsTW+9M+wvL8ZJL1EWMW9Vv+cPBKpAL0knuS98Dc3vh7x9Dzjkeu9+s0IPYl+zzwcYzq9fdRFu+qpa7yfcAa+fHOqvdVWgL4e8De++Q1Jvgsasb2ELo285ShKvY6gFr70bB082/ozvf5Sy72TCue9bnwMPt+noL0Sp+a96bervDjCvLy3u3q+NjFXvUTo7r3BY4087+iEvdlDxb0oAG+929YDPs2/eD4ysPu9aW1UvlETbz3q+ji+KfstvZ/DS71mMZ+8Yn3AvRknxrwJVMC9sbyUvv4LGL0VBoY9KbRDvu5l4L1XUUu9kz0mvtLtnj2Psra8","ypGivY1qOjwte5q8RKFKPfKsFr6disI9yL8wvh9qJzzGkG6+9p89vZk7tz1akRc+AIUNvp2DI70P9J29M7KSvG3+K75oyXc9gXQovrqoWL7fPKw9z6B1veScBj0SRyA8minMPA89aD2yh8a9M4Q4PWuseb3tQOC9+uRtvpYffr0zarU7DDu8vZhY3L14ppA925AdvultPL6jmRO+j5tjvoSjhLw2qWS+ZTwtviNJT71oFbG90L5qPA8s471hWca9jPkOvk0vIT0UcY09uGi+PKMOir0qyFS8utNbvfwbC77SOmq9hGsrvhXwS7y5zAi9jOolvnDZOLzwtam9UskWvpIgWb3jLV+9WbKUvN3Gkjw1SWM9FOquPazix7zP5RA9Or4GPiQSRz6xk7o9vyefOibe0bwgFBG+7uZMPW0jyDzy5KK98wurPb3nojyHtQE9SDv5PUwYSr23ZLw88hSlvf78Xz2qIme99F0KvhhWC74dv989bTABvQ59mrwUi7c8yT3NvWoZ+70KeJK9ghWzvbGroD3hujM+GIFaOrzsPDyu6pY9pXNCPZc7Cb0U+Ci+wl+YPbb8Zj1X6Nq9X9poPP9yoL0oP+I91pC8PSfzRL5kJay9/6k9Ps9aqT3uByq9Ag+XvfoSE75Tvqe89oYSvukAvj0aGNM8eXvBvRYniLvOOjO89WcSPTPYEb6nyRs+dVYsPmL1N77X0xS9ZZc1PqztvDxOX9Q93AsSPeePOT60uXo9YcygPdx/tD3nxMq9tQppPtSyxb1elFS94y1kPuDkIz212Qs9UwKKvHiMHz7zAJa9/um8PZj3Br5vyAq9Vn9Lvb6Ucb15cO29NbYnPlShAL4RiiS+MwOdvAGdGrz/ZLC9roJ2vL2gF74Psp+9xCwfPs/9Ez52qdi9SQuAPWNSjL0PIyw9ykAnPqwsDj5x9ZY9mY/cvLpnGz5eSpe9Zl0LvRXaDT26Dwu9lgAxPqe1Rj2SyBc+XRfHvYQiaD1hQRS+L4EavjD6sD2Ar+m8","JWBaPcUuCj4A4bK78NQKPr/owDtxJh8+eO7ZPQpcoLz6KWA+tVGIPc77o7wIlI89M/f+PGAbjj3g504+qIqcPXS4Ar2LUEY9xljAPXTMwjy+kDU+16W5PbDm/z1nMjQ+ce2nPVaJND47U6c9/yoyPgVZgj6y2dc9Ngy+PVBYuD00K3M+rvqzvPhn0T0C2X69BvZHPg80kT6h2bs+0uhsvKyGFT765U4+fvKfvSgyxr2K24U+CsxBPguB2bxk2wI8dLvOvMIQo7xxLM88IOa1PYTGYz6hS8E9d/DjvPOCdz7uexe+4JJ4PaGEEj5ACBK9dfxIPkM0HD0bD469ndMJPVw7Cj6qbpY8nEN0vZCkIz0UDos9j7nouWi5ET32QaQ9wkLKPAhgVTtkBss9ycAJvlk0cD1LehM8nqgZPFU0zzsexV483knRPc2wejxscC896Dq8PdbbmzyMHCo9iG0uPRgBsjwT+qg8Md2dPSEGLj4uKEk9EhEPPSUCUT4hAT+915koPtbT5D2VpUs9e59rvdeKqT0zFVc+4k4oPm1Kpj3cZWU+IpJaPsCjHz6NQWy9Jx2gPW3KRz0aiGg+GT+cvJM5WLx1LvG8+ZHqPKBcHT5GTGa8fSRxvSMn/7yRDow8z21LPfa/BD6mIN89McSJPZTOzz0h2KU9DMsqvFbJeTzEgN+9XPfnPajg1j1ehZs8d56TPVZahr0t/wO+e5StPAGVBD6rlWE94HKIvKFFuT0W/tK8T3w6vK9VEr6HFLC83bz/u04VLD4gv4q967g2vlijkjyfpqO9qbZNPSLgVj70BYw9RkPaPXYfgL0QN049JCqyPWcZC71k4BQ9bXq/vHkKSj4miAc+//IZvthLkb2fHYQ9SN9TvhTpKb7y2K08OA0EvQRtSz4MfNy97KZCvUfspL0jaJE99I+APdbZfr1QPlY9ydpYvezsV76/4r49d4+RPJn54r1Pdcs8BLNvO8mXi72tWsk8WwxEPmBdqzyNRnA+4cDnPXWmTD3ICpK9","rYWrvY/CtT2q090962W8vZbUdL478pw9FdoNvbh0BD3AYoc9obeFPTuv0j39rQW+/wYJPQKMqbytvYM9kwAPvdd/g720jZW9+DuxvYc1jr0kafI8AFf1PSYySbx9Nmc9dPhSvCFNeD1hem49KAGOPbih/Dz9FlK84c7OvRUu0r0KWaM997SuvUriwD3dpNm9XbagPZEe6zz+C508IThqvtZ7wr154D08AoZwvMYFDTwnHwU8g2JFvqtLDT2kvSo9Hf1nvb7ag73X6gm+Y5PPPeajE76j94+9gmQIvv5nDr5cAG+9UMYYPcEoIj5qjRG+QeVRPUo9ibw3xQW+SkXLvbEFkj3WFaw9POSRPTYHhL4jfO+9OeqAvmsjS7yFDyi+cYRwvrsR+b2fC1S9nMx6vnAz9jtwtng8o88BvjJhoD3H6Y6+BCBiPJIVy74RDIy8EIsmvqkAkr2cVGa8bHOBvOaeYb4xYNO9jaAivrAGE761KjS+CWBcPf5aSTtICR2+eWrsvbkluL6pP1q+9pYgvF6dOL4uVbG+SfITvtG2l77bETO+1IwMvv6qEb7Nqlq9wWDDvjzzLL0Tby6+vESxvgyCT76CnvQ8Y+DtOixcUr4ucpi94mUjvQCJRj2ouG++ik1Gvm/0Sb7Jeny+Ng6fvMVaGb4O0Qq+CxwyOgBQDr6HllC+rXQ9vSdb3r1b1Ps87M6OvWcNL747WIA8R7aQvKyJrL0Mgfw7+dlcu1qvzL6Yaz+9jTAwvmqIOj2gWcw9+OEdvjtR8r1j1D690Q4Xvu0+fL6Fo4m+vEwVvSLjXzvBZEm75+3yvRCPHL42raM76CXqvUSC0L6rfdy9Z8fOvcuao70IEf++ECZDvrHibDxE3jO+YU+gvjM+u73ua6W+qYg7vl3fLr1pfG67iEHFvQ+947zOvN+91bt7vsQAD771Fmi+8Z6NvieVEr0equq9bgq2vXimEDwqJfg92WjKvl0tqD3pAqG9KEyAPJXs773FEjq+SuwvvqAskTpiKf49","ZfJIvQnaST5GMZI9v9drPUaL5D4dY8e65g9lPYErSD6QXW8+lmAAPnW2ET1TVCG+XSMRvlZa9j1efUy+1UKRPJ5zNj6c74S93wriO6DFWL0Gof29oMYiPiMJEz7DWBY+zxylvaD0+z3GMbU9CIg3Pg1tP77HjfE+z4TDPfQ3oD25H4W+yT+GvulQNj5Rn/E8/pa2PMnYlj4BEmo9LyaTPZc2x7yhHOQ9iETTO9n1gz2giH49/WCIPUgyHLwqbhg+ZTY5PkxBILzcjaW90h9GPgltMz6Yhas98xX7vaDmzb661WM+nLBYPTAZir50VuY9XFpsvohFMj6RrWc8GguFPfb/CT6PYP895DFWvRNhB75LHs09ySRJvYGMBr+oZ109dmbPvfi+OL1Kd5M96GGsvgFgXj6SgMW9IRzxPK2qxr0YPhS8DPeAvn9hwD3IqZO9PHj5PC7j2T7DPPI9snV5vLXH8z29VKa+Di+0vU0XkL4RbMM+vko/vr2mJr/Q7OQ9IJXyPUVSI70YqwQ3LhumvVGwaL6kLiY87LVMvGQ3wL3LtIE9cNKSPXY2iT60USk9dA1ZvpHcrT5sL4u9+BGzvTgZ5D25r8A+47MoPo2EDD7baIU9+YY0vYV2jb3Wieo9eBRUvrV/CL7y38O9AwNMvAPFGT4T42k+O8l7vluT/T6ihi4+6/y1vQ+Fjr1vRhk+ZCLWvU5q6j07BV28JpLKviZeaD5IXs09NlIdPqKDaT609Vg86gntPWOYkT4ig+e9wQXHu6EECj73HzA+WkYAPv3FKj1g78o9/qYvPjJJkj4vVA+9k4oUvkyXuT3jofg94GJPPmT+lb98pL897cvEPdIaGT6dF42+/HtoPhTn+73d0lQ+nDF/PtEMQb0UPny7cSSEPu4GJz4qw6C9PunPvEvHPD6FTm8+fSioPSlFfj0p7Vy92CqePmn6y7xgZ1g+MCE9vQ2DELyjISO9YVkGPtBhZT34DnK9iHlfPdDnEj5Rq6w+dwCaPKlj0LzPEMK9","ekI9PbVwGb7jRTK+Zo8iPyAO9j14IZI+AJYfPvLqP74Z04M9IbihvZEfjrzv3MA+npAtvFOC7j3+Mbq+6WZ0vGhY9D3AZZo8PK8Ov3pKW74gkZk9mWsvPgmXJT7faiO8KtS9vf+VxL5B5sK+ZJ4ivlwwHT7CiMq+O8ODvqDV471TWw2+gpOXPq6m0z1vwyg9HRFOPbIqqz68gSW/R/1OvTrrizwYUQq9jwYGvqUr+T1IVd8+HdSPPoLMN70uJYi9bkeXvll/7D740au+ouFkPu5337s/zgO+sFDOPQdj5z18I/I7jDJ8vjMnlL60XCC8Op8IPp8XUL0UUMI76cmfvXL1NL7KRGE9ERLAvWrqv72RQg4+mhqCvn8PeDxAdMi7atE3PpS6UDy9HAe+C4U8Pg0voj3irQI+3zeFvpH/zb7GJOc9vVg8ParvCz7E4BS+uOMrve1U8b1Auxo97IoOPt+PJ76K8ag9YpAHPlbiDTzvOLy9nDsEPSCeMj2l2LO7iQVcPkC3Ib45bz+9VnVPPhZZAT2iMma9jpxEPidSkT0c/IO9lZpVPnX+jLsbbqk9cN7yPFm+ob2olI69SlUvPRranb0UK3y+J/UDvenP3j3crMa9X36sPQkuxL0TwoA+oSm8PcPtpL5Lp2K9DHaWPleamj77S0W/aD4rve90NL3vJoM90klqPeaVnz3kXzM+QZmjPL5VPL7Ai4M9zDZ9P5+ToT04KTw+vaHYPQdjHz5lRos+G7hJPYKLnD7xiwi9EsmGPzWSI7+44GU/V/KGPt+AeT4mkSw/fbpEPgD2Tz9+cik+3WErPi4/xL2Ouhe90WYlPjlWgb5C/7I+8zs+Pmiuib0Q5iw9APV+vJcliL5d5CW+XAqSvhSAWb2nPtM+iK16PqUxsD56XKs7zTCwPal6pz7pGiw/zWRPvubEZL7czs89/ZqxvZZPOj1bek8+vLIKPxlYS759kpY+l/NnvU6TQD334Bk9xH4NPngfHj45P5G+n2YjvRg0Vz6bU/w9","ed/jPOhqPT1EivW9ckUUPjc5Q77Sc6q+X4savvEbtb1Vy6S+hVd2vWdePL78heC89L43vpftJb6kRgU+xAkcPtHd/7wiKBS9e/EvvnWKU762rV48Ogv0ve5mir049US95Zm3vbuupz04Ft+850fBvV8TjLvTgAS/IoIvPpiTsb3mFme+NwZjvloGbr4Eh+y81iBGvc8mH76tsZw84R6lvt3txjyiARW9J0WIvUmzHDytxe49Y5eNvrzOFLzHIz6+hEt8vgnPDj1h5Lm9PFgTvkDttr3MxSI9iRDePhJdkb6UgV89MD6hvuTiCr4WhDM96kGTvT+DrDwn5tu87kuIvgG2zL2nOQi+R6mtPSyNVD4ilqa9LJyWvrCtz732Qim8pSTQvdn5Aj2iswc+iT7GvZQ5Kr6H80A8fggvPodlFz6Rebm9HXsFvvOMrb3nsOq99ipxPpsrfb0MFBq+d+UyvudUajxW/Oi9fax4vcoEFbxmSBC8/0cVv2lXrb22U6y9k/48veQZHr4p6jy8h+L6vfwilb0ltqq8uvl4PGyH6L3uymu911TZvYv3LjzZJJG9ur9hPUJ2Hj7OLPy7rBgbvrrgg76iI6W+OYJXvcVb7D3uliw9fJ8ovnSvtj7o6IC9+s0PvmoVub5sMIU8Jam3PcjtJ743eR+9r4DnvSyJTr18YHC92pPxPeWwgjytd1y9LDHKPnAfzbx7ioo8oAANvRS5O7t4PeC9yxYTPmQJ5z373/q9FeLvO6ubjL2+xAm+1A83PX8Ttb1f/6q9mW7gPRktED5T6Ne9z1mTPJRfPryfuYq7MEMQPsJfaL69MBo+sMwAvr/vuj+O6TA7by3gvcw5g72zzIu920elveDVaT3OF8e8Cg88PptSxD2UvYy78CTavTsHljz2h388jf6XPW7Eqr01dm0+IjCvPNGlHD6xfUE+348NPnV3I70+OJ09yE0PPXgzKj76IDy+Jzd7vR+LdD1abqw6FVljvslUbj6xqJE8mho+vS5roD0oP3E+","oCvPPSeCez5OStC9i07aPlm4Vb5Q7xG+DSpsPoSjCD+kSRk+/3UQP15XnD7+aR89NteePoPweT6dgAQ7UchNvkEjBT7DlE2/GhuCPtFWk717c6A+7SUuP2ukgD4mH8c9vsJmPgnikr5j5ZG+P0D4vqM+Kr+/GgW+bjNav9BCiT5XpJW+k0n/viCvGr6SQIC+IXlIvh0orr5Hj/y8EcQivSOKOj6Meco+QZ+zPidFOz0w/w2/X65bPtaqnz7nz/m9+hGivasM4D1l1p4+nrooPe5T3z3iWOo+QRXLvokB+jykAy2+fooMvqyt3z2WdOY9Vc/vvj3Y1b675jY+1hYyvSJwxj1AF409CxfevJl6WD15L5s9LNZNPj87TT2cbSw+p54UPsi0GDzX/7c7hXULPtzsJT78GJU+ULiCvfU4Jj7fP4w+DgP1PZWXIz7mvBk+PO9KPQFSAD4KKxI8gf+svd6Gpj7TYRU+sBk5PUAXTj2IQCE+jQxkPSpng72Bz2i7rnCQPgOYRj71ffE9epP5O1tNcz4PzhE+oRsyPlA97z2yPR+8u4SiPcQaAT4ufGE+PEmOPTTFsj2/iRo+dcaovXMGVTwUk0s8y/7zPCBYoDw7/qQ9VQLVvIHnJD4JYG8+5jHTPVvp/rupdHk+TqYaPneajD50Nh48CJMePrxTMj5V9FE9M55+vYFxZzvmJec9CdCePSflej0pbj0+f90uPt/mHj77kC89TvX/PexT/j1PAf09FW6JPstvlLw+c3E9RRVovUJMrT1RrNG9IkdpPlNtsT6shJg9H/flPL1XMz00dM892TP7PWZ2eL27J/M9tXcYPjzn8ruWPSY+2FUVPj1cNz6R2YE+8QzKPRDGXb1RaLA+S2OwPRRkmD3jFZ89mZG0PYTPIT4SSdU5YMz1PLzWp7116xA+zUmKPqf3Kz5lvkk+Lt9lPsEeIz4o0T88y/cHProBOD5mL5C9oDNFPtPDUr0MYGU+s8kWPki8Dj5aqPI88eyPPaldRj5y3ko9","2m5MvlbHHT60nbk9Jf6lvdRuHD7sUG6+ofqZvmJs171J4+E96s/NPdBrX76y6bE9s44HPo3lhj6Jda++wENDvv0N472Gz/y8GH8jvhL4mL7kn/E9Meiyvd4o3L18VhQ+YbvEPcmcHj5/R/89Y3xYPibWWL6v+tS8Qxh5Ph9FWL3jbPI95xjmPLSIGLytQQs+p2fdvVjF5btRbYC6zWM+vQUuz7ysaBQ+UGYbPnLj3L3xWxO9O36UPS/knD3vlR89I+TZvKFBOj0+Cco88KUrPvcDGD6NoW4+hu8wO0iTED6t3cu8lEaHPeagpjw458U8OBM9PZBc1rsjjJe8IQNsvQuqNb04Pg8+UP+RPJGVZT2OHEy+I8b+PJClRD4lrea9wOv4vdVqc73Ro0G+DtEOPqccZ7yFCfC9vhgOPguiGT32Cmk9KHzdvezvHD6oQ9o8GR+LvbZUubzNGh2+tfKxvUCPAL7gVp69M2Cvvfl6zzwgGRY+f6EoPb/PaT0YmXg9BBxvveKNXj0D+vW9boBlPh/VDD793lO9YV6aPdc0V71kuy090LgKPUV1i71nnrg9j/oivFitx71RiwM9z5HzPbncSTzouUC+nwY7PMUeUL3tJum9pmscPnFFzD0yEAU9Q0iNPSxKCj5J+Pi8u6JkvddJbb5+5WS9ygYkvlKkhL3/jYS+J/i5vqnukL2UnC2+HGuwPvEHgL1vhpu8L8QWPujAij2ytTC+o8PevdqkTz3K85+99oMuPXYNA70awIg+5VZdPleI6z1iB+48boBhPnhZ6r1rAcG9Gtz0vRt2gr6cLRg9iC2vPTqUJ740udK9GDSVPIc9nz13Kx27zoucvt7iurxkIJ8+GLtAPK1K0738u/O9OSa+PXweib1h0Po9MW1avOfd7T0gVqa9oatGPBQSgD6qmNQ+xmE7veMLRD54fTC+d62qO3QEYr3jucK7i3PwPRnajj0wiAq+Bj2ePkYbmT4ywpg+oUTxPJcvGb530xU+aa+hvAlhHz7byS89","fTIxvn5Jvb34FRs+ETqZvoM95D3+rEm+leWUPVVsKD5GHPY9FTrJvfVfmL6MsFq+n1A2Pb+neT0M8VC9K1hmPiUXxb3rO8g8ArgLvgZA4z2aXTq/NrydPMdpoD0sfq2+tQshvBbD8j2qgYy9rvcjPmW7Tb5ESRY+KRwlPpmrtr7is829aDenvaPt6j2nAZa8WazYvZgeOr7ZqNo9XJCCvmUTZb4nlLM9w5b8vF2ygb6FGRA+hraTPqTOAb7ww9m9O4oVPqTNib3QUeM4dM/jPTOSVT73L/q9GnLEvQDFb77H30895TY2vT9NNT2bDPc9PvXAvDiqFD6Y3oE9AXMWPoKS6D1BPtq+TlXcvaGT0j1ouXa+8mGLPaH1DD1iTQW+KMYwvg0APD4suwE9QcVlvGmP2bt4JIU85O/aPS8WgT6OuOC9MciOvNmtVD41jq09WpC2vcQ1VL6c6hO89q1LvrprjT3ULuu9Sj0GPvwDoL5APGs9f7BLPIT1WL6Ytv89lBuPPj99gr2umKa+S9RHvT1vl75d4d47IyS/vX7IZD663Aw+hnwXvsyXpb2rVcM9iNGAPVOlur1aJyw99teTvk5SyrrHh6y97onHPC0Frb6quGi+Kim5vMycwzu8PwS6MAgtPbcyY74tb8G8XUAEPo0E1L03Z288VbMcvm1b7DzvwZi9z4tYPlXBRb71BPk9gt+cPqiLer5bqia/lRZ1vnfzkb5LL+Y+5FR8PSajCL5uhwk/rOEEPa4HYb5eagw+0msCPjWoqr61fzs+dQ0fvpatPb7hQYA+nSEivgT6r723CbE+ms4+PQwnzb4fMRm9BshcPrH4273NHOU7osuMPTneEL4CuQU+ENzxvdf5Lz7lInq+clH5PiqbMj55DYg8noj/vVDJIz3bM4c9e93ZPXlyWT6d1hW+2g0DvjFvJr5RnmK9ou4LPd7oEj52b4q8RdU9vsFy2T4uP4Y+5bIsvuA35r61ys09DpuXPUnQxL46RGQ8hOS/PRUg3D2zh/M+","lX+BvoJJjz7aPGs+J4SDPoWMEr6Z1go+FqDcPv/tXb4GFo29qJiRPbeADT5yEWg+PwnRPWrGQDxnu4A+ydlsvnoD776xeXG+ZXsLvrUbUT2YpMM92B+ovYj4zb13kSg+yj7TvW8fTL3GMwK+nJ6jPKVTbT2+1hS/QvAFPvarEr4OpM29K6W3vvnN87zTEaK+k00RPwNh77054hw+1bqvvkQ7bD0q2Ys9J72DPXV0QTxYrEa+mslDvoXVVTqGqxO+HAoMvhMS7j7qkpA+72+3PqQLiz3pY529trNivn/zmL5jMzC+dWuLPnV+J72TVWA+/o4TvsncnD0/DRs+gsmhPjetMb7NKqk+y5hCvoZPij5AugQ+IzxcPl13DD+B8wO92YM4vPMpsrxPh6Q+1kyLPu9glz00/Yk9eWS6PTQSXL7vUmq+j7eTvqpnXr4N1/I9jn1+PpS5vTxcq2C+1l45PkugXz1fy4G+auqSPSWEN75YpDI+REAav/l5Fj3iV1++pcJ8Pu7llj5LUUw9VwX0vunr1z7mP4m9kwSrPvPsyL0vaT4+nRJZvW01GT4oMxA+bgqLvv5qGj2lbks+LkoHPqSRGb5yRK4+Utm5PRCl4T7xlgk+PQdmvbRg270REGg+is2dvmquTz2+JbI9n07fPWWZlL7cVvk+M46NPsYLwT7PFUw7c60RPqju3z1R1le+16jBPs+0Cb5xlpG++BHFPpR+mD4Wcgu96UTJvUcBOz5Fcwq+Z/z1Pdwpmr5gXtQ+A6q1Pqaw8r0bI8++uHAvvlR0mT2x0zC+Kpa+vWTWvz7wW3++8MW1PjKU9Twwp48+rP77vc51Bb5K8yE+IcoDP1TJtb0quYy+TaW7vSSy9T6EAYU+PTgdPiLZiLyF3XM8yZ+lvhKYoT6mEvw9tKOlPQZRzD4FYYG97jBUPhArkz4M3LY97t4Tv9Ahc77gzbQ+GY+4PUVEVj1eBKK9xEumvSbwWj7UauG+hGSGvk8ekT6HDGa9ohMbvrDLkz5YI9g9","G3KkPgzCIL+PkSc+l5wHPLCXdzsrStu+BBymvkskj71zHhS/kzvvvedp0b3iU0g/pMrNvVUVYr6sq5a8xV5PvT9Y/j60Ji4/t3aCvHUiAT+R9Jc+ShXavj7yFD8hjCq+1m1aPp6iVL6OM/c+C5CMPqVgBb7f/nY9YYCkPth0iz2q6i4+ocavPkJIxz43/ZK9NScqvjP9Iz4m0Fq+1zcEP6xWHb5UsBs/24jevkoKLb5Dpao+BkzkPfMk3D0wSZG+AX11vnQHS7/80n69oCw5PjSVYD1a+ZI+kKqiPq3TsT6l6xo/dLcmPAaUMj91HwA/c/YRv+Ab074QFKo+B2VKv9bYPD3x3KU9/ABTPQfshbzK9ro8M9JMvig1Yr6v1d28bMZQvuI4072uNQu+HHcnvrCq372HaRi9chi6ve2OIb6F9lC+/AeXPRfXLL5twBG+uu2IvtqO7L1AemK8kFrAPK6UhTuMUX29c9vQvZpgV71JFRe+Lo9Rvv1p8Tx+bxI++6k9vrmkPL6n7tO8Ht8OvYOWhb7nDwm+Z4ZfvYvqxrso+mO8prPVveLiG7yLR4K78R3DvcZdl77SMxK+6QIFvDutrjzzgd09Eg7HPSG/qb321QQ9wT+rvSzHIr2BLJO9VBeyvWsdHr4suR+9BEu5OuMKODxmFzO+qpO4vWRkID0rNQy+oN9OvYuDXz3V67A9IgGJvjA9D76GQ9u8QcmbPOGMsr2ksPe9I9J5vbzT8b2KpT++PP9TvkJiuTuxq049Rtz0vSgkGzu8edC9WAkivgFQ57z3S9u9Ege8vK2fIz3ESb69KMErvtwYyD1dso68v5iGvilewL3uYiS+utvkPGibpr2GlvY9g6odvmTD8DtYzqq9Fvl4O/x5e70T9xy+d+sHvgRnJr18QjE9V3g+vn4srD1RNbK9hpwJvR5Uab0lL+S9fk7bPUxqR7sC6R6+pRH5vUhQAb5yXam7Ps19vpx45rwulxy+5LLyvbx37z3/pRG+bDnDvYgcGr42JPc8","sHEqvRAsv7wiRiw9J+uBPj8Ha73TsQ0+GipzPjJmQDwF1gO+7hKfPO5YmT1vvne9FYAdviXCNb1mFqw+VFLwPQK8Gz1Fiao8hp0HPhmzSj6egBk+UVjfvekUOrx11fO8CTASPZyFVL6/jgY+W0aEvtRHST4dDdW98pckvi7oW70QRJe8nISFvBHu/T2bQIe9rFuRvel8wD0naAI9NbLYvKrhHz5lJbM7XKGqvcywKzwMCSq8bK/SPXJKUT05tYq+ox1EvTidtDw0Q3s9+ohmvvn52j0UUje+ndqXvK4wiL1Vxdi7yeyKPEhflz1uD4Y9/iwau+HCxL3LupO9XtRrvZjHtj1I2sw9/60XvtiuUD3RcfE9i2KDPRboNLwYv+c96LUmPvR5kj2Sina9XXX4PRO+lj2Zxwg+CtGLvc8nI72TXhW9Vp72u65SHrz8wBY+HmKUPkuCjb27zrk9sn6PPUADMr0aURq+/34FPtAz5rsihxa+KAwkvX49Cb1peY69roJEvPiVib3Yyvm8Q/7ZPcUjbL1ajxG9kesVvuZnpT2AmQe9n12IPax0iz0f8pI8Dm1wPdr+NLzRdVk+9VSMvX3J873B8Aa+kXnRPdDlAb28SIk92jivvfdJYTuK6Sm7X+W+PMoN/L0VQoY9s8I8Prn6Ar13CgC+61PlPObY4z3BnP08jHc4PVyjDj3Gbrq94aWAPXTOAL1Dco8+vCoEPqUU8z1W4Vo9eRBLO6sNVT3JxOw9IP6kPanURT4L3ds9De6CPRIimT2rqC89MJgyPtk9iD2CmoW9a2w/Poql1LytHBw+NqBzPdi9PL3c5A29kfEJvHJLYD6dJJU9WQYuvNmL27s9JEo+YzcWPi6rHz430Ts+jZmIPjYFFz5VIgc/GGlpPhL+9Twe0dI79HiLPbGNLT5CprG4q4OOPj7rST7JojA+9linvYCDEr4gqQO9BoBePRASPj5waqA7YvM0PrBpvD0fC9C9j9pXurG8cz45rLC75X5fPvcMMD7Oggg+","YjGLPY9FxryOawS93mSLPL8VfD3jVb+9M05ivT/tMz7mLpA96a+qPX3sPr1+oRe+o+FAvAUIKj2HtNW9hrrave/nmTxDZ8E721HgvIfjWTtZNX+9+hJoPWIFFj47RT8+bTSQvdKCYj6M14i8VyHgPXkhVj0alSk859WZPV1Jrz1HNhi8M5ZoPRYPBjvkgY88YgtiPgASMT5yAYa+DGqduuFlcD5+Fc896p4pPu9pJL2eD3Q9TgWHvTv/7Dzgy5A96cZhPWy8Qz2gawM91DGKPSKuXD5tRUc8KiCqvPnbcL0dAUM+6XMxvaR/FD1cQtM93EqdPWQDjrxud7w9GqLSPerRgrv52S09090CPmwA5D3Kj7S+Fto6vaL8R7yiiCg9Mc8UPpZxBT5O1AG+ugMhvTeucT2CPgs+rLJNvOonR7y+F7q9NgjAvaKuBT9065o8x62qPcxJpz1pdA++Lie3Pfv+CL7BnqU9aS8TvS0sjL1YDxA+jsNCPcW+RDy8kJa9uqs+Pk0SvLwgSVs9lohJvmbPKjyZh5S+NayRPf2QJL6enGI92HEnvYygwj3B7ou9hUVevs05ij4xrQ0+ALS8ve7vJL4lAu09kQAtPRONAD4NFoO9aMRvvVDY9zw3l4k+OfxfvsTiXzxnZuU9L6uSPVQBQT2wWMU81kEvO3DOUD3Z1qe+bA4mPgR3lr312LM93ttkvi1TPTxfZiK94EKWPXH1/b0MTLy9slfbvZs+Jr5SC6k9IyG5vf5Ysr3KtSI+lvORvbr7Tb2CfZa9UmVnvq4Rib5NIpI9WJmIvVnF270Z5N+9gZwrO19CJz1bYCK+i4keu36JorsqTNM8GIUGveFzlD0OSMy9bEd/vc0JBbwq3ki+uVNmvRezWT2zKjo8Bc8fPJZgbL4W1eS9Gv1NPBEDDD4d/fG85Z5zviUrLLu8GR69w6yavF97yr1rS9I9FBsMvBESnj26cDe+Du8UvtazpTwxjfG9N0uUPadxm7xXhZk9lAQzvR2kbb7Ao2s9","zm5+vtmuP77oqxY9/blevrCy2Dzuh4m+0V7ovYbnlb3jqYK+FoFOvkR2YL4urde8DyUavFttRL3cJlq+u2F7vs68PrxHr2o9iitGvrJaCr6rejq+M7PbvRtofz2fhhq+51pgvY0SRL7MZwO+5bcYvlmRmr6IpKQ94l6tvraWdzwWeUK9lZkKPgq9XL7Tkxk9qcD+PGDtjr1Zwy2+dwOLvXE4M76jzoq+GzjOu//XRLxroo48NC9dvhYVwL268gi8xp+AvmwRbr2G4QC+MFDGva9Bqrwudfu9werkvcr5O73x3La8NqQVvihRB74RiUm9N0/kveBHmr2l7g6+HOsePQvtF75X0Lc98KMyvhoqxb0V5fW7KDIoPRtdLb6Pmc+9C9XYvahGOr4jWCs9CDP7uuJazr0jDys84DU6vv/iAL46lxm+UaTXvLnUkr06khu9JT4PvmAYXr6Dbza+7FDrvfC7Eb4chII9NzBrvfybIb7iMsi914OIPb5Xjr1VUwm+dkR0vGBkNDyUHoO9s1UBPS+ai76dsIa+u9uPPGPpBL6bfnC+uj9KvhLPCL6C4729bUTbvTHtvDy4ZSi+17QuvsJQcL3X29u7ih49vcGxEL2Ns/Y7D9pIvRWdxT1/8Zq8cWeWvLxxTz0NUDM8aIVLPcF4KT3h2Be9TUHxvXj4pz31vIg+rxRnvvX3gj17o1s+MyqMPPoAiz6YrVw+KRD/PFNyhL19iMm9pDvWPXsYIT09G1w982O/vcWugT66D949rEo2PiHAxj0rmRQ+vuYcPvdp7z0HpOo7au1zPP8VL75nZPa9l3itvWkL7r0NNqC8r71VPmnXtD1GQea9NRUjPD+u1b3Fkna9LOA+PfzfVjxCPzQ+WnyGPmqYmT115NE7cyZCPaRtV75sqGA9NrBuvehH1j2OBxg+xsMsvlEEVz6P8g0+XU44PAJJwT03Xou+uLMqPaAP7b1gDja92ZH4veJo/jtzwwg+iGMpvQI8s72aXla9TBctPgTG773nKIc7","+y29vemfKb4K1pg9mDjTPcA5GT1/JZG9mz39vbvjur0QlcQ9XK4GvTKvjTxx0/+9HCOgu9figT1UUIC9ekrtvQzt6z33S469zlc6vbAilLv6SAQ+EuFPvQCHnLzfxoG9308qPQ/LADxw4JE9j19qPTvll71riTm+OTY3vsVAbz3e31M9GBpqvUX3lb0elcq9X/3Fvegjyr320za9C037PIHDPb1H/N47sDjIveM1Gb6T0gc9tEj0vY6TkrtD3Ro+/EllvXAZCj57VUO6l9aYvZcXrD3L4vu9QhfCvZ9Mqj35Pui83NqwvMuW7Dy8czw9FunpPYBFlr1c24M9TY/PPcSdtz1AZVK8+UvevMLmrL23osq+vhtvO1Mwyr3naCS84HVXvtJJjL20BxS97jdgvbUYyL0v1Bq+2zIMPqcEa74Dyw68LDycvc4R0r3Sh4C+tBu/vOVmvL0tK+y9Jt9KPnltnb3CJN29HLJRPjZZ472fMe48dh90vfWGtT0BdCk+nB2ZvJoYL77+Kye+l3CJPsu24L3H1SO+yOp0vkS5Zr7qKAg7mCAAvn8wQL7GPkS9WO2CPt8d5r3P3na+4L2Xvv/Mmr4DH9K+bsKsvIzeV77l21m+VdTevVsldr5Wm5K+PsJ+PsaemL7aABK+8T2uvE0pHT7qDJS+mc4hPaUSKD7I4DA+CV5WvuuIQD0Ekwy+WbP3vX07Fz2yJ/K9waWaPTA/sL1+p8s80AE9vi97hT1Zbam8904Hvulsnz6WXVC+FUx/Pvx707y4k9c7O8qTvhT5kj7i/uO9fq9zPQAPWD3z+16+WpE2vtrhoD4ptba+QY/pOnUIA741BN+9oLC9PCYha77bb+C+B/hJvQTChz6jo1M9NSXZvQlHDL/4/xa+DEeEve/E3TyJ91q+A95lvtFkDT8Mp8K90ki4vSx+eb2xDdO+GA/OvZ6lhr5NTyC+ZrqRvmOejz1U08e8Wck1vnunqj5jjQ6+wr05vfdhLr6eoeU8Nt1BvnLRTL7ngQs+","t9VkvpIUQT7rUKQ7jbJ5PE1qvT6DgdS8mAjtvWivxzvrcJy79fwSvd1Ujz3h+Ci+2mcRvtsiAz4jJh++q27avRfhF77WUje93Xq6PRoZfj0B60C+a9Navjl6m70kZvQ949FgvkO/ij5fMSq9i8F2Pj34HL47r5I8B1k+PkTAwL5mJme+cf3sPYjYNL1+1ma+w6+tuptbVD41KgW+RHBQPtpWsr3z/Tw++5OyvL8CKT3nLgQ+x7FFPmbamjxEq4k+PtoaPp91Tj6nGls+CanjPZVjDD7ysoc9mXM+visUl77nmjw+T0WIPpPcEL5LrK2+ZypEvafmgL68RIO+zsy9PelWTr3F7Iy+couAvjcpub4pG9A68BIwP4ixiz5LM7g++Uf3Puy5hb3hd0I+TZH8vi9xer5KIE2+LTGnvvjFUr4Dyva9EnnDvpNzub0I8I8+Ouc5vsw4xT3jhJa+rIEZPouGv77UDIO9+IBbvn6nkT64GbW+YBGdvTWV/T5S+TA8J6CqPeecij6ZSNs+Gt1KPpaZzT5/2TW+fRC2O+3UerzRJD++Rr5MvBij7Dy9UQ+/OLlQvlej7D4fWA8+tOe/vrFaTr6WGsU+mu+YvQN9mb4zfdw96010vkNZpr5kztg9s/NpPXEcVT0t9I++H/HSPZ5TPz7j/Os+UEuTPTmoyrv9Np49NlkzPhERrz1p5LI9RMpiPVRtTD4i9fk9GipSvXrNOT4tNBe9Ty0MPgKF4zsNihs6/FWsPSMhEz3+7/U9v5JdPrrOxD2IkgA+UYVCPuodg7zuy/U9DL8DPrwDn70lrx+9NvaUPA3+6jxLQ029kssMPTc2VD6YPeA9/rL7PUM8CT6sEpQ+wmhgPKheEzwnhhY+BmBrPRBYAD7ZKOW93P0nPjk5VTzKSGQ9c63VPbTqZT2qU1o+vIdFPuU2pb3MRJo9WbMLPhIZkj1OLxI+bjwCveep2b1eX6q9+nbcPdofpb2H7Vo9wypFPjBJAL0TILQ9LLVdvIl07j3xXJA+","xB4KPlmc4T0RYC68s+eyO0ULFz1bFIS93AkYPrpog70kgYE9Qa1Cvdm5sD2ZMEa9jaDrPeLIHz4KK+C9SePevUi+bT30bSq9t34iPlGsCT5VXVK9K2ckvakqqL29+Qw+ClbNu5sjcD6gHBo9C5VbPsBdLD3ujps9rneOPB2FOD7v4w4+hgXNPZtHez6XnUq9e00CPZ9xvz2L6Q+8m9NnPu4pJD4jqmM9k5VYPpQE0z2+usI9aEspvT/2ET4vJ2g+fkwePXp7oz3FrE0+8Fd6vX3vnzwvKnw9oFYxPdgtVT7FhwA+WwWcPb4b1j18AAw+bEHwO59aRD5dawo+ty33PaKiQD2ysv49kTJmPkIP1L2H4cM9tUZOvlzlVL4NjZc96xvIvEzwIz18xQq+oWYwPpHdYT5G5xA+PPaTvmN3lT01rd47YYTcPZcShb1Oe36+i87FPFhbij1yRXE9PLEFPWxpyzvHb5M+7iURvQYPij6hBFC+rRuGvIQpojz9KEE9rp3LPX7+Lr70vi49odxTPPFtSDxgM6q8ciwAvvVgsL0V9EK+FhPGPHP8bT26lCY61GpmvcZkHL5qnr49uhcTPlPSBT309N89d6slvc1leT5RTGa9C00/PXvnpj3/f0e9dBDZPETXTL2vlV498rEXPuTdfj2NCBW+8l+NvePVJ75lL3W92TtmvUWjur0e9+O8p7t1vmWYxr2rxlq9eWZxvfFSWj3kVM098W/SPTKGurxy2Ly9Yd6jPOZisj3Z1e09RJioPZ1lBjuogVK7BqRdPducgr4Murk9VxvTvXLWi7qEK0y9nMjVPfEya70LoSm+4n1SPpp5hbwzJsI8dZKyPTqrS736YAq9vMYaPbLQ8L3TDdA7cKeGPdy5Vz6qvYO9gdLgPcAJOj1yPd09D4tuvKLC0T3AsoE98xcPvg1QYT50YBc+VDuVvGp7/rzOuss9j2SuvTr42r2XLke9R7rbvNm3ijy2AIi9pkmjPX6YVz0gmPA98IC5vatFBL49Nec8","Sgb2PSNotjzBs/s9LAQBPvdLgz2+X00+WB+buuoNCz6R7bI+d5v4vCcHTb2f3489mqTEPWrBcz0sabo+aWQIPsemIz7yV2I+LrMmPpPmoD7Eyuc8mpozPnW30Dzqak8+PK1/vOkYCj7LaSM+rNmdPqxA1D1RuzC9x56ZPm1X2zzMJyA+Bm+OvnALlz0nm129JUbYvcGWQj0taso+unI2vTlDQT4spwE+GgdZPbhCgL1tzsU+mrQGvQcl5zyjKVq8cW9tPsXBvzxawBE+vrx1PtEp+Lu9tAw+C/BxPS1jlbzcCLE9F9nVvPSPsz0yWkW8Ffx8PhCV1DwxVLe9E9tSPgcfeD5VxhE+1ZxwOzPWNT7P+z4+FuuPPCS3BL72XJM+gvIkvCENlz5Cwvy9SIVLPt+ebb0NQOw9DJP5Pbmf373KRqU8nv1vPgLdTb1Wuh8+rLAnvUGoBz72h6k9MhJpPseAqj2zc4A8cnmnPRco9j0Rbhs+MtXlvtZIeT699hC+WcrWPclnDjpJAEK9R8f8vWecAj7XrAe9KAdHvV30ET6edAA+dE8JPSutXD3gk3m9JjICPuvApzyaELY92rRKPPWZuz3yFZ68LXZmPnlKV71R4xe8s+yoPU6NHD5PXH+8YBelPbMlujy6/tg9RXKtvQU0oj3+iJk+++vyPT3JYz6iuN+941YCPXODFj1gk1y+rAQ4vmHCJjts0Om9CsT9vU0uszx2Nt0+AhXPPYFCRD0Ftgi+zKKcvQ9VHb3XGyu+XXRwvQXASz1YTei9imggvTv+v70rDna9uYcavVXsdT6TRfQ9yruhPZwMaD0mAbs96z0VvkH1pD3GJcU9p14fPdyHtjzEuRs/1310vnQVl72qn/u9vHhOvl0sEb7Q+Ju+3m9ZPP1Uw7tBF9K97zA1PZ4qwL3jjYA9SAo9Ptxvdb5Am/q9vulcPmkXyr1W+YY9WLYGPuJNmD20KNi9pfG5PJSlR7yTm1o9GcpcPn1Z0T6z6o0+yJrcvpqPAL7h4lm+","/mdFvijgoj4y1dc90j1mvVvrZL1018W+punaPgVF2L3vFuO+s9VGPuyOkzrMmOy8iqYiPdUxpD2PRzE+QWy4vXvuYT7ySwS774YsPtzokr7gjqO9N/3xPWMiHr7aFTc8sL0QvpqjET7qExY+cbMUPo9QDD9k4fI8HFnKvs/Wyj4mb4e+vN0pvQn8dj7fiEq+tH+RvKYulb6DWoM+yehOvV1HxD5vXrg9MXQxPnKbDb14l9A9K/z/vE/dfb046wy+ObMwPoOsGT4n9iI+aIf1PdDhEb3Nb6s9nRwDPtrM7jxT1b69HqAevhTOYz6/giS+jam2vnzF+jslBDs+9xsyPqj5o70F9sU8QGmvPQb2DL4lr769lv6ZvSQEaT6NFV0+mbMfPpa9iz10CXM+oSKRvdJ257w3xpY93ZcWvgkqa749VK0+rhyevr/deD1b9ZS9LoOZPvu6MD5jxng8iFTuPTzdKb60yJ49braFvdZpwb2X6749h/a9PWjkQD4k3jU9yQ7BPfW4UT4yP088R57bPdyiID4BxuM6929EPlndAz7wLEc9C7uTPfCQHb4MFqA9Bc8BvkokqD70FDg+i9ZIPjjcdr79cU2+TxvBPXHBgLztNAW+tVcWPvD/tz5cULQ95S+KPdZHYj7+RQA+f6BNPmP7rL5sGkE+gM5kPdO/qb2Wok6+v3++PU6qUr7zwgs+/EawPRTOl7y3xp+9j+kgPlZLWj26NwI9aFKvvEDFnL0tqnU9UXZCvJWMpj1no5m+ddNYPso4Ar6De64+6L44Pd/O17ytUR8+nFRMvmD4kz1dHKG+YDtTPakQsT1TdEg+jg04vQ933L70jNC96+ZJPaKKZz7xOua+h+rnOnHHQD6kkrW9GlTYva/uiD4tOL09r5qMPF2r/TrsuLm9EAIYvD0LEr9Mv5M+4+W/PUthGbxs6WC9beGZvjJhnrxAnUA9DQsxvu8C/rzAAF4+3/BmvpLpcT591Ao+l3+nPTjysz6IP5O+e7DxOz9/rb1dgjW+","yDILPSDS0r0ngqq8s6bDvT+TRL4JxEQ+4BENPbStEDxMRjE+iWa/vZUFyr3u2Zy9KbEzvXHKDr5JK1c9DsIPP6r62L5H0n++HSgYvqJUELuKPjy9wghNvpeWjbwfMUs9Dy5Evim6wL0ZU649WPitvHN1IT7FTpU+iFIFPoq9Qb4K6G4+uWLFvarSpT6dYZm8ap9ivhDuk73NyFK+Kocgv+lvET7FdZQ8Bk16vV+Hoz22wFW9ZlpcvpY0zT3232G+UYw8PllCnD4BtLo9sFeXvGhVhD0plEQ93p11PkgA9jyv5am9bFvbvdIKDT5WaTq+La86vrW1sD1kKou8pREEPoKGir6vzpG9lX6LPP9D8Dwax0C+t5LSPfAg/73W62W+IfX+vVPZ670JLBY++aLDvnycyL09ZkU+kC+SvvOvzzxQ4zG9FVgYv8MRtb7Hl4W8K8lNvspYNjze7WC9+STAvhPl9z2x2aK9yJW5vFMU0Lx96eg9o8qKPiLKar2K45O+7sxUPS408j1REu49HvGQPpi8wj2fhaQ+mVX1PXFRVb1LzFi9N2PHvRoxWz4qX8k9AJ6pPdq3gzzk1r6+0gkRvmZNED7WW089G4MYPgwk5b1bMLq+Ru8XPgXt1z2NKIG+oTstvoFqgTzcECi+OMdTvum1U721gWu+4qkFvpq0IL2EqCo9rFHXvagF1T1OCz8+SlM3PogKxD0BfDY+QJPoParzaT66Kic+dTrivIYVNzyedbk95aRUvYB0mj0tVpc9vH2+PdQTPb1g5sc8+2UPvfZyiD2CAgs+4xI1PtK5Tb3xeqQ9+tqVPGSz+zxRY1e9r3lmPQJC7D3Pay0+mo9aPPRygz7DS2E+1zsIPv0hJj63wCs+09ZEPv4viD7iOq09AdWEPQ0inz1K7Kq76dNPPqHoNTzXzb0+UgXvPV6AIT7dd3u9guKzPEUk3r0JvIY9GupEPnqn3j00aa49vDuGPndVgLyA6Yq9PaSKPsUX2DtkKZE+iFQYPnbrKz5AjLw9","xJCsu8YM/r2jOMg8yLFzPd32cz5Ll8G6H3U+PBHQQT6Rw+89+oadu5qODj6gmH49sa6jN7K+sT3y9eC9WgWZPdiNHL183U4+78ebvRgKyz1jEBW8ai3/vIBEBz6GhbI9wdNLPq5MujzVWK+9Yy3VPeAlMT5a2tm9c8IKPrApnTxwNYc9os/2PbPsAz5Xzdo8wzMEPhdLiT71DOw9Z3JfPlsZGT1PIj88K0jDPeh9FT6HTXi9tcHCPTeZFz4brbM94TSBPcIE/r1xm/w8roPkPdJTUD6/vF09sY0suqKROD7c6YS7V5MGPmpTXD3eb/C939IWPjACAD6Dxy8+W+UCPg6Blr0GeIU9+LAAvegLsTxIn5a9qSg+vgjqnL2aCYs9OL+xPVhe9T3Kgti9W9aJvDDx+DzaXoU8Mj7ZvdCyibuMKLK9fgwHPleJBz48FYU9XVuhvdoJzz3GQgI+EYLwvFnN4Dy0W2W74r+vvA8hvz3LWJM96ZTMPAeklz1FY9u92L5APglwnD7xSqi9Kp8oPDgBN72beUG+6iNrvrWYWjzHtpw9PTzJvSK69z1yNxK+oNJKvq4bpb56sPi6yGTHPY/4ND3DGbK9NQa4vev7yL2XvhU+Hh6qPMhWgz7B12g9BtAKO1Xa8z3zsSk+ESyoOki9tj01NEu+R+a2vQHWAr7MkYK+rD2fPZIWTT3vBcs8W2nVvUBFDT21xZG96iK7PWQTijxGJGc83jbfvdWYgDzSMnI9X3YovEg5vr0ShxO9BEosvqHAC700IxK+FbxAvr+KRr1pprq9EF+mvK4MhD3G8gu9/zNZPUN+mr335I69BIIJPeCFGz6n3ii+zxBIvaMAxz0i6MU9UgyCvbssdDzgxZy9k2trPXqrSj5/0Co6bIojvvQtIL0GjTy8XqspPT+XLz3sxC89hogGvuN0oT3XF788Pl14vMd1CD0mjli7nZKqO2IAFL2YSJI7ya5SvasEEb7MW4q8HVd/vUq+6L1tZrU9P242PdKX8zwhHdk8","FcWcPdP0H71MA+y92H3IPVVJQz3sBqA9pHkdu5mYXz6lBjw9NcJZPlaR8DsJXoa7N9vbPcADij2u1Am+5VNOPD2Hkj7FUPk8tdQ6Pat8NDycSog9qoVjPkKKuD01jQA+k5vBPN0XCz2UVD4+wOHaPaqImD2SWUk9I4vWvVQJMD57Cc89JbgGPqTXD7wBKwA+hH6EPtIrYT76HFA+2IWKPlAP1j2MbhU+LfrMPW9KeT7mYaa9bLC4PgZ/bD7KXDw+EwMDPlhkD75U1gQ+W8XOOzNHYj6m8SM+B3jfPbxWhz6LgGK9eMhmvUofiD7XRXw+GmQWvXHrFj7yW8c6ZZ55PgowdD5DAw8+SqbkPOKBTr29FxM+CsCpPMGVnT1Qad49ickOPdpUOT0rSg89uqxcvORn/D2OAo49Vh/Lul01Jb1h6X0+hXwqPonoj72CgXE+RLjdvV/v0T0LDi6+gddgPYJmPT7QcmQ9Wo4NPp3SRzzlRGA+WlQkvuy6NT01rrE+xUv3PSQPzb3UDs+7p8YePt7GhD61Bjk907crPpAOVT7EaRc+EdWlPRGOnrxFmhQ+mUwPPDz8Xzwpquc9obE0PMTNnDwwy+q9r0yiPZSxZD1pvUA+lzOGvdj27byRH8M9upUSPgc/Q7xf8DE+cwCxPR9CTTvS83w9fRAFve9WbD5HEBW+icy6PTeuYLxcQXk7a3ScvutL0zw5RDe91EHkvfKIwL0TrgY9l7bKPV96PT1NSUi9/bEZPdmdwDx59RE+WFbFvpYUvDq8i4a+rgbpO1oKtzxy6cA9zEv5vfYwsL0YQ9a7L2trPa3hSTqjk2i+/nMIvhjiAD6llQC98GUCvgEvRj7baIu+EZoQvGRODb4zNom9kEwmvkreob0bAOS91MrKPTUMAr5jRbm9bJ8vvmbw/z3ElJ6+pcaePcXNYL6sDC+9AmSZvTGq8T0yyAi+heaBvScj9z1B8nY+NVANPu7gIz1tRhA+QG+pPgMsxbv9Pao7nVumvaGy2j12Z4W+","TN7UvZKTvzxSjWW8YMgAvvGuRr5OQh2+fmP9PRni0L3BIp25v4eovTeG+Txa7ys8RX64OzZpQb1zQJE9tQpKPt+GY77MJga9W2Auvun5P74uU5G+boZ/vb76+DqbPZ29lkhPvAony7ztVuu9t9QTPT0vnDuc1F89Thk7vkbpHT47Qgq+wLvHve89cT6laA4+bT4FvtQMdT2RTxg+Wum4PcEnnb3yO7+9GkUfPSo8ibv9aJ69opjPvFAn57vYJgE+Pn+YPS5v7b3eEty9NImTvFjI6r1Ht8Q9dC/jPdTYvLyUuNW9/DyQPQ0soz33N/682kMvvsyP1z1Jq+S81ovQvSGrMj0XeZi9xZi2PHeLqD0yfHQ+Pa4zPs2sNz7bUUQ+OspeO0/kVz5mgxI9zicmvXH1jL1LI1+9o/bQPH2z9z1IB8s9fFkQO1g7HLuU5IE8bylkPhJNMb3D2Q0+I9quPYBVQT2Hlp29BVkPPmZGlL0AiZM9Hx6Rve2eVr6A4689KPWCO930Dz6MJQM+9wsSPhFuJD1StJM+55baPRjpLD7KXFI8cTw0vUp2BT6jEge97TdLPZNedT4/tJo9ZEwvPSNm5D3oBwA+e/IYvibl+zysyQk+UVgyvR2zMz2zqlU+J815PRQHCD5QEnA+1D9tPnKH5D38Mqi97+mIPfpOUj6oUY+9vE/VPHfx4T3QvIw9BFYcPgTHKj7hOm29WVXqPCFbgT5Glh28aw53PAnvIr6umh8+TAopPgcOiz1YYbw9KZgePrU0Pj4MZnm8CACcPGnfoL11cOc9PY/FOymzXT5/KiU+3CAHPgM2QrzlaSA+PHIvPhMJMD10EKc8lxcCvW1Yk70LO8E97OexPd14jj0rIFw+zkBMPjpP4z1rxXg+a2rwPWcMnT1GNZw9TF2pPXb32j3fCDw8RTIePhrnFT2EvpG9PmFwPTaklb3LF/09sXTDPSsy3jyl6AE+kSFUPs36PD0ELd49wM1SvV9VfbzRnTk89a4ePuWayD1Khgw+","/tA2vUNWBT5+M/U9j/oLvbudmz1v+Ge+MipyveNxMb7jN2W9ALgsPUGmSTwATpo9DC+WvKb6+Dt+/a692WSPPXItB75X2ck9IOY+OmghZ71ngjS9fQeMvalvET5PS5q8EuVwPfW85jy5ULE++zdVPmq/dT1OHKA8Wq+rOhP6wr3mFus8VVqaPf+4EzvOilG9o1/Gvev6xD3QqGU9cLl9vI46nD1T2ge+VNLDPXr1rbzlV4u7LwZcvhFxcb1y5cy9IQoDvvr2UbyjSPY8qTH7vP9qpL1zpD+9NurpPR3BNz6ICda85vOYPYbTLD6yrgo9KxPDPb1iOTzpBUu8552TvXqQgL7VHCI8t1TGvQEFbT2xAKG9JlVlPT6up72QviK+O/kdPCECh7vNrGA9hHQavlrCDL6IDDS9gsnlvTfFt70yixC9Ee80vuAyPj3O6oK+mpZqva6xzLzWlQG+lRYcvSHkNbz1jEY8d9qXPfhsib1eekU+h5KEPO1PGL7BZZk9Bn/nvW3iCL1x6P29B2w7vjrJGr5c/VC5I4tSuwBhsr09Kdw9rlOPvhMO/7z2T4C9+m8FPttnyzyj24G+LONkPTv9vzw+AGs9fj3qPPmAKT0vVhO+rM7cPCv3Ob4SVIG+ioolvsqmrT3U8R29dQgivj+XJrwTOuo9nJ+ZPQVmw73+5Be+hu2hveV2hr59xdc8j60mvsZWg76yViY8E9JCPWujvL0sqA68fEL9vcCv9L2WccG9ZthJvXZcOr1mlG2+v7FIvrjqUj0MyTU96xcwvs2MT77CdH6+Op1vPRRjPr4GgdI6MJByvUScor0ejcu863oEvnh1Rr0xc4691mcwvsq4h77rocm+IE5yvgMpoz3t/su98C6hvd6ggb4Py0K+ZAxzvrTkQ76htSK+9lUWvnZChz03BgC+GCwhvmX6tr3SX3Q9HVuMvqYBRL1KNWC+pS1rvnqjKr3wltW95F6OvdJxBb6Z2DK+evp1vYU5Cr39NDq+i40EvrOVPr7Gzru9","vaaIPVdrn70bYWe+3/qouy97dL5ZfDK9MDQIPrUdljv/CFq9+USwPCxRkb0Jc189UNgFvqcqv70VOMg8fQWhPdjPm74xJqi9kpvLvaP7br6CL7s91Xqxu7ZCEb5NOU2+3sf2vUsweTz3ee69Xw9rvpq1p70kBd89TPxXviLhXDxxf9+8t2GqPbIw0r3/s408g9JXvnXLKjva9/E9OLGhvoyaJr1W+WC9WDDwPCcssb2LKLu8rHJJvVqPa775hSe+i8YLPssEQT2qQwI+5qRYvsO1Ib6Um9C7qizvPdUt571ST7C9pG4KvhfrJr6Y+069IT7ZvU/4yLzOR8m9BEOCvUZc2D038SU9KRw9vdppnz1az8Y94BWqPWyCKT44um+9ILx5vYhQ5L0NeGM+kN3qvI5Oxb06TlM9zY7ivF69JT0Aj6Q9xH0Ivub5ibnm+Rs+w4aCPdlCjTxVUsA9to7FvTiDFb7waRW9hfuyuh9g5b3FOWY9Y5ebPQ4JRTu5iUu9g0mqvS3/P72zhpE+bZkPPkToJ74xkxU+++KoPbuo1b08HgE+U0qjvfFS7L3+IAY+jtjKvRM0Sj61cVW8Q+k5PhGq6L0IiGQ+TQMGvm2J5z1Rcjs9zNKVPZ17Nrw3a8m9vNunORK8pj2Yaoq+szIJvmdg371YvZc9f9QNPqJZZ72HxnY8nxmevejKGDyBX689pBsVvJOewT1SzmE9EZrMPfJa4L21ZIG7oa0Zvnf46TuTcey9oD8oPWZt4T0CH8S9ccGvvV6VRz0AJ8k9ZxEFPiCK5z2y35A8b9W+PcZjID3wtbi9O8S/vSTRxrvXhU69JxqgPVs93L5xuNu93A9XvoOVLT51AIg8ojVMPij5Mb7O59e6GkkZPsEoS7w2WIk5aDFavVQODL0PxIO9VImLva0R0z2mhUA900+mvEs/Pb2VWZE9DpWyvEERYT7donI9G63jvHwAGD4JJF08VyxuPEK2ojwAn8E9PkB5PZ/JTD5hrB+7e9c1PZTTyL1GhoQ7","Sy3wPYmQRj5ClRG+fURFPjtVhD25/4Q+bIvqPai2vz3iuZU+AXilPBBusz397tA8J+s4PXs15j1IWYA+8HLkvCj5Djz0xoM9NvJ1PtN1jj7aRVk+zuKfPbzB9bwdWKY+0MAFPiaPqj1UBx0++R4wPlWXhj5VYCk+itzSPV11n7sgT44+zijVvaxZCT0fQcc9n7vLOyzTWj56L58+hJncPZehzD0XcxQ+RHRHPoSpCD45q34+oAkiPXZTgD2JPp88o8yrPS72Qz1Z3d89kymdPZCytz3qc0U+p1YsPXUTBj7ddgW9gsk7PvT/Bj36QyY+gPsBPr60qD2VygA7JMOZPYj4DT7wzaW9h2eGPdBJkr2v1mI+v3rzPSXnpz3ji6Q8IyUbPbCbPD5kHjY69JTMuzO+CjwUlB89fcxFPLOrzj3zy0k9MbgSPmMG4z3zvt897cwqveLMvT3qhBc+CN1WPvGSzz1TTh68pDwEuwNvIT3Dtew9wP02vpwg0j2dM8U9LAeRPQe2Nj3jnZg+TQ8/PrXZdz4ElY4+EnrZvPeMWj6OkSE+rzZtPvPuZz6v/Zc9+cNmva22PryXSVc+GXWbPAGutbrNeoY9RXF8PZLaLz1aaFq5rujVPTMf0T3Gmcs9UCZYPGyDED4qHpY9SnKlO+C3ir1TNWA9+ngEvdW8Fj5n5OK92PRkPghjfT0oYVm+/PTpvaE8dr75v4K+GG10vtca6T3QLfq8KIX/vWxkhD24U3q9eOW0PZ9Cxr0CJA6+eZqTvQUrKj1ZRy6+HleOvmp72L1oFsa9yiUavgnaJj4nKfk9y79FPn+3Cj662b09p93bvB4DhL0i+1E+yeboPMyixz3ZmIy9Xp9svktbAj7J2BG+6rlEvr579L2BuL+9j67RvUoyEj6xgN69mJxxvHZudL3EwRO+1TfWPPmqj75uxqk82HOlvD7tNb5GagA+zkD7PXK8BT57NgO9wrJYPk7zB75nHRG97RdrPmz8AT7/4l4+K0Jpvp6+YD1xZia+","+PnQPJVsBD76mqO7eNqlvZ8nibwCnEu+I/qqvOq5SbwAUwE+PemPPUlD5z1g/JM9kuKHPYC0wbwpDri9/qHevUeYDb4JfbC9LAZWvH2AeT0HS02+9Xp5PUO16b1HGpk9lDplPXeXk7zhlxK+CGDvvaGEPD64S7C8a3qrvSKxIzvdOKE8pAYJPoE407waD829gnIYvVT3qz1D7DE+Rif2PDtaeDxncKE9eQNNO4i+mj286Z49a0H2PWQNJj2NY/+8j4NHPVjwtTw3gVW9qv5VPfOSCr7+VJS7VdUNvoaTaL2RxFc9Iz2TvXmfZjzbF2U9i34Pvs4SpD2xmj29K8nvvHM5k70jWWi9StK2u/zCNzsy6sa9I/Xfvct3Jrya7Qo9N5x+vu0skL2xCnG+8wVKvq6G8by+NtS9vfjEPHRWiz3n6Dm9E+0NPcraz7160NW97A9tvnenvLwP6oi9kpHzvbRBx70ZrEy+RKnmvR9uiT0cZ+C8Kk0BvlAJvr19IhS9BgqJvn1Qkr7F22G+68TIuw47Q77O2DW+JiIGvhp4b76wUTA8FDGyPQeA3b2opoq7X3HDu+FtNL4yMG2+Cf4SvqxTL76fVVg9GwFPPaaKJz2Xqim+RotMvOJeyTy1zkS+DFaHPOeD8T3ZSB++1wdkPJoXnr7ND4u+ezAPvqdmoDud0469i0BQvcVjMD5jlJM8H2eSvlnmsT3Y1R+96WGgvkTFcjynAoG9083nvMYTRrxWO1W+t52Tvc8hur6D+oO9g5cxvvO+Nr4O7oW9PLN3vs+X3r1DogC+yZQYvpdVNr5kcPW9YLEmvs/BHj4KwgM9isFMvuhOBL4TaIS+BDTMvQr+dL06GsQ9jZ0IupKawrsw/Gu9IV2LvdlAsL0tBhi+Bp5WPflXIr69+eq81J2ZvX7BH715W+U9Q2tKvtwHI76/r3y+o0JfvG5kkbyVMv29+rggvkgO5L2X/N29u78XvapQOr2rN4K95ExNvvGFtD0piDW+dx+APaBipb0s+Tu+","S9LCPVqRFL2dkZk8pFGCvGE/nz6RsI+6nKwLvTLMUL3vdIA9DanivZckvz2684y8Z8H3PPGkxLyAPs28HnWEvVYQYD7UhlC+yrcrPfQdwLzHdLC87dgAPrfzPj6EwZY9xVfAPSPczz2aNiA9dbrNPX93ZzvKtQQ+8iyGPBnuIL1S/Y272m/6vinokD25ymi9850KPiTuIz79RFw+qK0mPmSmnD0Ypis9pqClPaBhbj5ouZ0+hWsQPa+FWbxb+Tc9RSCvvBo3HT73Y5a9eu0KPnAvKbwB3TO9EbMNvmGedr3wQpc8Yo8jvh2jkL6OpQY+M/SHPBw45r1lC5s8wXEFPvrC3T16ntm98HycvNOyJT4JUlI96jXgPfEhbL3oCQk+mwywPWTe7T2aI0+9faQfvZFJJr3V47i8uRVevBqiZb3YXgY+5/UGPVTzKj4lZEc+te2UPtmu1D3RmKS6Z/Qruyfnmb2NE6k9TMkTPqTlvL127sO9HP+dviXylrtrCoG9B98KvclXUz66jIo7e+wvOj5oTTzfJDa9NGWJuxHQmD44nRo+D9lJPrnZtz2t6Ik9GSCEPfgRxD3P7+U9MzdKvpvxGb4ZSyI6EwrKPYDPl7vlkYe9NwKLPEpzDj3Wwhs9pqNUPqV1jLyu/7k97eA0PvFfSD0xusU9Yy1Qvf9Orb0N1z68L6GUvRQQw70qZ2Q9OOEiPhyOkD6LHBM+IEqNvu8tLz6u5lC8MD1bPlfgRj78V8A852BLPUPaljyXaDk+A59VPvPXorxGpfS7BcGxPemX9T2ZPRc+PruyPWCQPz5mXOI7/cRJPU6YG7y++5K9MaKPPduQN72xUUw8dulqvrWc6bwgQ4g+teYiPpHZsD2NBhA6jvZ/PjxaiD3enkY9FiJgPGFWoj1q4kW9t+x7PYStprxnNSs94KgUPiybAT4tvvs8sFTAuyeMhL3EdKA8KL9MPUXlU73pBEI+QdMQvNZNUT42HyI8uUcRPmgIhj6xt/M+fI1BPpbQ9D38nIM5","jJYUPtn52T2cQwy7dGcdvjc5aT1PpZo+a/GiPQ2WZL5tMEc+4tsHvr5dt72Z7CW+hgszPpFR3D1JfNk9h/MBPjJKAT8V7ig+lKI3PZyLAz5dx3C9yviDPrwG3D3E7ws+bnJOPmjrWj6vjhC9zhT9PEfCAr0nMNi95kKiPRePzD6CvF68+BMHvtJoPT31lH29QZACPbociD2F6qE94kiSPW4T7D1WgDE+AjzFPKZ39j399fE9S3syvSBR0T3C9w29XwOnPXyejr3R3Mu8lrEbPSDvJD1rz6W98vc9PlvQMT4eyTQ+nLO8vIYXv70JxLa9VqIEPzaDqj3LYyA+jODmPf8Pgr24M3k9AU1hvfoYRL0RWQ49pKFTvgN8sLx6OYU+2UnGPSP+VD208fW96QoyvWWXgL3YBi8+K9NTvq7yu70t0m6+LxmfPtVgprybEdi7YV0fvohdRrsgsci9U8MrPemClT2pxzs+lelUPh6WlT0FkrC9jF2ZvXSFUT2JX3q+47zkvebqSL2zJaS9VV5wO7CI3D3ZUNy9aMQtPVP6+zxjJYo9ENkKPu9KyD0Q4ps9Vx4GvhLSRT2HpIY95UNLPvito70QPPQ9hqoVvsO8dj3Icg0+Sj9GvdDO2j2DG8Y9TqnavIkZmz2Lure7b4O5PZfyFj2qQoW9ULbKvUc+yzz3tre9/JlJPgShRD0mjQ09CPa1vWBBaz3Reky9oCg3viiz6rygyAa+8IiCPfYVKr7FnjW+gR64vBE6NL19Bqi9/GPAvV4rQ70/bl49WH68ve6jQb3aBSO9BE6vPB8RTb0t0o69R18lPsSXWr1u3IG9pPX8PVkJbL4T4WW+w6hCPvvvgL1AGOc9oP0pPn+8G74CJji8+GJ/vmXK3z2x2Xy+C9GgPZxpMbtftfU9ekbIvQb2ST3dRI494lAzvrL8ob2rXa8+XhSrPejQ9D11xwu+noIkO0jZFz2uMvC9v0h/vtSnBbwmgks+eMxhvVsux70VeCa9AA6KPfivyD2F9ig+","uaKqvr+KML0DRy+8I0kVPT9+pr0xBNi9BmNfvgZGqr6Uk8O9zeIavCyQUT3LLZ89LMaMvbqcw73SAhW+/JcHPiBJJr3CbTO9mhDcvAi7Er6Bgkw80/+5PD4cnD0U3L29D80OvqDbkL3OOLc91xexvlLkor0obXI8FFLhPRa1FD2xafa8dL+pPFEjzT0pA9Y9qm1dPAwai70QX+K9o6+dPTsw4jtitaS9McoGvtgm1jwxZC8++XyFvtm8Wb4bjB++x/tCvmHapL7yk5G9q8QXvpYZgL0+tSu9bcyNPYYOoLzd6R6+5MkrPXyyPjlkhIA+NKjlPIDRdT1Cgke8slVuvR1QOr7H9+q9t7+RuKlJZ713PYE9fc78vVloKb6LX58+WubfvQ7omjyxkfq9GNCQvQWkjT3aqTA90M6BvT14ez3YrMq9Mu3yPTjsir2zJkw9bYsSvo4EBj5Fs70+Bnhivlg95bxrJ1Y9qUV+vaDMRr68cbe85EgBvanrar1W/E4+QnPBPfwIV74f8cM+Gq22PHcPjDzwkFw+eZjePcpoEz4n+3S9PhYTPpbrXD68jmS9MMRCvsEmCj4nK9Q84R9dvO8bQr00xia+zJ0hvntOBz239oU9l4cWPnhior3lNb49it2EPsyO7z1q6Rk+e6obvT2v8j7q4dC9tD3YPW2cGztB39k9yu/Duigs8D2ydI89crF5PQEVBj1pEVA9A+rKvJyKVj6AEBu97Y+9vd85sr0vyzM9BDSbvZGdDr7p1vQ9npbiPe3e9T0K+1O+uS4YPqDWILv/XEs+Ei70vdKZhzzkqki+vf6bvXrrarztfRi+6v4+vrzumLwby2Y+Ew2JPp9T3b3juEs+40rOvqHPBL1UK5Q+/rArPt4bOj6eLtQ8RGNGO53Erz1HBs29Z49evVlolL1pomQ9bXUFvV0W4j3o3Pg9piyBPqxQRrzwB269YpxmPgmA0z3w/zY+V9QdvtMdfj1layI+d/7GvUMl9b1MS5C9tShHPurCy73AqO+7","J2qwvjmAlz2vzAS+kUf3vQqOY75bfEO8TKa4vQwYl75iGzC+G3QDvrxcHjsD+pK+OsaePVOQTDvz21a+3ouGvZKcRr7hzQS+Bg+8vQxaEb675D++V0u6PYCkgL4zLIq+43iOPMUziT0bRHo8q9LDvFB+3z1CHNo8J2nYve8PJD4/q8+8tAnbvVqZZjwTSDi+f60bPbwJrr3R85U9c0Mxvm+rYb4B9Bu+JfgfvGqqKL4DEgu8S21KviDtcr3EEAU+wTmSvQ5QWz5tqD2+sxg/vueEIr2zDKq9MVkRvk4Jx74qgjO+i5rsPe9vUL4Q2Ji+wUwivn//7D1e+yM+GdUhPhh51bxWVb493JFfvug1Ob5OAlO+nkKqPIrnjz3BYOq9QcyBvVmaT75TvwW+h/mhvSCVBr64JzC+E+FyvPV/tb0Pgg68EfagvTBb/bzKBja9MpvovQl9Fb5Ze/m9Xyhpvv1P3r0s4FE70erPvf3vJD4ZhFy+dMRwvnmnLL4rXxy+A8pevVetar032W6+53ZSPbcxKL7L+Le8p3TxvNbhar2l8Hy+pa6KvRc4cL3wBTw802sVvkBeMr0sQDm+0abGveZK9b0D3pu6vDsfvoHNtb1C5UE8RQlVvjTPnjyW+H6+ExH3veV6pr3RZpO9mnRVPSsnQL57ZIW+ImMOueAJhb1oPhK9M9SQvU5q8L1owxk9TDs3vgMm+b2Abhy9BT99vtsOGr27LV6+Fs0cPAAVCz6FSkC+8A0ePRJP7j2XCuM7tKBsvoOHEjxh5di90Go9vvThYb3AXiq+I5XvvV7JFD1dhFi+hBWlPQqGhb3/4OM8ztmpvBNRn71o/zq9wl0uvmdK6732+ru87gsgvjy7gL3l9jm+NgsXvv0v/r0iDzm+GeUCvne9Lz2dSYM873Opvdik8bw7CNo9JeDuuzDSBb7RvSy7cHSOvdnSi73vO289t0KrvThK5L3oudu9uVwxvs+YQ733sJi+ZhZWvpkJzT3m6iQ9MJHkPOzm871Dv0G+","lg9GPofp+bvcjka+zOAXPW8QRj78Ci49ASiTPacS5zyNNm+9VjrzvZkk0T3e/n49N8GGvY/kOL3+oDC+TK3avIOyDb1oZ4A9ZwesPqrDv72SOZS9zi7HPAWHOj3qm8o8XeHOPHjS3TxbvMU9nJTIPdjGYLw/14u9P1p8vYfkij2CBgG+0kYnvjZaXD76IC49U5LrPembhT7nD0U+YGvDvXk+r72qOVi9Q8bgvVnWMD5bYge8LxcrvaXDRL6C8kU+8kynPXQDdz2T+Qk841rHvdV6bL6GAdy8v+r2vfgRyb1wE+M8q5cJvn5wx705jwe9I9a7PYZh2LwmbVW9hoQtPq00Zj7JbZg9x1fju1FBnr1sSA8+QxcjvjKn8z1XZwA+sxqWPe7c9b1ydzk9I/QKPtqwtDuBKaC9sX5wPXVnPT3Cvye76lsavI/oM72qY00+W2+bPhXv370gVgy8sNNwvYpIu73oSIy9oFgtPtzLUjsDjUO+LYbfOhQWNjzf3t89j9rnvcEzzT1Dgac8C/36PQ9vxLxfy6c9i4mdvSKEcz6gtkC68FsHPW3oGr4DDqK9yUQKvUp6mb3bFKI9V1LHPH2dIToHz0C98CpVPdhCAL3hSTW8EV+9PbqHL72hmQg+ngFTPRQpEL5mZoG9zNIrPp1Dr70YuBG+ShSPvV6sjD3JIjM+MfE0PnCJVT7lS4M+TJByPbVRNL0kJDA+ui/9PbbRHz3/bpQ42cWvPXN+OT7FER69nz0PPiCiDb6SzbI+5lK7PdQp3b3PCEE93kukPWZgQD5x/2M9tYApPQ7Wzj2iaIi8h+0vvRSmxTu8Vhm+qpSNPqPblb63u6Y8KpgPPsvlWT0ev4E+39gCPGuIezteEoo+9GOqPfXYF757P6c9mU75PTS3+j3bNZU9j4fBPQ/kwj16JDQ+54NLPk9wFD6VgdU8hY2OPg3Yn72JylU+quCIPt7zIz5FvnM8yRcPvqOQgL1cwK49lPuoPgjhZj5BK789MKo+PcpnaD7LgDM+","KBKqPgnMCz7E4mW8kfSbPROhxj1/QJE+uFANPufKKj4efU8+6aDZvNnf0r2lgjG+UNQYPjc6Cb1bmjM9+7OSPaM9mz41a6A9mWqZPLS4CT68VCQ+lfbjPVjtiz2sEYI+6//Avaeooj1jvCG9st+WPOyTOT5LJ1y+lDNfPe4SxbyvxRk+kNCCvXrFij1pwRY+I9uqPdbXorydkTq+oUhUPdb1Fz4hhto9A6+fvPQ1Mz4Extc+RDz2vbdPdT5tCBE+aMb6PH107T2pJAm+ZZKMvNm+g7yQ0KG81kSBPtSzHj5WZDW95YkjPvUFCT7RJg++9ckDPjO9UT37lwE+H+4auvOGZb6h1qo9fLYGP/mVBb5Dhkw+CFQAPdYK/by6zc09DFqnvWlzAT6zuSi88gRnPgpRmz47Hhu9jcchPj3VRL7BWNa96CvIPb/JCb/EsTy+06zrPJHJ5bx0m3i+N5NvPpXLb75xeNk9utEYvntjYrzgZoy+AU+cPXIeqT2Suoc8UZZ5PaNQTD5oZh29SOnwvVxngb0zboY+myD1vu2hcD7gDlG+qhIqPbWVoz7LZ0a+n4T4vTCH3jv3fFS8gp+APuDSfz3hdgu8uyOfOxGZID3WJWk+LKuEPhHIWD5I2Te+/2sZvhZC0b3Iep697aVmvfcfo73s11g90oAlPaikbr5gz509sDSEvJna2butsbi+C6PsPV0uTz4XLCa+QOJHPnTV4T08Bpq+cygzvgQzYb6p3lw9aprevd9zzT2lvMW9ygF8PJUISD7Xsm0+8P6APWxdtzvq+DA9GouYPSn1HT6fyUQ+rjdSPW+rvTzSKjW+O/ANvdUQ7T17kpw8SGB6PcAitT6MO5A+jJbqvtNOC75sE4A8p7nEvkuo7D3cz5U8HC6EPkKd4r1WPsM8WDTIvVAM/bxkd8c9O8ClO/mCJrzs/B09ulG7vX4vzb1ZfXs+FL+fPp53xL3Zmps9vwM6PiHO0D0KCoo94cgyv9niwz0b+To9Lj4hPhcCCT6QSgm+","MEfnPbQ8AL7djl29AW8+vvqPO74GW2G+9+tHvrAjcL56Era884gTvv8/NL7lDzS+EXsSvlMoRr5qzTi+rAJKPhdyYr75ysQ9pHqHvmOMN70ytQA9uudgvoByLL31RVm+12qXvtJYFb4rhog9OsUavl3jLL4v6qw9I+c0veCPUr4WKZC+mV3bvUOSLb7TwA49rEiavK+/nr7u9T6+upeqvneUi73HgvK9wb7zvX7KxL2urS2+9QFKPdtxQL5AwWe+8O+rvXM0xL7qvJK+o2wOvh8CGr5rahC9PkkmvVM3dr7uHyW9M4tbvtzYKL4cSgw9T04MvnQ0lb4J0TG+Fb2ivj280z01lqm+Un4tvdgcjDz9g6i9jumMvtuDEb6fJ1W+ihaZvTIe/z1o8/C+FEqDO7uGSb4aoja+jo0pPkir37wvGDi95UMmPoS7jr3nBoK+RisJP/IA9r5Qk9o9319FvnQ7u73wXNW9CJMYvAemJr6clIO9Om0VPTU8Zr5f9CA+JJdPvlZ/0D12YIK+V/6WPoH6a723vO6+F2hNPmgHzL1zFt69wuslPekFBr3kPiy+P/+BvtgPu7zdLCY92yaMvJBKCz0efeK8B5imvF3Clb6licK8xdU1vt+zcr3y3y6+UUYwvHGCCr7VXLQ98d9kvLSQybz956o8p2xIvuJH+b73zBs9lAKyNudLLj4iXlA79L6IPkySBz75xxw9R6lqPu/D8rxii02+Tf0RvRyhqLyrszO+4KqdvdKVuz2frMI+0AryPdMjDL6IluS9hB3ePTNyUL2iKAY/y2xcPnU14r2ivX2+C/JCvawhWr7sS+68E3/IPX3rIr3Y/6K9WwHmvVzUGL62bZu978o0PVjtPj3t+g8+qoGyPp/gnrwSRq0+XwCDvW0ihLz/Xd698x1XPtc5Er3eq22+BvgnvtmQZj60MYO+mwGNPJZEqz23EOo9StjZPHLLSj1quQY+KkAyvoipfz6abYw+73FJvnxA6L6US6C+f2yOPn/eIb740sk9","VTsLvt+bwz0vLyG+Uqa0vvyJM77ZRNw+AirKvj4zFD7ur5k+i7ogvooTd736SUK9bXjdvlX/j70XSwK+t9CdPUktGr5PMay90eoxPg+sc760JlC9J8STvjc39b7uDm8+9ZeDPoon8b03c3G9wXoWvC2LIr77b4Q8Hf4OPuBk370rAAm/9IIlv+oRvz6vawG+Mx3gvUWTgr6BpC693YASv2pnFz4SfaU7+4EgPkXVPb43GHK+zDuHPpqZNT2ENcA8u/X6vUXWjr3IiVu+zaVYvuEyKT6AxeG+Cw/5vq5chL5V1n8+UYBxvWy9Qr/hssC9ynjKvju8oj32Lvm8m9Y4PftzSj0U5Dm+293UvW3uCD1ps9G9chm/vd3VBD0DyAy+xgrpvVyEGb62Zh++4SJcvmnKDb5sKAW+6DALPnUZijv3oMY9YNLgvYWkGr4T1Pk9XqUgvrT3RT1I2m2+pkFUvsDJFr5IrQQ9GDADvkUAOr7MCQ882gX+vZq7AD57/A+/U9bwPXxTfL5h+pU9htwzvvIFc75Erja9WUdhvuXRMr7MLlC+IpQnvq68F71kiWI8DrUpvtBRob4Efgw9KdBdvUimYb41mfq9wRkjPqwvNb7av4K81Y+LvW1RADy65f07R94/PQ8cazyiZ1W+H9XvPaqilr592jw9jBd1vcr4SL5FBlO97tMZvl79C74G5Bc+T1RQPV2tt74o7K89Dru0vf4ChL1+Gc29OG74vRsx37zHyZ49EqegvVAIED6aVmG9QHEUvl7boL0zCV2+8V4Jve5C3zwz1mE9Z8AHvhtEhjxEvZC99V5uPcMb/zy09Zy9pvNEvqdRGj2d/Sa+uwfNPFq7yj3YyLk6g8OKvBhNBb7QHDI9VJYgvnDJ0b0eo4O+0zMvvr7CW766V2Y8DcOvvBpotL4AECu+l/MgPTnyV7zgBRm9Spk0PTdvdD7vHQU93sLavdNaGz3zcv49el/ePBX7Cj7oywy+dUV5PbN+bj49NR++Fw3mvIB7HL5lRPM8","fUyluiFmSr6PKUu9OK9vPiSYjD1Oo5W9vrZIvasZgT2OOEa+cQGgvqoMIz45hb29+rcbPPBsBb4sqx0+hjBiPrBYFr7/Y2q+kmNKPv2CID7vhq499ImcveZGjz4bhM+915M8vZfxKr6/A1O+eUC9vRgHiz0jsbM9eVQRvijUTT2WB2I9dqTYvSJLj70ocqc9VKAJvhBtC76+NC0+/ytQvTvqJD6UuQS+Fpt/vqh3iD5zHrw8b+23PkC9Ej6cOIo8EQBZPiERiT3Svxy+pH3wvgFFJb772KO+Glxrvh0i+T09HQo+zLHQvToCMb6+1hU9bKqEvitQDD03FYU9FyvUPaAklj6zpX+93XIcvuEURb3YXgA+8rlqPS04er1IDGs9OTsjPtVn5L3ERpy+3qH8PW5YcL6POYI9Wak+vVUFrL1VijS9gsk9vGfq77smX4g+zrY/PnjeZb56Epm9bT7tvXhT9Dx97gK+9ZKUPiHuIT0bx+i+rrGiOhc9UL6Zd/y9QZBhvBcR0jt4e9K9fnCjPXKXCj6u+oi8h2HwvQCPQj55DJC9CB5fPUhIVD2j7948VgbwvPhRPbwcwiU+Zi2UPcqOrD1eGgG9WTgUPbsyHb4OiEa+eSLdvbUTXT4dPIo9ftUUPtFc5rwfSz09c5yFPtXPNj2ZAzK9rpopvWslizvoZoI+slXhvaQ0wb29p48+KYNnvZkeSr09zUk+7Ph/PIyPhb2G4h29wjmLPhKyrrvdwt49sJzEPbBlFb4XmWA+YyyRvnjVcL7EkN09TrlSvAAcLj5ZszK+e4ZhvSj9rj5tYVu9ihIzvU7VLb4rQYe9D9MfPoWsK795UA++kr0tPp9AyDzm5Kk7JDvavdhKTD5Iqqo+u/8pvSb8kDx8IGo97EatPdYscb7Uk/69YIjNPSAJPz6uT7Q+wUdfvdXR2T3TxWm9dz69PkncSb7jOgk/IerPPWmlFz6QKjs9qr2Avpo/Mz7Ai0a9OhusPtOZhT3xSHw+Z2r9vb2w5j2h3yo+","JdtePhg4j75oWCE+b5J8PlYNQb3i40g+12+IPbbhuj0afDI+GHS2vMU4qz7SJIO9BUyKves2/bz3gY09tIsdPbHEzT7yCHG+bbhkPmiP3L1XEC68IOdzPlZ7VL1T4dw9vtXivMg/Ir6K9Iu7mE0ovr9/hD2P+S2+wZvbPHu6Bj7hqdI9RR+qPST9Tj4vPog+e2dWPRpOmT46Ywi+BtgBvAb5CbyzCCy9i+z5PWbVgj5RXSo/12EMP86aDD7KyjS9cKqgvs381D0AjwG+JJgtP3k/kz0r3Bs99/T+PS4Nbj2DYCU+q0qIvueuDT5indM8RWxEPX0v+z1F5x0+FUq8PeulHr7Ba7s8tlhFPtZUwL3myj4+jDYHPu5bzTtj9s6+ViMBPpeyeb7upL++60JJPjgSUD01hlM8z+K2Pevipb2Se6A+CFZ/vtMO675GUh29awd6PRO4EL3ji6C+rWZ4PiyuWr7Y2QY8MEuQvoTlAD4/c76+jqLRPg4U/DzkMIK+ibYhvuT4Zr7VxzQ+x5Upvt6oZL52g5c8TTrVvl5JYT7GB5i9zpElvnjK3D5RQfO+QR7cPPGbgL49nl29HY9LPWKOpz14vVS+g47UPVffcz4fBiI+PmmvPllhaD6RQie+QxkAvjODrr0Tu4o+r5OVPjpSDz6Qnhe+42yPPgTRE76TAZk+ImCAPsugWz4sEAc/Yv8XvpGQob2K7Io9tzQAPzpzWzxPdiM/vLP1PYeuUD6IqXM+K+AePY/OZb2lopG+82wdPyWSBr8gdOk+Ky0wPtqY0T1THWk/rq3VPu5iQT+ttJA92siTPhPv6T00/J+8T/yUv3cwbz3/8gW+ayvQPqiAa74Aag2+ZBW6PJlYrb7MzGa+LtdZvpiGb70+ZC09bejXPh/Vxj4XG6g+Ut2YPo2L/b0ezs89J9bKPgfivb72QxI9g73lvhUsXz5oje8+DBUuP1dhFj8Gi5A8CIW7Plh7Fr9zgya9B4lLPsZCCT6V1by+0RTKvlxRrT6dvpS+","lIGGPUx5Pr3E+lC8wK/BvNxSgD6VveI9wFfLPoB2CD6YnTc+csA0PrMfDD51tHo7fdwqPZsWlzzpXLo+Dh9jPj1o0j0cdgg+uiH+PaLQF72ol1M90su5PWN23D2HH1W91avrPZVSuL3qVgk9jdLCPSS8Iz7HorA6RAyLvUT8VL1OgN49m2/kPnPlTj5VZgO8braMPYHIlz0bGQk++uDCPkUjgz2KeO09UfcMPqvopT1P7LQ9N/6bPlzmlT5TpNo+GQTIPW6XHj55mEM+UoGnvPikOz6dTJW86xM6vjnDqD7mOXe6x4ITvar+qT4JHey8E+OlPRl9frwN2Sg9UTmHvXjPJDsJ+jM9AO7fvd2ZCr0Clos+tXffvGJO6T0xlek9WrNbPlP0eD4aLY29Xe3BPfluST4yp4w9YlyuPsgasbueKaE9SCbgPUdEfD5yiEM+CpVNvVEvvr1xkQu+Lc0BPlxL9ryGtui9GvPDPehXhT7syT0+BWGFPcs4ZzoUa5M++p/uu13MDzmKemE+OcPwvB+/87t1gCM+YpNhPnWnyT2tY0I+hW0+PpLKAT5bgJw9td3qvfdkgT1YfdU8EqE+Pin8b7wRZvG97WEMPlT6n70P9Yg9tfM7vawTkr2L7wE+rd6yPesZPz3c/48+5R09PX+fhD5m3E8+byHBPuyLeTv9JQG9hQQ7vdLRD72Odfg9WhfwvojNWb04zvE9uOw/vkuTFr7u+y28QUbIvZvkob2AAl09D0AOPv0xgj4qxrK85UiSvfmkPj7qR4K9a+CoPXzyKTwXfuo93CrlvUd97rwG97U9uTH9vA2S3T0ybNq9YZYmPlMQnT1/2iu9cnjlPJ/PNj6rDZY82u+XO/sNmrzLPhe+W/GCvr5SAr44uSE+gZxYPtjUKj3rrKg96mI2vnrF1r0DkG++OHSOPsih0b7UyVC+N+uMOqqXfz0T8WA9J1ULvsBADr4ipbY9dZpYPgdstr3kdBQ9IsKbPnvoQD2J1NY7VdH7vQsJoj5wBM+9","4xNMvIogy7ynxIy9iY8wvlQaFL696Ds83OC/vZ5B9r1m82m+JT0QvjNmdL7SepG9Dh+CvgJ77DvWhmc9bHqTPqsiM76OiJy9zV5PPDnlnb5ljoK+GjzBPYxxYD0eTza+KoCJPUW34b2uRGE87EqcvAHyHD69sEK9JQLBvWBhCz6aNjG9ZZ2cvSBDljziGRC+DbrPvHriCb4vdyM+H8NVvd28Mr4TySy+KNICvvFMCj5yYpw9X+sAvcrGqTwy8dS8VGdvPs9Tqr2BASe9+aUiPkJcq73WvYu8MelRO8L8gr40W6+9bm9EPg0Hcj2nJrq9iUVKvR6XEb5D+Be+qJbOPZefr712Jl89hwppPkKuXz4RcBw+cJ5xPrxOuj1tuoc+HymhPo63NbyOnOc9DUd2Pgl8nz10GW4+DBaCPpvbAb5E+s09qkrpPSwPHT6frn4+Yye+PSghjz0E75A9hIcgPmpkYz43j7I85z+pvbA5Bj6mVSY+T9OSPG2NWz2p+DA+EXuJPf1nqz3VzGk+pc8xvcoIwT0i6pk+VxiAPZ6VXr35niI+dvd3uUjnvT0Dq188uYg7PpUyLT5eKyk+OlTSPZGt+z3hOI092ghIvWDuZz4bs4c+XYOIu5yWaboULYM+OQ/dPZasaD3jl649TXp4Pf6VWT03ZSc+tfsPPs93bz71hY09akDvvN3hbzwplhY+MrlYPkpfiT0mRI49mpwZPlgcgj7NizE+z2vcPesYcT5sa4q82VNfPn9r0T1vZMy9xut0PFzUTb33cvE7d71nPl2mjz4qNi09UdZIPqoO1T3h1Ic+XSBhvTmxKL1q3bK8DWeMPQ55PD4eUuo89KO1vD9S3z0E8n4+pdK9PdzPnj00b5E+7NR0PpNkGjxzMQU+49akPHs7Cz5F7QA+UtUwPgZVwz3QGVc6vRXhPYMLlD5vagY+eP2NPtGVED6YiaM9OKrWPTECmj15ziG8cghTPtZvLrxdeNc9U7CUPduoYjtxG4g9dQIgPkaiyD3dO+g9","wMGjPQIVTD0hdqc9vuh0vti6hL291lu+ifmMvtAwI75TwWg++w4IPp4NZL4mY6k91RWEvQ3pYj7wJGy+49gmvh7ugr3Z6Bo+FgMovnNxlb6S1ii9ZVdcvpRVzT3CGx8+D+cHPpDmrz5kkvC9q6+YPoHsQL6LFQa9VpISPvyN6T2pwOQ9+0+BPPmYRL0H3Bg8lxveuwqxHb5xHhS9CoGrvM9ckL5yD+U9sisrvRS3jTwE6BQ+U+CQvmdOhz3sZmW95H31vb1sPL5HlkO+KcktPgFaHz5sA/E9TeDcPJzC5zyVxvm9xXANvrJNwj3wu3S9EZudPg8PLr6KucE9ij/zvGxfQz5lLCM9IcBFPjQS0T1eRAC+wW9yPqrRzT7ZGmI+kACmvcZ/Ib4iuUe9S6BLPrJygrsLOjw9U5ruPRPxorx8O7i7y3bZPNjGQD36ksI9o0bCPQbDBL6MdTS+0amTPfkJ77yAoO89zq2yvhNrEj02QVK9ql9VPTiOgT7AAq899XeSPVjYSr3YjyG+Bq7fPWwiKj5l8Fe9Ff4YPOwHqr0CEUU96GniuoFgm71TblA+0qYMPjLGGb2ll8Y9mRwnPm+xiLzgoRy+5SbYPO6OAz105hc91wvOvAY3ZL1u4cM8jpZFPv0TWD4jPM083bOSvXV/FT5jSSg9KbLKPRBox7zcj509CEevvZRf67xYpWa+PoYTPahP470syDa+lpQUvA8GML6O2KW92zM4vVU2Fr597Ue+/kN8vrwRDT6L9Eu+a9MMvsJSDj7iIO6904akvXEVdr5ECgW+I17jPZdAs71PkY27eFeluxXRUj0k8rU9UuMFPbOs3D3TDA6+6FgzvVjeHL1H0T89cG8jvvhlkD3dhk++WatfPbPUR70tXBo9CRMSvvaY+732cNq9EnVPvtnhSD4eeji9n5CJvUvZub23Vhu+fM+euySw8zvvxg6+u2gdviYy5jzu+l28gUdAPnOv/jxnnWu95fDgvWZGP72PSva9qWEaPAPDX77rITe+","f2kPvrJa+bzNEwM9nlJDvoWQ87vi1+08t67nvfk9iz2ol6o7ykAZvuFhCr2OHI29kYxrPDLrJb4Fwom9SxqVPKOyTb4sugg+hcgMPZxauL0AXhW+ZRF8PSaW6L2uIo49mxiZvbarSbyF+O69vdKqvU5RbL4zfRM+tX1hvmfjg74nDDm+7+H8PSoOV76bNJI9pM/Ju2eIzDxjkQK+lWZ7vv/BUb4elTC+nBP4PAJHRr4kdYY945RuvdBFab5s0/C9zqMDPgeh1L1jayY+/06DPL82DjxM7wO+qMbovQasO74msyA9a6r9vXwGJL6uDdK9PX4nPlg+IjykfRm+f441vRw4bbznGLa+TAzxvth2jj5rgzi9jdjpPbwHIz4xqAm+gRrOvnW6uT1ioGq9ad6Fu4giCL03ft69fqsqPoihiT0ahxw+lle+vS1r7T5Qdlc+fRFDPf3pcT3eyac9gOL3vMz5XDx8oli9tE5zvPojLr4UJys+UG+XPWo3Gb5snhq+4+rzPRc0Oj52ikY8WievPT/OTr1Jnxy+GyDrvfzGb75Lx2k+CoXCvbJVczyKGzE+EfK9vWaNoL39ktu7+N2/vJlyhb1C57A9PojgPRJBs77ksna+Eb0evZueCL625OI9LYy7Pa1eLr2fkfw8Y1Gpve/LU75KXgM9un8Rvs+4ID5PTw49Ysiiu+Qsfj0BjKq9I106Pic7ZT7GBM69ndYCPlA0Iz5nUJS9CWSPvZWo8L086Rq9h6jQPQzbIT6e7XW9cdGbPeyOKj57Z4e9vASNOyY7ijyKoJk9astfPDzFET0JeG+8U/rbvRMnD74RuqS9+8FbPEjAkjwLtK08eZ6YvSNRaT4XNUo9XkhJvHubkT2CBIk9iIXkPAVLKb565zc+IEBqPQ9nBr3jFT094xA8PouDij1DCEq+pZjGPQxSIT7ZGwc+KmatveNa1j0F98y8hpGku/hwLD1m6rU96Sn/PQO9mT1C8OS9/XtlvW75i72ouTU+nyS1PYLp2D3zrI67","CWupPDH24b0sU8Y8MeIJPooTD7yIsGQ+fohAPox4Rj0A9ek9lzdkPgnBRT2znLs9iJauuyrmtz0dPwI94aYxPvNKGT7CJ4Q+UqXdvNL8XL1R1fo9QPVAPnIX3zwa3cU8evsePpuT9jx1+m8+B78pPvyt7Lx0Dv88NqmxPVh4vL1sIBk+rA7yPaUWKj7jpAM+zYHfPTZpDz7TdXU+uHGvPvhKJz7q7da6yW3vvMNRtzwGkpg+RDgxPtWSXj56h4M+QP8IPlfqlL3rxyU9aOCePbmYUL2Uxsa8Bt8cPJrHSj6IPjc+D/7BPVw+MT3MjdQ7buqZPRbvBD5lId48cLUVPsmfBzzTY6A9fCutPe8FIb2sxB8+v5OsPdMkCz6KzYO8E2c5PjQ8iD6qwq29C8tAvSqNPT3ztZQ9WmWRveptg7w6jBW+zCr6uxm6tLxtmAg+vDyvvTOG9rwShS+8MYdOPVUemz0VI/A9D+YkPeaFLz6pQzs+AWjcvUSAQ7shoIw9QGbxvBKS4D0+C6k9u4IwvJ5BtT1wyOM84VzWPYteHT7up4S7pdIBPgehvD0kqag6dWoFvrRPxzwbCjA+7BySPWtGJ73CtM29cBhWPe/A4DwTPKg9glr9PUShg70UKl4+eP+ovf3rob3T9089WiKzPQ/7nT7eWS89ibqPOsjKkj2yKfO9NM4SvIHOnL2V5gM+LjiPveSyvr2qwKe86k5oPsKxNjy/uko+y2T8vXWwWj2yzwc8yNsnvYR0zr1OhkW+nYsZvj6ykD3oYyY8rvTyPVtYDL5M7lI9K897vRlrSz2SwBU8DwizvGLOzz4Yj7i8fE/sPQmMB733OMi9mfhRPpp0gT3gip0+dr6mPbZHJr4cp+O8GHpBvq1s6j3maPQ9CINePcsXCr5GZvc9qBURvetGwb4xWA4+1I3fuxSADr4NQk++ridhvdVOYT1rfHm9RQS7vCF51r1RGRa8OgtHPSgAFb4W2589JQ6DPSsxTT1XeHS9JiQnPt04FT11kfW9","NzUpvmwCgj7zUwM7UXfjPU3wIr5z+QS+szEVvcqEOD2Wkx2+H7uFPYF2BD7uP9+9JCtyvTxpHT5vVjs8ESv6PTtrNT1n0Ji+xz/JPQOWvb67pEq+pNXLPRjdYb0w8Xe6gIXBvAl1QL79kEW/XZqhvfgkRT+nyt89n2oHPqeQVD4Y8xw8yKa1vW4CBL/TyBk9yRbTvfDmhL6l1Oo9A9M5vgS8GLzzvoG+pBhPPlFCi70amiK99MFFvWGwE77Alho+w8+8Paif9r17Eka+HG2ku3x2ej0x6VQ9X28XvQ7tDb4fYv49X5+XPSo7BbzabZ++ThDvvSPdpz1Jz7e9sSRvPXc+FD6zTO08NnCFPNif2r25oBg+fqbUPd1DhD4EFLE+bOrMPo3mYL14FCQ+YQHMPMj5Gr0M2IE+e8kgPgoLWT1MK0c+VVILve9XZT4Dkzo9KpcwPIdfgT2XtQs+570dvVJgCD4rPji+J1daPVqMRD5Rg1k+XCEOPM9RWT0N7CU9KweJPp6a6z3EKTU+LOGavLA8yzsldIE+L1xKPWTjbj7guyw+zctXPYl8ej3+3Zi918zjPdzVsjxL7WY+SCIxPugcDD5+GAw+K5hePqaNSD2rlLG9tr6Cu1MQoj5NZLA+hb2YPZKL1T0Ok1o+7gNOPOi9kT3skwk9YwoRPkCZuT7DWPU911GCPrC3gLmltyy+Bca4PR6o2T5nRsI+VtcYPO6OCj7zGsY9JcIdPm/kmb1WEsa9ES67Pdw037xKYLI9Hm06Pg6jCj7/BLs+jxvFPQqIRj1lJlI9LXk4PfRA3T2VXHs9OeQMvhCXIz6dpD0+VveJPuxllD3G8Qy7x3pFPm9APD6Et6m+Cf1aPRbxJz7lWPW8gMP/vBm9Kz1EIK492/CkPe8bzj0tsv89F07zPMln1L0GDbI9OfH7PMMZMD5QMVi9rsC+vRnHmjz059G8MQ+KvQOsyT1Q5G08WyqQPcLyxb139bw9kmwmPvDFqDt/l0k9cnpYPpkLWz5LG58+","cu+BPbnE0rwC0dA8VUmRvCqfx77/3ba9bqz8vSnMDL/u/lY9J+gkPmCMa74SXeM9W3GgPYwdBj47XY88JXIBPluVgr3uTQW+YescvnQ7gD3FHWW9fysnPpucKr7+rIC9cxWNu8wL3jrZeqe9p/zlvaxm8rxSqYa+oqgKvW/iID0P4X4+hbCavdQeM75WdUY+B0wIvSiISb5twV6+EuKmver0WjzfDgi+wRSsPXxkJTzjXX29NUT1vfD7KT7HHQC+B4ShvuxjL70CDDC+00zCu6VlxL2JVEm872wwPh7hMz66NuS9T6ckviBXdj7dMDe+ji/aPRVTH73Tb4k+4DHjvWlwSL6jbT++LzWMvWpjXr0fsD696tuLvVeDmL4OT2E9P/Tmu6UgIj5OH5c+aN2HPNDGtD3E848+nvb1u1SXGb7DS2S68X8LPoWzS7izJeI7DLDavd4NML2/HDC+EquoPXV0QbwnsIQ+RApzPm4H2b2ni3a+zDqrPDaN5D72zbu+7D7HPSXZ/Lw8BxE+M5TXvbWb3b1I5j4+6VazPTGuvr1LezS+Pi41vpw9yz2AXIW+NUDpvce5nT4CytC+r86Tvv1Fab53eO29PNaJvWDLGD57R3S9Sw/ivf8g6r2SUMQ8bAWSvXX3hb10Mt29VlIWvoVa0T2Km+U9CR3cPW7x5L1Ui8S9hWDBOlsIuTwVyIa9xaQLvhqKOb6YdE6+EI1MvOfPmb2KZjC+3FEwvupID74zkLe8ploRvrUisDwGpMG9lL1Evor9hzu/jhq+GPxCvgvgY77ZE5W+PNCNPevnbr0BaGG+wucJvkVupjwlI+e9m4+3vP83Cj5vJ/69SMmvPO74lr4rHHy+GF43vajzsj38vYi+0PWAvlt02b1gOo2+d8lovIsR4j1SbuK9Wx5Mvc/KGr54dJ++g25ovRZ/Zb7rShy+0S/XvXH/kL1LQ9C9WB6tvPQms71qZNC8xIaXvQck8r0rA3S8l6dFvjm8Lb5COXi+Jl0DviJ9+r0S8hW+","pUsUvuoo0TuU0+K9ZdyMvTmog74QSfi9ZcWWvSAFcr7iAoK+8jRDPOobBr6IE0e+A5xVvcuB8LwzoJY9zlSGPIMc4DzQzdu9HYyxvGHjhr2SOUa++JxFvtey6D3kEnq93y//Oj4fVrymhiG+omFYPWczLr4aTvY946EXvj9grb1dUN29pBqCvoYTHr7A55Q88z1evs7Hr7ynV6u9qMRCvoK2WbziAge+SA/yvRc3hz3iCRm9eeQjvkedOL1oEIC+7SAVvmjhbL6mmua9LM0yvpJ1DT2mgXM9tVKevdZTMb4oYcQ99+Mmvl8zkr2SQbE7HW1YvuNGH752qbm95s+yPE0jAT74C+C9mTaGvMadtz1DKM88++YbPv/ISj6xFzI+a5IEvsbRyD377Qk9VaOTvhgOd75mMHu8LgVjPq3HlT2iu789lrNiPMZMDj43lgM+76BMPZaukz2UPZs+mpq0vA+J7DzXiCG9ye8UvRE+Kb3aJnQ974oBvZl+azwa+KQ9DZKuPG3Mn7zHtBy93k0rPpvpZD2JEgA+2/1nPVzT2bvnfvu8gQzDvDD82L2KcQA9rkLAvV/Hhj6uoui9Ik9rvdtDqj0ree091mQQPSD5Eb1WChE811IevdlNp76Kag69ClKTvcjxy7zhEQq9wz1xvYfDYr1CHCI++z0GvF9suj2nX3Y9UjogPvMv1L2M1qO9JSZVPqk0EL5p5ey9vJWQvQ1Pqj3NPCy+agidvUsm57xzjAU+OjTuPIiSM77wl0A8h0q6vbBSoT245ug9cykiPi00nT0tYYq9zlK+PfGzLb7aDae9UgcTvt8Ziz2trcg8IfUlPWRmnD2R14q+plykPex1Qb0bGrM9kVzXvf9i67vSTDS+7BUIPnMmKjoifDG+Lc4ePqXPDjytYN+9I+M7PSHs/LysVDu+6A6mPTHpIT4c89Y95+g7PvbR5z1YAvk91UHGve7b9r0IaBe+0qZUvZ2/Yb0L59i8c3HpvYde5D2JMHc+8eThPfudJL29o1s+","pkfVvbLvUbxi8867tAmJu7IfTr3mQPq9NbmDvVFk4b17poC+6wbevaHEKr7aDZ69CL1Uvnep971M1w++r6EBPkAvCr6ATDY8EzW0vlnCr76ENnO8JFULvrGfBL5V98m94WW3vVh7Tr7fd/y9u/xMvgeKo75qWVW+CMZhvmnCszxT84G+cRTFPdbKHr4yRdW9zqXkPHIlcr1Zs8i9EDUfPQfDSr7DlSO+wk91vWeYMb1AU7e+OW00vJwBMr4T6lu8ck8cvl7Dgb670Ne9KG4mvtbvzr0Vtjo8Wo+VPq6pYL0AdqU9IjIavmMp0Lx9pOA8dItnvtR+u73FjMK96CGqul8hsj0y81o+h8rcPTdC+Tut1o2+1HbJPfWxTr7/QPa9aE3xvFObML6uk2m+rZmUvkMo970dzrc67nJXPRMrRDyN7YQ90vmIPUyVY7wqI+K8/fgsvrb9T74UhLw7bdNGvVx8Ib2wGre9XaWkvYeFHL7DdlW9N0SDvtVYHb7OYb+9ous6voPy9742KG6+k0sKPUtIjL26eRO+NhtIvZrCpb26Xhq8vJjGvW2eAL2c17A98rKivgzPgr6tGlO+HZYovjTjur0ymH++wvf0PDHbRb5RKTi+lNNHvrR0iD4PSZ++U/ohvUyzmb4pg6Q9M/3ovf3HkLo1ZGg8/8VlvuDGDL5z/wA+rbcFvhKyBr51cys+uUgdPVOnHL18BA+9wkeWPeIEzrwWCdS9qGaHPbtbML1VUIy9MGfQvE8EVT5ovAU+37dmvX+/xjyawwy9jF+8u8vOQL6p3qY98qNYPTQsnL2VtSO+pNIhvhqT/rxOGv68TA/BPS7Jmj4lyQW+9o0DvTzNsb2xkEa73Fs8PPh1sjyvmxw+bqEuPtT0Pj4eQ7S9y6eIvEWho728QKc9ZJu6OkSoB753XiI8INYCvqbBoz7baAQ+EBh1PAUzSr3bNh+9VNcWPdhAAj3qTMy9EiiZvn/OmDybpMo+a6a+veFsLb7bMda9eO4VvQfInr24JuM8","LGQ0vpacUb3NZnG9JmuWvrtn0TwyHjC+Yj9+vmEXk70yeKS9GHGvveGQOr60VWG+S2K2vZISlb45Amm+bI+FPZYJab4zoGU+YtbIvVoMiL3uv+a9PfB3vlsKPL3K7Xu+49xkPllVqb6wdxs+WvCgvU0lo76cg0O96AlYvrHNH767ayQ7kiQfPpLZYb5+ZMy9XooPPUA2Fr6pqy++91vYvXi0wbt4Dki94QM/vtjjC77+JI49TpQ+OvSAJ74Ql488MX5Evu9R5D2JIcA9f0E7O3gblb3g6aO+YZwGvooPKr4dZXO9d/N9PR0ABb/ZQQK+IfoKPv3kjjylAkW8IZwAPnRf9Lwsb4q+Y9LbvYaIQr6isFw9dJJkvn17kr7UXkG+KvaevgFMKr6A6Re+IhGTPAtVO70y0xa+qYkfvuZi9r0BT4s8+xzWvSQjnr7ewP++bv/cPF3hR74ARr+99/SdvmZxQD6s8Ke+7VL9POL+2L3MRKu9+Or2Pi06Wr61NyS+cMCcvZUX0T1ROOm8hbClvbIZVzwSfTW+ZADfvlD0UTsVv2u+T5IAvkReVL11ouG9B7lXPsCfRb3MnEO91ME4PcgoFr7NXM89K7VNvq+TyL4o5Ti+IphCvf74DT74uQy92fQmPuAUEL6L+qK82XyuvZ5eV74xJpa+GS+dvIScFrywSxS+/4AyvvoWbj3w0NW9ti0TvLGUIbsfeYe+Wl/avsKMlr3C3928ChvmvNqGir6X8HM9GHoUugbfvr7jIKM7IShlPglIRTxuXyy94YXCvXFxkL3ONAc9QI0hvemhFz1Wl509CwVBvmsgVj31ZkK+dEeavfa1OD4Jj2K+aEtwPePCu75Xus2+R02IvZ33Qb02Tgw9jEmSPbDWpL4zqyS+HVwnvnQrLDxxAp29eoRcvYITmTynHGY9NIITvmHkar48uRS+7j9jvlsSy70aYjK+BiOoPC8A/b0k+nw+d+h9vrmeEz1zOfe8Gw+nvZg6AT48IhO8l9DevfxvT776rVg+","nUQFvXlIJL7iPpy8HomRPZSeJ70U0Ug+/vd8PkZqrj6+pna+4nIcvohwCb0tiYq9jha6PaN84Tx4wm49WbWSPQw2Bz7Arwc+ngYdPabS3T6YQli8g9mYPZrMAj6zixi+pR8xPbz7eL2aN108nP5qvqcuCD60jam9Fw7TvlNwrTsVUQG+FXIoPTa7iD0hAsw8cQIePnHzTD2gOLc9GuUMP2zHmz2PQTG+qq6nvaCQo70fnNi96uymPVXMBr4+04Y+ZGo4PgaQgjz8SBU+4xMsvuSCS7688cA6pP2JvXXH170iWja9KzblvHsBjz3+Fdi93iOHvsAi3D2Hr8e98UWoPeHW+Tzxz56+wF56PGVolL6Gc8G9dfTzvf7TE7+toLC+j0XJPPsNZT7FBIa9VTVqvmT5QT6cdoG+/4lFvXKzgb04GAe+0zptPVoepr2IxL69g4KgPYouFTuNs7O96fyePbuNWb5ntrM9Er63PpasOr72AJc+7f8ZvrvAcL5+3ku+BH0QvhQT+TzDGDW97BFWviMzqDsZaJq8bSKDvosf3b3K+wy8ZCM2vD5fa73lfba+TpIMv+8/2z2cQ0W95E8PPqjE4T3z1cw+jJ4qPi57Ij6x1BU+fjzfvac8pb76Hgs7/FWOvn6F170C1mu+RPeCvTNFejzrlio+O5JuPL8oID4Q0QW+90ouvlVOur7LNeC9johHPYDmcb5rJLO9lXzHvolSlb6EFZa9IPDVvZGGVL0cg8u93BUUvpPijb2/KQK9SHBDvuLzHL6MHY++bPWcvj5trb1a5Ya+AWgpvZLNlb2ULZo8NbcWPGmjvT3n4KW+rbNfvi14cb3tuzW+Da8cvi/MrbzinB0+th8rvgznAT0QTBq+XPoDvhx1W74RdcU8Sj/fvSyzOL65ADu8TqUTvtPP4ztlo22+oshgvmkeCL4xM1C+U+EXvafXJb4evW++mCgQvtQ5RD1rOvC9KE1dvuj2vTzzHmO9Mv7dvVN7OL4YdcW+nHTjvcguFj2HMgg9","Mde/vTsII72QW1O+DlqFvNmQcL7fYJe8ZoKrPbEP2b6m9hG+/COCvTPrWz2fLJS9lJqAvOROi70kQQu9CoJTPZgo374fv1a+UFpLvs2UAL4jIsM9TWk+vkmpy75C6rW96YsPvrJpNT1sWIC9zW6nvhyOn71DEAE+5AK2vfwQtr6Hccq8ya7evZcOGr6sA7k8AgdMvBs1XbzwJUy+xn+3vQizqLxhcT++Gtt9vY+RhL14iTa9qgKHPIdphb1bs+e9sqm0vFKT3r2Y66G8wRzpvCRNzrvKLOq9dX5zvZdqu72gGmS+nhuavmUjv73eraE9Q/fkvnxxgrxXuxK+nGbAvFI4ET68s0++ss7WvaBSpT1Tv9+9IPl5PvjeXDziUNI+Yb1LvQ1y/L2v5LE8TMXqvMm5mb3AGBG+YeY3PopitTxW0Zw+BOb3PYL9ALw02Kg+/K9PvqN/fT7Io2S9tJ8XO8OzBb6s3Ei+vywCPSafP74ucoI+GjNZvTJW0bvpGg4+8/11vXzJOT3jOpA+pOjLvaJo7j3MYYE9wDV1PtM73L3WCic+VUg7vqe8vz2iY/w8+ewkvXDRfLxYQG2+UE6DPSyC+zyvOWy8/vFEPciYrb4u/Qy+JNHbvV1Goz2KE36+UOSEvQUUvT2/9bo8y+Pavao8lL7sUMs9cNeKvUTL2rpWKMM9pTckvU9dkrvC8pG9QM5wPa3MIT46Nje+u5wJvCC/nTz4YZ69B1HUveBE0L0V6EM8Zi2Wvb8XRL1aVb494WONvZ0fkj0f2929WpKpPRK4DT7P0gM9HVeQPcR/iL1R0KK9nGg2O1Cngj1UcP+7aCvRvW7X4r34YAI+pVx4PMruBz0L6CM95NbTPCHyAT6I0KO8mby/PZCjh74yqqg9ObNCvUzwW7yloAm+G8q2PHJAFL29xwc9HODUvTfZSbuEVzC+USyNvcjQ673+nG2+NVOgvdP+Db3TqrU9SANXvP06dT32K3O9E5KtPD6XFz7RBaK997gnvGFp2T0cxiQ6","5xmhPeyURj7ckOa7EpqUPvs6AD09gos9CvmPPaxK+Tx5vy0+X+hcPGFXEz4RZo89GHwvPgvJiT1JSA8+Y8dCPfLU3D2uhwU+BlqoPTtwbz4eaZ08/QSlvQm+LD0/dAw+zeMSPtAONj5erzU9eCuCPm27YT56UI68+60MPkTDjj4g9409w945PXZ3YT5flPa8Oe8bPhmvVT7hTFE+7dJKvIpHdj47l4g+pWv1vPgUhL1YPjO+3qohPb74jj0/ikq8CpBfPJS8hT43bw46UKL6Pc0/SD6Prn49Zz07PqNwLT5yu9E8HmsHPkzzsD3B0Iw7M/euPugH2T34zYC9F9hlvfFY7Ly/6RI8zP3/PaX8lz5rMpO7GPugvVZ64T25jy6+/4pCPn+68z0HVG8+9NYhPt4KHz0oAS4+8Cz/PFeFhDz/kgI/kHddPdktv72cT4E++FoZPVn3q7ytkIm9WmpDPuBobD40pLM9GoBSvSOqI77ah0k+XM7ZvK/ZNz4IHas9EQJPvppyNz5kTgw+DgHkPZxt1j1zcEc6cSgMvvLNWz6rBFM9HDtrPuG4AD4TKZI+rMgXvtzvWjswLPa8+yILPuyCID7CjME+GMVDPQY+Tj4IHlE9FlaTPVMChb27Vx4+MUMpPu0BabxAdQa+BlenvdEdKz7GrXQ9sa2hPRbjqb2Z3wK9I+envCAVF75MECG+27d7vpTHLb7iT7a9XJFQPYMaHL2r80w8AQ67O37NVz0YXA49KfnfPewYQb4uQAE+Tr8Lvuikz70GR5m9CVJKvcjDyD3s+Am9SUHFve6nHzpSBFM9Ki1HPXEylT0jvpi9/bJtu/yfKT0g1o48+q6GPED3mj2sXDu+mIn6vRQllLzXGS++f9wfvhlH5b2XHvw9MGwuvOSBRz74Fd28X6B0vStAAT6pFC++GsQrPmBDl74mzjm9f0SRvszleT0vPOO8wHcjvZNFCb68Tuy88WURPh0LbL5Pzj2+h2FdPlh1Ir0HBOI6GCFIPkgI373L8u69","7cJYvdb2+TwmqQA9F2hFvUS9Vr6j1Y0992IuvjQo0jtfQZ89iRlHvC8bvb5MRNc9C3/AvaRJvz1QLHY8dXrWPcZd8LwSE+O9jpJaPqwdvr2BG/29NBQtPjurQr0dDhk+dxbKPIsxNr5fzkq+l363vXwiLr4tibO9qpeePaZ9KT3nwRs9KwBivmHj0z2jKMK8zP1TvfL7m70Vx5A+QYCfvKbEBb37k709ymXrvALSaj5e+eK9BbNFPmFXAD6HugO+Np17vcJlmbwZSR69iY6APTHBC71573Q9Ky6FvsxPAr5fkxk/Up+TPscB+Lu/cOM94VFwvQ4rWL5BrwK9iIZMvuxWVT0eaeA93FGdvY3+BT7ZK9Q9lk2gPVryPT7fS449iKwzPgu5Sj2QgQO9eYUiveGfBz52PUG9LpOmPsPAKj4iwV8+ixCbvU7TgT4tyEY+5jEYPjCHVj1gRYA8lfKePjZ3LT5wQ948GZAOPorV/bsYS4o+6rwWPrRCiD4mVa08PquIPnjiCb6+8Ic+kAkMPkAVGz5Egwc+bd0pPvtwMz0bXms+/OcjPkz9gz3oPnQ9SxrcPdprZD7UOTk+BTyePYokKT79hFc9jQWTvBauhD4RT4s+QK9bPlUKBL7b0wE+OS3Mu6OYdj6KKYM85kcLvDkXJT5M7LI9e9T9Pbplorww2Ae+aLvIvBdZz731hI4+YgpgPa3qjL3yuQ686A49vAP2zD15BAk+JxHHvG2WEj5p1Yc+OP2xPShrqj09BGy8Na+BPSFxkb2LHTk9SN4UPfg/Kj3aVZM+qD2bvdnDRj5ZqJ49QzTYvYh0Ej2K0AM+jU74PUVTdz6Tiko+amvfPVhkJD6u8gM/zv+kPkL2qrwPm4a97Jp/PvlVZ72n27o9X6JpPP5m6z1Xqzw9xBp8PVNpBTy0/yo+W/kSPlHZ7j2HBaM9coNnPS5gGj6/W6c8XYLzPYJ2Fz7iAB29NRyYPqvicz3QXA897ZoSPqmFhL3E4iI9UQgWvj8sVD6KLcu8","2YiivYqjwr2BIQS+n6GMPewKgb2vPIW9ttc1PfnwIL79UJI9P5hFPlN5HT3SXqk94YeqPY7pDbprrCC+RBm2PVl0kb36EtM9r4I+PotDyb1HFqq9mCQ2vTCuc70J9129LhVdvRbiKj22ry8+GCYOPv3/B74nvxy+U0Cfu6qe1b29WD0+eglnPUEUFL5fxTe9P+FWvtgAm74GDsy9qCknvitYA76B7ts9g5BqPawy1zyD0jU8+O+IPS8GRD4r3aq+bUqEvWCAGj4uOd29q0pXvdE/ET4s/So9lw5aPXyIhD5InAm85DhVvoyi+z0TDqQ+C+3qPd06Dz5ekjg+J6mAvaascL1TC0G9gtZyvcoB+j7JFWS+jLOLvY6B/LxhpsK+JX7eu5oEBT+Toaw+TS+bPYrSyT1KWQA+X2uwPPxyDL0L98o+kSzpvrX5CT4o1UU+Ppt4vY9/mD71oCq9qZWqPcywyj3ceV8+oaeJvmQd5z1x98U+ZuOjutdz1D2npzQ+z5gZvlY0p70ZfCU+PWDYPimj/T3HuYW9KDekPT3OQj4UE0M+bsksvuEKBz331KI8SlgrPr6jUT1FiYE8Cm4yPhE6Rj74be28YCqavt7LBT79qAu8kQyoPhzItb25gsU9X8Jwvms4CL1zcFo/Oe4EPkwEFL7cG1u+EXEVvJa9XT0J9yy+ekDLvTXomT2c2LG9lnWSvXmkQr4YDC68W3LYvQg6rb6nVhs9/dGLvYD78b0OxMO9SlEmvb3qJ75hxhk+WZGjvnW4vD34gO2+eutXvSG86L07lCq+5bcoPZOqC72r3hG9jJQsvuVbVLzvCjU9ukRYve8Flr40YK+8AQKUvbIonb3jMqG+cEvfvcNgvrztpZQ809Ktvn+d4L2juM2+59FWPUhJyj3fvR++Le6jvSanP77qpBS+iih/vnQlgr4uuRM9EYyyvQ6NVLzkv02+tR0pu/hzOb6D6kU+QO2ZvkIOCj38P4W9X8WWvYeZJ72qJbK9s9KHvUifV76Vgw67","LBpGPRolhj2zuKu9VLjZPBg2Fb6dboe9tCUqvjbv0b1t58u92k6LvecEJL5uFiG+Q/gKvoYftr08/2K7Ei13PuikBD7jLic+kgvyvCXZcr5Hl9i9i89CvekzWr00QwO+S6KgvUuujL18fR6+3EI3voOrYr65BgK+J4G2vef8WTwDlWY8/xWLvt1AOr1yjiq+adA7O6QOEr1UleK9KkdVvlMuqr2kska+wMUbvbCDdj3qO3m+ARSgvCjbQb5dU4C+h8b8vbXfNr3ESiq+Q7EmvuvhQr6Pdta9hObdPTobgL4Cqaa9TOg8vfk+MT0cBsQ9oNz7vDVNBr1LFAi+EPoJvjesMT0015I8zkRxPb369r1vlcg+3VgBvsMrML20FJ89x5aCPvC0q70+Pwe9u6zlu3rzq73AmpK8ezpQvvIGmj3UnhA8m+1kPXfPqDw8m8y9BPuqvASCAz7WV4Y8Me+APQs4v72BB+s9/X/YPX2mfT67GC6+8ok5vBkekz1zkki9tLiIvp/hRLyS3RS9ilypPeA0dj2vpqo+/xROvfdbSb3mXD09utYTPtLDzzyVvSo98wK5vZnkLr5+cjq+CufEPtRNHr2OPkq8KJ90PXKoXT3UKdW7wqtXveePZL5Yb0C+rTA9PuurPj7YZRK+7zXCvZgWkz37GgA+fdgzvQB/Sz7mE6C9JxADvvEN0r5j3iI+qXwEPX82Y76LzEe+Qackvjl5mT1sxUY+aPuxPqgcP75i0FU+ZmyQvrnSSL7vxDe+5dwBvlBIXb77rRM+j2MTvtPQ4j0qjj4+jGFHPRTCV72NsSk+FPtFPjevlT64SKs9VTRfPebwvbsWKaK+6tkzvR0N672gQUi+lJzGPmOvrL36Ki2+C+eYPYAJoL0c4Ns9uBuAvkbpX75vKqE9oPeHvsGyG75j6qY9KZu4vgkN+r1d7VK+4o1xPrp+LT43S/O8Ow4WPdlvyj1Opt09lqOwPGW67L6rLd2+EeLUvcS2uzvYVU28UThCPZWIRj3GdNg9","6CS+PFSlmr3ikXu9DHeGvWjcWT6oHnA+WutYvC5x1zterSM+UU5cPt/EXz5BbQU9dDeZPSobLT6AYuk9joXqPbixdT4LjUk+newDPhJmo7zv8VA+pHRmPgtBMj7qinC9JCjvPZ5JSz1dmSE+S2ZIPeQYhD5FQJK9NDbyval+8DzO140+38bbPQDXnDyCrg0+4lCNPtlMCj5heM89j6gwPsB20rwh3oq9iNuAPUAAqLyrJYo+lT0pPmtSOz7FYD0+bREOPgNpSz4NV+O98YGKPZ3mMD63w3g9E4zovX0EGj67+SS8SZekPSbfxT1DaoQ94sgOPjVzuj3mWRk+LMeTPhh4RT7U7j+8Y/2dPePxHT7/BIg9t5woPlH7Qz5XN268Y9mrPXCutD2i4yE9qtNnPdtspryvf6Y9uBvcPYS1iT3jD6o9u1b/PXeTED0ZCYE+xuZ9PqccKj0nT6u9Ee4qPmtcGT1Wt729laz+Pavf/LwJluQ92FUuPLzZPD46k+g83z1cPiWcHj43IZ09RuKbPEDVPz75yGc+FoV/PfqZ8T1CxlY++ecgvbYGXT7omLk98qoXPlqaiTuaQZ0+shUxPtoAxj0SkJI+2yI/PWulTz4YDTG84E4PPmXz3Dwf1kc+9XXjPJJ/Wj73/eI9HWzfveZEZT55eJE+0JC4PFgKmD34nhe+2dM8PdGoHzwGdbQ9LTfXvQjViL7XLiq+PUtpPbDhKr5ubrY8Y+wNvqnlaD72Lwk+YF08PgB2DL0CSgE9vR4PvUfAzj2mtYu8Hl8EvZO/3LxUbdK9JR4LvHNk1r2nY3M9Ehw+vctZZj2pO+Q9MVQrvot8fD191gI9zBhxPYtBND4uA3O9WsPAvVP1Rr0ICbI9wRYFu+dmDD7z2uk9V6emvS1HAz15t7G7kFC+vBPHB77Cry2+HDlrPsRAqz149v88gsnWOwTN7L0n7I883xzovTjKuDzUn/Y87ppWPgc1/b3Q6ji+B5HzvFkWPL6iTbk+ayWovLaDBj4xFaE8","avAHPue36rzNp6I9BoGGPViRAL7Dbae+NIdhvkdFJT0UHt29EGuVPQcZoz3XjIQ9+zfbvXipkD5UdYG9EBhhvZhxwD4XyxW+dq52PggGRD05j2Y8hmaSPifS4j3ieYA+x4WNPXS4GT19lie+3oHUPUEGMr57xPM8uoOZPtasHT3FWcO9T5lVvl4ZHb72Xd49B1FKvTC4VbwNmqS8c0m+vax5iLt1LPw8k/wLPr3ZUD6Wh5Q9ojE1vhedCz7MPKw9NmNGvshYA7/XG0C96+EYvt4zcr3V4PY+iz+EvgF0Lz0ieAM/8oioPlaTqj1iBga9H2dgviJPR77ItNG9PhuRvhSXkby7a26+mt65vWmzBb6YVzQ9TIVqvrRKWL6FZKy+53GRvpFZNrxU4kO+D8IQvkopJb0g6f+9ViA6vuS0sTw1pp69ZhvcO0SEy75KJPO9vetcu7lEMz0S96i9ZdfKvSt2zD0RE4u9R9M0vt4rDb48vBW9pFqfvSw0Wb53v3+99RAhvphVxj0KBha+LweuPTL3Wr5qFd29QN0uvmyz8b2HDhq+6/aIvr//Ar7Ncyy9sGyhvsTm/L1a1a29ZevmvXQtcr38s9E9w1RlviCXVL7+8JS9yUCdPWL2WD1nSRm+KdqFvOFyKb1PTSG9PXX7vRkBi74HRQG+BgfPvSSSwr3x3ZM7QDWDPfqcgz3l11a9fqp3vit8DL4jXdy9xiHovVzzBb3kaZa+EknyPB+yUb4xvT2+iLVhPbV5iTwakQe9bvAFvhT3OrwRgWC+KrVfvt8Ju72Jevu9QbO4vT1bcL6Hn368EjrFvSTBNr52Vaq90RE8vluk1b1ZNhq9CkQeviTOYr6qGmS+XeEavqndozwpFGW+s+kovilqBL6eoXm88URyvd9sUb6VvCm+aj63vQ6fmr3Q2fU8qmajvaG4O74+xYa+RH9CPOgiSb4CKMO9S8TCPQL5Lb2wmLk9sUUavgT/4TxWuAa+CzFGPZ6aSL1MPN08FzJvvkxoAz1lcpg9","L0cnPpo8MD1MlJ09VMTGu49c8Trc0+89cyChvBMhsT1FZAW+z6WkvhaGZT267Ya9eKJIPXR8ir2lyQy9uarbO4kOxD34KQY9+ti+vGUKnTzUnu499zaoPXFT8j1Todi9JfRdvCezSz1oJCc9ypT9vaUwsb165ws91tCqPeGnAz5ebja+2rUBPGpJ5bpTkQy8ur83PiRCeD5J1Is+rkmtOsf69rtpd/O9hZwIPfvB772a4ss9NT9KvSAxBr53AIc9ICyVPfDE7z1lf14+7YYpvsZ8Cb1i8lA9PRRevXdTz7xRdw++dzTyvIdycr6HkbU9JVdNupSuCD6Vnym9IQeqvPuV4T1nTaC+/xCVPCxaOL0zN348wnFKvpA6Cb2J+VC+6/wnvqplprvYIwG+N64lvr2/K71PMDC+sEyXu1A5Jb6DBKa8xSLlu6ig4b1R3AM+95LAPa5zpb7ekX+98HDDvSopvD2MthA9Je4bPwdZaL09MbK9hIY0PdR2qL4Xnjy9S8WmPV9jED4GqhU9aBXFvL/Ovz1ZCWY+4d8nvnzUmD6M8YK+0PcyvVagoL1XYYs8ExO9PV2WCr5+Zie+HAExPfWddr1yE0Q+XBqAPRfSC72mzBQ+bTMUvj+NSD2VXWc+58axvj749r3QaCo+ldBYPhMhTz02KZ8+n2+dvbPw1Tx9hGc9g1YYPtB34j6kOOy8VoKDPvPxcT4KB8o9nCuFPrGhyT7LOzm+5WG9vUAMPD22C4m9mMUMPdANQT7tJa++FffRPY2Ljz5OlDg+ojKdPsw6sLzNxY0964XiPdIIAT4BkJG9Y+rUPU9XVr4XO64+9LojPqhX0j5XIN8+YBidPoakxD0G1Wq+JNojPqdaiD2aRiO+zkaZO0n0cr0lBYC+byWzvP9eIb2Dcy69q0wyPic2Iz71nBm+U2GZPKirkb3oXI0+jPdKvvKb2D0YCzM9KgI1PhhoTDtmn3i+fJIdvQfpVL6f0vU9vmVWPfjGaL4Vysg+mVDOPQkK2z323Vq5","J5IYPQ7BtDzCMkq+BggVvnIuLj3JqlQ9zMMtPkjTHb7M85M8Ok69vHK5EjwbG709HifwvBcPT7092R+/kIc9vtWh1jvzIc+8Po7TvvzqK702HaM9mefKPhK6Tj7NpBw92Fi4vZS/Prx97Su+O/8HPnXwqb1R8Rw/wj+1PIxtgr7Bk5G+sBPSPRhHqz3+JZK9lm1lvUUHQj6rrRu/RUGevTCMNr54KpI8AQ8FvQlIfL6wGdu9XzHMvqBeD74hXoS9wUqcPqKd0T1eW4e93F5Uvb1fuzwPr86912y1voQzCb6jd0c8BfVHPCBqkL61UjS+hm8KPYiIhDyoUhs+leG3O4f82zxmyiM745ZePmvWlbtnAoA9QXqkvQhIT7umvZy+tuGiPk7rNj6R5gy+CAgXPrNXMb5pP0k9EBmVvZMpor2rRne+CfHSvZajS76z6/m9+ASuPi3Yy77yJRW8PdBCPfIdYz2zWUa9ssqqu2bTEj5ZD+q8RissvvhMjj536wA+Ej0KPkuWfD7U7sG+jBbEvdRSRb2Hhbe80+s2vnHAoj06Ghq9acAZPcz6xT3PYvy9WxzaPiOsEj5qiuG7f5cbvuBwKr6gBZk8wLk4vnBRsD4b9iO9AkJVPU6i2b3SD4A++yMwPUnU271HOHU9u9sjvpvzbT6oXI++2rO8Pf8gcz1bkHk9pWVePhBw5D13/kK+iDgevlP/QD1GN9M9OoctPfwhDz6PqlA9FbXSvfv8yz7VlLg85fFUPp8tkj6TZR68pvRevMzTGT66CNa94mc/PrN25j0npZW+hGj5PSd9Ir20MgC/CSsoPt1C+zzfTds+CpqePkncXb7+/1c/IhPUvipe3r7+eA68lb8zPk+yor2jt0Q97Vy/PkfbuLxL2nc+Ni0WvjWTCL4rpjm+RBlwPnu3Ij6y5cK+nvCgvWhsi73cRhW9j35yvtv+mD2/HIu7cYfAvR5dtD2kA5g+uDgCPnZpij5H/Tq9EWiNPrQvF74zrpI+M/XLPV5VQ727wxK+","zcunviBfT75gPQC9cqJRvXuamj5q7FS+cNQePqMiij6soqQ8hsawPZvJoL0MVt49ck5xPuEvED7Yihg+xybbPJPhc71uNEW9Lj9jumQyDz7T0UU9xwKuvOooEr4zbCY8aNOqvbZhOrxsAiA+oCzGvsTqCD2cdUw8uZo8PJ7pwr49Jhs+0VdyvJJABT4X7aW9V7+QPHNI4jqpD5E960MFu7Eohb3xYB49k/2xPXa/AT5Q7Z69D8vGPpXEoDhnkRo+hVqtvXge9b3ymqU+4esgPtCanjyN9AC89IE5vVxrKL4Ej4I959iavMe4AT420m09jWJRPqTsYb07mns+Ab8aPsW/KL4guxq+Gnh+vXNxnj3DvwA+AwSvvfGz1D1krFU+QtEoPfJu1TzwfMy9jncRvXkMZD6Mcjw+48GMPnD1iD2cjpW7g2YCvXbRvT4PAS0+wt6EvMJVPD2s1+09v+GXvI9RBj1I/Y08i742vtgoTD6e+mg+2NT5PaRzoD0ZJJQ+jZiUPuTpKTyJXcq8lVHyvfk/Gz6TzUY9cxiMvPmBHj4Jh9W9Nsn/PcuYgDzlKty93Tv6vWACMj4IWQw+kDR6PEqHx73RkhW+/cfMPlfP0L0rkS4+wJEjPgBfpTufNRk9OWmzPuGzET0hBew+RigMPT4xjT1R/Z29j28lPnQTEz4WmD89RLpmvCV8Gz2iISq+R5FEPl2/C77/EB69GRePvWaI270RRzk9KSZqvqVD5z0aZfw9qlF7PLEXib5yAaE9cmtKPi5mrL0NOBa+MxRjvs5U2z08HPA9wkwgvqkfML7g7mu+sACiu3HpdD00VDE+z3KovTWrhbw71Oo9SU2LPJcLIz7T8iQ9dzqSvoHV+Dx2j4O+A5ZePWZeL74uwcQ+f+vevEfMGz3trYo9Qs23PRls7LslSbm+s6yCPaa4uzmdWVk8RR57Pd7h+b1BkKs6ZW8evLvFOj4fkB8+fugPPuY2B77M1CI+ufzNvbqPZL1vuC4+bCwiu6vO0D3yOTW+","Re+nvJkMaj4Gulu8f+OavWJNMb40ZMa9xPGMvI6AMr51qCg+A4BMPu+kuT3QLVE8tF/APXDZIT2KmXY9N/5GPnxPNz3fKR6+zLArPTsGar4kHD2+/bRIvYoxkLz8mrq8ZUD1OkTthz2kXSA9EpErvsYiDz5/YG0+GzHUvSs57bwhyOA9UvtnvoLZAL50bLu9huoUvom37L0MQKo9hPGWvpZMW7vctMi9l+3KPf0jPbx6ZY49e0AAPD3pTr5Uw18+cxgjPQoRF70iPxC9E5WDPf4H8Lqopds8NGrDvbZKOL41SMu9cMbVPd1cyLxS/wi+UnM4Pl6x/D3oFBM+nA5zvWjLhT2f+Fg+qhEQvWQ7xTus2R07f+mePX1eFTx5yLI+n/58PV9oDz0T0YQ97ASTPXd/KD0qRPQ96KXovNdnMz3r63e8toy4PCuQVj3zTww+ytYZPrz6DD6o+xY+fRcqPrLCwz02Eco9YEqrvFoEIr6eJSM+/0FzvRO5OT03fOy93/REPZrBtz6IEG09gGJ1PWoQTrwd/no99+DnPUgWUD0TGTY+GsLTvbWppj1c8D8+idenPis9jz61i6s9DrZ3PmlZBT5CISo8DnoIPkDbzDxeVMW8Tp4qPghhqD1AwBI+KTfHPY1Uvj0FlaI+BkTnvQV+TL3HrLw8tJoUPs8Mqj6zuuI9p8OpvJv12z2S0T++eI6/PQxCKr7FG2u8kfLvPvuNYT5fM0o+ZcMfvCCCEr1RQzY+ObK1PXLyuj7a6um9Jx5Evj9LRr4ftr49qUZQPdPtoD2AVLS9aSlBvWYiNL1Vb3Q+P+fXvKhlijxtLL89L0PmPTjFLT0tj18+FMu9PR1eVD0pT5S9vU/jPYe8Gj6g/YA6p/wGPgIDCr4aid87qSfJPaz3ID6DnzQ973AXPvWX9rxdSQ2+L4c9PrZ+VT7Noyw+GKr4PHZ2rT2cMg49j/A+PWgGOj6FnE49ZtVwvd73M70hOWI9fFfFPehEP7zaJ9s9JRL0u25NBz64UnA+","/nYCPqVX3DxK6Au9K8S5Pby8Or4L0Li9xOiqPfCl372w8B++7XekPf2YEr5BbR69m1n6Pd5y8b1Sg1E+Zv/0PThjrb3RbOC91K1FvpVyf7uP+Vc7yDSNu8ub0L1MwVi+8IcYvYnKEz3ZJSY+GT8zvuOOET2fW8W8et57vjLDnL0bO0o+lI4PvVQo+rxWfdu8A3eTvugXdb1O+gm/BofpOzcGNr2Feee9qBOSPfhbj71kc/K7lrSLvkdbRT6zQRy+HL4wvVn1szrNt0y9vuGVPPKzjD08TRq9DLdRPqnW3rxJ1Ea+/xZePULBDz5gd/c7zVwAPiD9gT0hg6K9r4SQvgILlb1su349e0s1PR9k9Tw0ABW+sJ/nvT2Qj71DGwa+qeuEvXIHfD2YYi+95uNPvH26D7zWcGo9OcbAvfPw3j1sfxG+hKw0vnzz7r29gKu9Wvl8vhM3ET61Q7O9LurcvYqyGjxxNMw981DDuxqK0T3LlRc+EMPXPRlzdj0eacM9lZCpvTi2Dz0LHMo9tncBvgcuTr28PJe94n3ePea5GjzDqTc9dlh6vcgzqj0zlRa+mfEMPVSeo72SZ3i9bGWhuscfnjusn/I990sOPTOqET2FVoY9Q1sVvaDBKr7J5kq+GnrKPQWSqj1UtYq9ccsmvfmC7bsx0oA97ynfvQ90Lr4cnpy9/JdavYI5jr1SHYC9eEYJPDf7hr4etD6+zE2AvlI27r4EngC+iBULvmrjfL75bEq+2i+fvZfbWr7VWje9pyVjvjBK4L0yfwm+If1+vpJWY74mVTy+6MIDvExvWb6ErZW92j6uvDj35L03+iG+N4jOvfjsLb6lQYK+x3ZFvP2dRb7cEng9CfM5vqtdgz1IPFe+U8h1vjLCjb7cmcC9umDXvQ0ogr64vqa872rJvYpmWb7d6ya9okkIvtgiNj2c4Ci9LEyvvfDWg75Cfoa+tilJvgcrSL7WTp+8yKlNvrShX7202Ry+VVmevdQWg7zAmiG+wNrYve5Nrb36zC+9","YGq8vK+psr31Sxe+cY6XPSbaU74H3zW9pgddvnFBRL7Fdp+9rrOFvWI83L0DD2Y9JTyBvbsj8L3T6y6+e+WcPdhVnL0Qaeu9Am8zvv7OOL5LZE4+242avVb7Lr5hIXG+bptHvWKdJT3DXRK9qHIRvWcRiL4qPQA8jsMPvoODFL7ZRoa+Tvjfuxe6tr1USsW850iUvemNQb6ZrB++W5aMvkV6PL40YFy9KPVnvojHJL5Tkse9UvhpvkjYsb2ot5u+s+W4PD6JAj5GzE496Ht+vjuZGb1H9la+PS3CvTYIOL6oyIO9Q4lrvZHmT71wDyg8kRwyvt6J4r2svSq+pVDvvbKFcbyJtb69cdMqPQ65vT2bStg9p76GPaghNT3l37g9hU1rvT9jYL7hSi4+OXo/vnQ1AT2pl0K+496TPm9Yjrwj3I67DcowvjfjHj4fals+5K72PSYNLz1kSRK9Kzi1vZM7R74Motq9duqevf4RfL5unk89M6Kzvc32gL4VRRu9PC80vvXI5L11urU9ac4vvTntQz6a1J0+SujkPZrtk72DRrg9YyONvnoda7zWRzY+xd/4vBhGWT6IQHK+wvRyPiYcpD1jq6A9Mw1bPgp9Cr6k+lk9cWojvvGHRj3kaHW+8LHXPb21dT063Wk8mTl3vRNj6b1cBiM+uLvjvB50Tj7MIRE+yWk3vngy5728IAu+2DMYvQw5MT0AYCm9ZHzwPS1MHb5givk8H44Vvs+NnbxmRdS8XwyPvZ39abwZtx88TaAMvULhRTw3g/O99jByPYOg1D06Mqe7ffDnPbnD1b0qv7+9LplHvdTriD6M6po9UpJsvh+WjD3FyEe8aQ4KPfmqUr2E1OA7vL2ZPSAV3T2BIbW9RGGjPacMc74Y9tA88Jq0vYWAqT1lUEe+IqfmvaBjpDyayJm720aNPfYIzr2aBmu9P3DCvXHWHb7858W946+rvJEekTysbea7EDy7vKms2j2S49K9LQrmvEVtRzy5b1C9YxqhvTtRJ77/wLS9","/7JoPlMOlT6oaF8+luUVPkGGz71WF/u7oqMMPv0gGz8ycsk+5oSNPAJJGb6eO/a9+aNZvRw4B70DTcE8shnSvVsJEz4bJcU+H4iJPgPLrD4edCQ936WlPZgiP73wFEg+Q4QivmInmT4qCEw+NpK3PY4brj1xEms+nOWXPq8BMT7gA5s98xWZvkeGmz27A8o9K74quzJH4704c8g+i/6PvWdU47w8cOS8+BEGPdbvCr6IzGE+EcP/PTZMIb01BxS9MVY7Phj6Lb7i6m8+bBo0PeMIA7083E29tn2aPSPexT2VvNK9NPIEvvMACr6y6Qw9hkCHPnt26T0DqdC9ta6tPWaxHT4H+hQ9XJRnOpfxRb46OWg8O+ouvjAONT4jv3U+n8kjvod7tr3IYos+U/ECPRQHaL3BqaO9jSX5PVXVLbzO8Mk+dWedPtTHpT6k1Uo7OAO+PW+QF768tBa9jxjHvToQP73jgbA+Pyy1Pac2oj02kBK9FskTPh41qryWbTW9ET7UPebSkD14VO69EuQMPnRMRz5XXk4+Isk9PlojvryPr1a8vyJovYwTsL0YV1++/gRivkxwcL0pvLq9OgOLPNJR87ybMGG9jJpaPizjoL1YeIi9l7ZsvY5hjDy60Xm+PF+TPuLBub2MidO9i7YqPm2bAr4dHFY+1lUXvDjhar3VSRm+rIFePtG8BbxCtQy8/5dSPIAoN75U0la+0Z2+vrHcTT57sgI+ifvZvW8QHjyphqs8K/j2PcHtGrysrlC+TsyBvgVQVbxY9pG9Jk9YvnJ1nj4Tg9u+Vzu/vOCyNT6IMMq8opy8vCbzyT2FeS28FNXCvNgBUL1nWg4+dXHCvqqDyD107ry9aSHtPViDG77rh3C+btNuvTSijb4qWoG+RzkfvscWWT07xQY+ZxUpvNdT1T0E8Tc6v9oYvQRVVr5I99i97hcuPn5jhb65gtM9mmqRPexrV708kto9X3ZNPmbbTz1Q+bG9TxAvPoVxrb5saCg+cht3vYiUsr20QCO+","RtdVPoW5ybwgKFS7b+hAPGAlfL62wHU+mJMLPY1fkj77u50+Ya3zvUHYvb03WtY91U4PPmzbMD0l1WE+gDfSvUChnL2fHhG+GnGMPV8QhT4YjwY+fV5yPUQNPzwT+t28CseXPdKKer37NJk+DvWjPeA/br7LilO+SD/UvFw+O77xhgY+QCAtPhBMeD6w2xc+YsDoveo7eD4dwO+9oN8gPtKGJ74vQV4+PYlIPYJAfT1sMXG7q6rLvQMQzT5vq0++wPudPZu5Zj3P6Tm9fiD9PSOkYD17ymM+gstTPoCj8D6C4S69gSpIPWvIa778W9E9TqKbPqiC1T6bK8e9Y3YyPtLiLLxbFBG+0lSZPRp6bTwmxFY9zwC3vT/bC764vRI9xuLZvmPqVr5vNNe9S/i+vLKl+b2+71m8+ovavB07Ib1aTBu+PlALvts3s71Te4m+ghp2vT6yAL7GfZi7xNCmvt3qHj1or5m9kUQHvokKG75Hp529uoR+PtEnar1eQIs99vWjPGtZQD6kxTy9o34lvufOWr2DQYK+1OBtvuLDKL6vA1G+uTmFvqqHVr2Gls+9yw3avkR/S74zWn09JEUIPnD5ZrxE3Ju8+WDjvVNyIb5mrq+9lCTJvWYUYj3oyFq+2VAava1yQL6PaAK+KHzlPK1WIr4RRCC9fbQfPTuOQz1bwCC+LS1qvYr+f71UTxM+yR2Ivsv6sT3mDrA8+TgWvjATab7T7KK9QhOvPdRUVj6wuCe+HtENvjTSJT2SsXg4Y6Rivi01T72Edpi9+IK3vY/5AD7Cqc+9/BAsvgIfQr6NedA8vN8qvna6lz3ctMq7rB1au8gyjr2V96C82Dg7PUH/+r3NuAk+rrD0veqdkLxWgP48ocVtvi5JDL7EYJe9fLZuvV4mKb6Mbxw9n9KiO6Ap1b4HsG298ruHPJsgn7y5z2e8YtA0PSnRBT5brBK+CnVTvjSGp73xxfW9/m28vsehwLxElGG9yUr9vcBHiL2aMpC9K1A9PvBgNzzvZJC9","Jv4DPk/tqDzQSu69/Pv2PZOhsj2FZgY+nm4SPiLegT5Q2y6+K4UqvkQaTD1p8rq8Md/uPa8l8jyUmRc9XebePViXrb0TANE9NQ8uPkqu/D2nBj47D7WHPkul0z3M4iE9jYqOvcRfS750ZKS9drS6vcNwpz3Ygxo+xWyNPbhljz2QNWe+xL4OvveyLDz/z7a8dS+MPd2yJT5gLKY9ygmsPXANyz0VaiO9HcUJPmpe0L02g2I893DMPS1xHr6JVZA+qMMCPvXGDz2ksqa9MpY/vUF6fbzG4ZS9+UUnPn04Ib5W0rQ9Rvu5PWh/d73aGi6+aLUWvpsP4T037aC9RqgQPYJ2nT2p/pu8DVAkPjL3qj2+De+9IHvNvbCF/z2FblO9LRhNvlgyZb0JQTS+6Gm0PVitlbsH7Bi+nP/IvUJIK701i4w+gxAfPdJ0MD4ziYs92U9cvahtFD1C5e89zcZZPc5FcT3JkFi+i8WjPcoyi72Xd5q+w96kvdrsHLw089S+DLecvWuL4bxtmMo+NzouvjOyZT0DU5s+dJLqvYvZtz0IUMm+VmzhPU9AiL2UyAa9MnHNvTel272fBBw9Gr+5PNBA0jyQ64m+80uVPQUiDT76Ct89pFiGPEJLzT1JWTY+ubGrPF3WXj7C38e9fqTqPGV8nT3yggG9wVGpvluTE77QQHK9DKwRPkLFjr2xZEQ+r8qhPUNcNT4BN0U+xXRCPm1hOj4QugM98NUvPvs/kLzlrC4913MWvV/iXz5UUBg+ro8xPvbh8T3Gr5w9ZgIkPrXBzT0UtT8+iQT2PUCpDL0RnpS9v7cJPn3wKj5h3P49jI7JPfUMOz1YPTQ94+OpvR9gIz7D7wG+KX8OPmP2UL28mVk7x7xEPpEsUT5Vcg4+IBT6u8Yyjj34+oc9QMwnPp1E172arCQ+dm8aPFzbEj6nH8Y93ig6vc7/0TrDlio+jVxlPZusvTyHJV290gUwPuTElj0q6Gg9yNo5Pp8fKz5vEoI+3mlDPgWT/rsVwBY9","e2E1PkidMT7S9R49SRGkPKe+VD0jVxo+ajrjPQoTOj5fkDc+DHsFPOLMRT0NIwM7cl9YPkXpLbwk+zC9BKkHPTGPuLzbf8+9xTXZu/Fphz5oSgk9Hts5Pi4SCr0y5Wm8lZwEPhImkT1MHBU+ph5HvGWzqLvwwQA9c5vdPJAvLz6JafW8e5LVu2Js4D1Y1tQ9y4tdPoljwzxoRL091MFQPqeD1jz6Y5o7rhOxvCAGBz7HELq9jl2Au3o1VD6VhfE93HX8PdiCcj0llPY9QJ85PutFRT7K2YY9i5F4O+OD+T1vKkg+tKmyPO+nTD6e7BE+cLNTPiK6Lz50nWq8f7KZPfkkK7ylZxs9TwXpPYTcM75RswW9M+g9vpE1vb1EPh49Uyz/vera7T1aUPO9GwJLvTGTT71Iafm8d5Bmvsv2z72VOxK+U2WMPK1C2z3wHtm9I+SQPf/hjD1LqKm8zbwcPooPHb2+mhI+HtgquzDSLTyF65e9ktN4PUh0Lz0P46S9tbXSvBvmqb1aRYo9gJ2cvVmCpb1eOom+osHhPNzGFr1o2NE5Z9J1PVbtmz2l6qO7ChHzvY07FL78KhQ+NC8HvmKfNT1qVrM9OV0Zu6ouID2cfBC8nAzvvJ30Bz46QGY+pEhPPS00xry1IdU9r2wYvZU1LT4Eo2y8vmOfvUYKJ75wuz6+VAq4PTcU1zsoU7w9j/4EOvdYR7sgZks96BT2vSEO+71G+8M9hs9xu2oblD2rwpq9TpA1vCBRyr3dXOS9+jweOQ1pUT052/O94P6ZPZDvm7yOMu28li4MPY07gz2UDxE9Tio6PvOEBj507fy9CQWmvD7SKT7GRAm8KN0/vG54w7v6xAk+rG7PPeFntD0Nj9w8WonLPVjWOT6fk8m8PkgPvn3QsL2NCDE91MFAu5NxZj1jvok90v4Ovo4uwT0+Jvs8xEbAOzZWsz0poUU9sdVuvX6drj3TqAw+SmGwu81WOL7oa0G91SaQPf3J871M9bC8cA6gPSxA9L3CVcG9","GMjqvb1njrzDtw8+hBGOvZhBEL4t2h69Vz+CvQw3Kj0JHxy9Y182vn/wML1/Ykm+zGXqvTx5+73eD4E+oG5KvDGOfrteiJ69O3JcPZYiY73ROOe8xMKjvBupmLwQnom9BhS7PDhhNr5zXjm+TLLAPUp0FL4GJI+96IQbPezLXD2tejc9XMDqvkGL7DxNxhK+XJlDvUaYQb4HUOC9LymKvov9br12Rxe9qVCLPWJxOb47LLo8qYlIvvXrD7t0NI08VqBgvSd7rr2cV6S9GMHLvH/qhTx0YVE9vgNdPOUZIL2AsJm+fKAYvpRtFb7/vpO+z2mZvpv8dz0MCjO+dDaGvdf1Hr0KgUg+rfoqvRKXuz1FCJW8anFmPd8CLb5NeZK+g2CjvTeQzrzTsyu+2obQvV/qgL695UW+xDkbvWaQ9T01bvu9Nf5XPS6qR760c1++Aqm2vVfXMb6upOC9tZzBuwJ74b1fmIA9IFAIvrf/G74B4RW+DIuavY4Dab40Flm+n4CQvdzl8r301Q6+E0UxPbnl+b0MBxE82l+GPJjbvL0MPxm9OIgNvmjlrj2b3rQ8PZCjvSulDb6qV/W7p+Qovhoeibyaxv+9z0s3voAntr0jc969OJTJvVYppL3G0CS99qRGvemwPT0D+JG9ckZfvJgk9L36uPO9psEJvqb4TDuUXCo9FMKBPeAxWzwdGjs+yS3oPYcVSb1/KTA+/dcbPqteCD1fa3C90B42PtMwY7uMrou+LaYCvpvYxrwfSrG9I1VPvYLzY74uUqk9+30FPr+rFT7rpjY++i5jPlzQnj0rS0U9D2Q4vpZDcj04EQC+EClCPoGdBj65qV69UCvTvTQ7FrtiXh89MujyPRu/cT5vRKY+uennvekscj4K5ji+eHD/PG+0Jj0lgXa+lMJtPt4DpLwum/M9cPwZvugx3TxsUia+rm6APBgJNrwapoQ9JQaOvVyol73etgW+Jk+LPHkRXD6mz1I9db8JPaGmDb4S8mK+vlDhPaTxEL6KScs9","2YyMPr67C75uaQQ98JZFPTiXNT5616u9XbXFPSq1Dr3gpcw9c1fcvF7poj0UNs09cA7vPehhqL0QV6e9icFYPW84zbyaJli9voMLPkT6iz5p8YE+LZhkPQX6HD7TbQk+gRfGPQ6Y2b2Q9AS972clvRp0Ir4glxO9NaoAPpS6ij0MdPm9pJpgPWSoAT5PQ4+9a/OKPbKFKj3sPF+9V6oavLy4Gj4m0cA8FrfVOriSJj59za29+NzDPUwT9DzrEDO+osy8vSKbHr7h6IG9+GYGviMM3ztVnFk9V80DPjZM4T0gCPa8X7bJPDz/sj29V20+9RC4Pft8gr2obja8RyKEPbI0jD4+Euc9oV1hPtC2X702gK69Hm64Pc1SSj6B23Q9F5iXPh8M7D0RXJg9Wd4nPs2tlD1Nbzk+IxBTPeM+TD6uUpS7lU94PeNYAD5kOU89fzVrPYuNCT4isCE8f+x/PqR/sr15AZg81DqLPlKWDT2xQ/k9EWbkPUb8kD766Us9snh8PUJGvL3vpF4++SfbPC7jcb1jRo09FtxPPpo0qj1ffUY+UC/oPeMmAz5tgKa9ETCQPmtW3L13Zps8SgjUuz6DCD3tgQ89bLGIPs9ZUT4agUA+8ilHPjwoor3+4Qc+CRChvdJyZjzNd6K94xxbPvj82j5nCho9Bv3JvIMITb4UzsQ9j2YfvlXOlz7cJ4Y8SLYiPiwLlTwxDYU8oy1uO1hXcj5t8kO66ISMvVr8Bb66TZ89IkuXPfqtbT7N4IU9CaV2PpSDmLzPVNk+KPCiO0S/KD3qsBG9WAoxPg5pNj6pYjS+74smvk5biz5lFxA93POJPfsg2L3JOoM91EabO17fjT7/L908CJoPvAn6yT2XlDK+GoiQPc6pwb2XILY+GAiYPWnMiL1Nbko+ng6vPPii7rwlp+u8T0CjPYFzl7yC19a8C6+WvYEJAz4QTba95CAfvKaIGLvjuYg9IOO8Pbx0lDwqKo496f5ZvZSO/LsBZpk+978DPozzTj5YFp69","JR75vVPyzzyxg1Q+4OENPkEDwT2e8Ji8zgJLvqGT0b25EK0+6KpQvglDIr5bDgQ+hcARvh7Rpz1pRiS+tKW6vjAFHL0WqJo8+9wCvoYY0LwoeTs+4WgKvp5xVz0x2Tk+Xp44u3/OFD61VWc9YlUiPt+tUb5YIZ08ETzcvPvwkb01opA+mbppPgACi74sDuE9AfmmPQO5rb1K+fa84YwzvrfauTzXZxY+hOfrvf5kgT13qws+CttoPpYSDz7WlwK+6NmIPcYWJj6DZka9I7Q5OwfaAT4IPQA+PkNcvdLVbD7LFBC+lqnMvSo2dD2nidG84YyuPtW+sL6d38A93UxCvcv2Tr5fu4E92IJiPuYZHz618pC+ePCNvepVij5vzeW90VHJPa6MFz4SFEY6gEgvPSrm6b1QtdI95aXpPfN5Gj6Xfy2+7hqTvcuqBzy7pfO9T7i8vinSrb0P8cc55EYPPkF3tbwWtp0+97arvfyWHj55dsA9kvKjPmU3Tz2tBa88FB5fvqzl0b3+nRY93hf2PfAmJz6Vrk+9MYpePjNLOL6QtpY+7N/qO10Eqb3gAUQ+APZMvloDp7xyom6+LVXlvXWh7L0a2Ak9SC4ovcfUET56Y1s9Jof0vQKFBr6nhL28ztoDvjOSQL42tgi8BMhBvYYVsj3gbKk9Wj3+PVmlhL7tWZi95ss8vroGmz14RM094zCIPmenhj24rKQ9GGfqPRpTJD6BrPk9ju6Gu49Lvb0V7vG8PxZHPYsTZr1gZwo+KJSyPrz3+T3RErO8O3o4Pj6gqz1zjjI+fcAovf93ub0+2aK9wJsRvhEguj0UVr291w1OPu1IbD6ZBSs99wIkvYFDzT16Vyc+afEFPi0QjDx1dC28KbFAPlnZKD5rkgM+G7VwPD75SL3kumq9ZYJePXk/lD53hh8+xjBGPg45aD6CQt+8BgIMPt2hyD1tdng9qTmnPfAC2j04Bhc+8vAYPjIO8Lz/W5Y+cGwqPv6+JD5eeuA+2gibPbL8GT4y3gs+","wbzYPfh07ru6V7W8kF4cPGdIHT7DT0W+AQeZuyS+Tj3Phno+CMyivZLn97xcAXY9EHadPIt7FD3K+cG7edsLPimjlD4l7I68zFvzPfscFT5WqZI+CDxjPmiZcj0Jyzc+bJK5u8cG071PTA8+jij7PQSRYT7VFgg+aKNcPvC7xD19Zls+9mUqPdYKmz5IL28+Dz2GPsYAIj4VLRg+IaVlPX0cLj4Mk649ar1yPfxgnz1c0DC90opWPS3pfT5+rFQ9vuHfvT/L7739B3E+8YdlPgugCD63dxQ+0mq9vdZRvz2PUgY9wcbxPRHJVz66/6E9HwBoPsIUAz0kgk8+Gs7mPZ5HFT6bM6C+OrPEvMv2DD7wcwu+ak0rPXfFYT6RJvM8hCIbPQwQXL595Ai+LqFZPQoqZb2hV9y8EypTPjO6Zr74tC69la+mPfTKaz2vS7I9AM9WvJ2TIb5YUzG8X7Y/vsG+sj1HYEy+R5gfPvH+Vb6j9z4+lzNovdY6wr0wv3i9qsu4PbaWBT58RrO+2qOEvGgqXr4reqO+fSCyPQ5hhL3n3GQ+X+EQvolN/D32+GA9FYGJPady5b3ulFA+giwsvsZBDT3H0AC+JC+GvfjvVL45i7u9cXUgvpsxfD6DMho+KvWpPPuOir3hWwg+KR00PfWhjL1VwbM9QUCcPTNEHb17Vx++W9PvvcrGv70K4Um9K/DaPf4xgzyBd/m9kPHLvHtZfD4WEgG+heOJvlEKp73xUDe9Pgu7vRobwj3t0uk95W1PvqhCLD2f1z88G2bEvo2/575hdxI9kpRbPS3IYj0HrGg85yr9PHVaX71alrs993pMPft6dD5yANc8a92SvHlNOT5+2IS9I+MRvj9JSj7ymQw9tpyOvh8nhz0WuT2+el9DPmBjTb7I+jC9r9hPvsGsHD2nfSE5t4G0vc33Hz4sQHS787ghPdvJyL0Io4i9E70xvhjAAL6pJzG+E6+qvp6uHT0v5D+8TO34vTEsk74KlSi+cT4YvrU9wj1tWNE8","eai6Pa2Uwj3oVdI9tstTPWD8OD3ZZXI+i4IuPskr8j1Z8Ds8qVVQPtXf6D2EGyM+wT09Pv01gz3YC7G88GLXvClN1Dwy6R8+lJtPPufmOry+E9U925M4PUidOj2Q8wo8An4mPor/RD7bIOy8wW+iPby97D2M24y9+FfnvRTBl724r18+wXQRPgC/yDxKYd29ak2EPlEGSj0nz/s93+EoPq27Pr0rBbG8/NYkPUVcVj5rCZ09YjVlPr2xBj5+sLQ92AZaPu59ij2wJCo9UN5YPqH4KD7wMsW91LEkPf4grz1eIU4+VV/5PWwiRz6l5wG9DfGAPoKkuL0d5Tk+UH3YOZ6c+z0N4MC9oAoFPmh0F753jGU+rkjYPSEW1z30AxA9d+bGPR62XLwrs3A97gquPedYHj4tz2o+6lMyPka0Ez6kH8g9NT2lvS9bhD6Msxc+5mwaPkQVRT7ERyE+PUpuPVI+Sz7Z1Wu9pay7PHkJ6D3MQvk9S+u0PBy3Oj7oAwC9VFjTPQiEgz01syM+Q7icPSvSQz77WQA+VubKvcnFPT6TVmk+db77PBGXiD3cj6E9/0YwPOFVFzzgvIM+LVR7PjjQKL2GDfg996mrPZaNhT3iuFM+oyg8PlJLjbtEH4w9CfOUvXYQTT6EKpC605X8PRyJhT0nNxY81re7PTHGv7z8RzW++JA3PrtUFz6ODmq6JObXvSZOgb47lwC+S+3fvSe7hL2LZ6k8xEkCvqGBZLyDiju9i4GhPVyrB74KAo29zU07vMBuED5Pawa9PqVrvSr7GT783qc7WvfXvVnP6L33tYg9/eE1Ppe+KT7FQU49wVMrvqDNJL03nsC9ma3qOR+XVLxxTSE+OtJbPd6doryVCB+9pCxpO4KhHL7RFUs+4hPgvJkxIDygUU49YpJEvfLHkT1bl8y+q7RAPnFfv70DMtq9oIPTveqRwb2/LFc+y23hPOHLi73k9pA9c6wKPhn6LL5qNYC9xzN/vdfBRT6WlY26CbYIvpjtBT1FlTa9","WNnQvaCB2T0DHBG9az0hPZg/JL4gWec9tLAKPrRHB77hTQY++6JEPjlJaTwwfoo9V5zDPX0WhTuuqmu9pcSfPJslqT1wfgM9HSBhPIWaH724uzM7l4S4PYdqjD0kZGO9WqJ/PZ8PiD0QPL29vWcoPhZ/4D3JQRs+A+CLPSJhx7udHBK97K28vXacG74p4Qi9nII/PfnQNL5VlPY8/mKHPSJeTD6kpvK7ULXTPCJY8zzY6sm9C+8zvesCA73DQHu9yZ7Qu1xvh70su7e9jYQBvfohFj0Un5u9WqW4vdrJ8j1JrRM+DD5WvWNjuL2O4ry9GfEYvUviE76Q+HU8V7MGPRBuzr0xULY9JiD+PTlQY74ajS6+KVyKvvkj3r3kRBC+qrqpvvyNpTw7JkW+0tpMPCLGqL2HvzG+tD84vtsm972yRga+56gtvruYNL6RaTK9NpFxvX8EDb7XF0++em72vcPuar0PoZQ9VCM2PaAL6L18Hyu+kdCbu0u3xT0GrQ2+A2E3vqZghr40xjS+ejevPcLJCb2ivh6+CxTWvByGlb4au1S9ZHIUPnOpLL4f7h+9dYEfvlVDT71ZJKi99NFjvk7oR77L69e9R5TjvRJuTr5IdbG8M7scvgutUT0EyfO9lrsGvutEgb1ntpW+3PhOvE6pIr3wGFW90y42vkyk1L2+ktK76Jw1vcUMIr3DJVG+vPMHvkPSJL47MLw7S/mGvi/gcr2uN0u+V0gNPD4Web1U2T2+Wf2VvaC9RL65cas9zxNdvsOSJ76X1ao9nbHFvWnXZ754NBG+rwBGvluqJr50nlq8ADUPvgtPVL6V5DK+t6IfviW7L75iNTW+U9Q7vg7g2byTG7+9r26IvqPnCT6+haa9SQ0Rvh0Y/z0RRwu+fbrUvKvD8731jTS+RLbdveAiir3UJoA9G19cvntFXr7jjyY82Xk1vt8FWb3O+Zg97uWlvQISRL7fIQi+nVrlvRA3r72Bf5u9Bd4QvoCdrD0IqEe+DTuKvVi157zq9kM9","zMcgPjc6tT3PrE69Tf71u6sjYT4JMog9pYrJPVNTtT1eP+Q9Hq/nPK4+sL0nB668EtsMPeRsr7r/ZmC9x4nHPaXp7j2AJ6y86XsWvj9hWTxL4me9X1OzPVDELj7icwc+MF3BveAyFLr/wpU9P10JvAgRhz1cu80848LYPSHvtT2Q7ze9ZeQdvbPGAL5w6XI9UqpRvS4E1D2UTao9MCAdPSUJmb1Z09O85tSTPD5OeT3Oljw92yskvrufsjxLIpw+ky9KPZmKxj28PGY9rdKQPcN0xD3QBay6f03+vZOpQb4OWj48Tk0Jvfgwkr7vHDW9Jbm7PcxEXj7R+uq97OKdPZbzar1jNlu85qjkvDdt9j1ozQU+JopLvpfey70ujXU7rn4tvFYrNj6ES9w9+j26vcPOib0oAoC6D5QKviiWIL5upg2+l4DzvdNWob2C3+89p7QOPjcODD513QE9vyYRvghglz1EqO69AAJBPMcfAb4yB069bMgcvd2xHb4DZuG97vf3PUYXOT2naFM8SLEJvfewSL64sRY+Jy4DvoG0nDzVZzK9v8MaPlpzFT2Ha7c8OXZqveuwzb2tXgc9BZyEPBG/cj7kElc+oX47PkRyVD6Moiy9MEC/OVU0ojv0OAo+Yl1BvodqrD0YTqI85n87vaEQUrvx1vc9efF5vU0BvD1RP1I6osswPSgZDz0g63497dW7PSpKbT0iex+9QRisPX4cCj7UhD09nLYdPq/uXz7KOy4+t1UtPjg5rjyJbRs+QDpCPd+BBT6Y0/U9/aAjPHCmszxDzSM+rYNVPmKs4b2MSgc+xpaIPTW6JD20hEs9nq88PSJalrwrc6U8Q5hmvZlHVD49W4E+UAKMPmcqN7wdwp88T/gNPjcsij4wt34+t5fpPPsYKz677lg8hPZHPtjsejrv+pM+DsYaPm2sQz2b8vU9eFwCPkhaoTzu77U9D3taPicY1TwtMww+y4YqPuD7LT4kLzI+GiOEPjbCwj1mKy09gFO3upeQqjoE6208","cnOOPRZVCb0tfGU9vH8xPl588LuIGAg+CNw5PcnEJj6bUGI+hlmxO+WKNL7FUvE8vsVrPSoeUT73gHE8C9CnPYTmvTyBkhq9KlrGvRx+Sz4ZmSa+gXIaPkROOT4HsxM+VcAxPbEQM70RGP073gMLPvcAYT6zbBu+tLMMPv996TyBtfE9NKq+PtYkST5l5dY8/taXvOqGkj5vAFm9uTikPkQBZLynir08DhD+vASStr1pfkQ+JzZuPuVTTj5t/JU9BT3YuhpTBbx4KAY9HyysPcpuVLx9X/w94VxAvqcQQD4U7qI932QNPtVI8j1JH569YjElPu5Qqb05zoY+lzq7PZhhnr2NLn297PHLPSmISL45xY+9rj8yvokxJb7rSAQ+UoZtPKIiVz6gnUC++76XvI3XVD5uIfU9XqGovVmn4z3x7r+80BD4PeED1L25Uyu+TyMovu6zhL60Dcq96Qb4PH7T4rxtIjM+f+GsPKq2iD0od0A9yVQdPic65z3hS7u9jbYMPrfI1D3ao7m9/iQdPNKouT2NVx09T09CPT26AL7vKMS97ADkPYpqMD0q+l290DNdvsBJob6ELIq9MP2ovV+D3j0bYZC84SmVPOOg5z1wnlM+81O7vd2lJT3Wzl8+2GX7vWE4o77ggqW7NK8wPjsNKD45MDM+0E6ZPoHvhr0/+ZW+dlsmvhkX5j1/qIA+PabmvFwOIb76Ux++3VuSvn+imL775nE+Hp8fvK0dq73vXNY9oJoXPrq2rr2r5+M9jWJmPqcyPb+xEG89NfzQPGtAFL73B8I+MoixPSiVxr3uUQC+gvWCPrfR373oHca8rPCJPvH+Er1UNMS+BtaJPXOrH74AV8g82YylvT7m1z1DcFm9wAwbvpsKRL2jRL+9T3QAvmGEnL4qzkA+p2CBPV49l71Mgjg9hkmZvXC/pD2X5Ck+jGsoPr8kJL04SMI9e+Tsu1pm2z6YGxU9wimYvLYz0L54vB+9nAAVPyB6X754yr++mDSbva0XCL5VmRK+","q1YNvlzb17wClSS++X8bvoGviL6QVJM9lDqYvCoXaLt3oAK++YAXvQDyl75+NbQ9YvRhvpEZWL7Lf/c9eTEPvoRUTL4jYgS+ZSynPZeyzr7zWNm9ChKJPHvPRL2GxGY96bYyvUiKS75pu1q+EbO1vaSHGr7Miai91W6Ou5w2UjyI8Xi+y2E+vo/D3b0ph+E9BGJ/vYOTDL5d1V098JWCPUslcb18FaY9gbadvYPdcL7923E+QmynvsAZeL6qjIE9wfCdvcKUzj0xcsy9JCqpvvVoAr7myy++dTFIPuj9VDiomCS+YpA1vvGx7r1T2B4+JCijvr9jkb6/ZJy9VQulvUD4NztO0pE9VUDWPK3wxT3/YFe+ewZSPiAIST2cfXe+yWKNvkMpmjy4BSC8m1MzPdzMW74+1yo7iEOcPr48ij3o2Qe/WsApvhve8z2n31++pEwuvoqG0L1gtly9amTzPH77Pr1g+Di+TykBvCmV373SBn++nqFsvd7EN75xujq+RTzKvi0z+r4NCj2+GAmfPRxwjr5TU+29SH1gPoLxtb1FFG6+wdYlvca/rbybI7C9oZYMPtNn6r6KjyS+lBCFvpArrL2G7vy9UMY/vgL4mbt9h4O+2z6mPN3jerz6ylu9K1vfvh0DUL7tSyG/5Ze1vHyVu76O5om+ONSPPV58Oj1KEjw+UMdBvrypPj1UFTA+FmUEvluMBD4XX58+V+Htvd1hgryBsbc9+ZSjPrscwL0Mt489DUb/vXVPnj66JoC+HG9MPneZQL6hDdK9VAAkPo0Bur6DuTk8WagwvSDCxz1Vu3c+h6t1vpvclLyTz8G+DpcPPX3mor2lxpW9kaunvb4YDb6Aifc8NXNBPs5+FT7ZOpc+1X1HPSHiYzzuvgS/YsgDPSnhVD0lJsi8gcw2PrPTob0Uc7y89W8xPVSPPL7OuaI8O59cvT2OgD6Ytl++MBYJvaL/0r0aDcO9TAUUvoTMPT74Fqo8UBIlPbHYXL2Pe9k8JC4ZPqn9qj2sf+Y8","N5k/vAbBKb6cFAo9S3LVvdecJD7TVmU+icgCvjpq7b385BA+N/DTvazNnL3BUgu/c2lUPlV21TwqQnS+dhwePhSdlrt+SwU+EGCNvC99fT2CO/A9IngAvpzBNjwr3Q+9y5anPJ7yiD2sB2O+eUNpvM9neT7UCIW+qMenPT2bjT6E5/E+ADJLPhdpnz2kOJA9G/pLPqRnnL22aK69OOsiPjTU3L0VoEG+i1MWPUzITzsVqko+qcYOPdrJur2SFi6+7bwJPrdOqz5IDD++9adZvrHDIL6nINu9FGIcPhaFUL3H5pm+VCy1vXa0Dr7RihU+13QlPvJnFT7F9SQ8JLwzPrr+Cb142h2+xywIvjG2BbtR9yM9kcdFvjRbhL6PeVc+6nDePbDhjj2ENou+88zdPYgdLr4kmzQ6yLOKvm2GHb4qFwS9wOKxPQzuoz0pSZy9hXFEvi+F9b3fCa07xgVuu15wML307Ke9qBzlPQTwiT2aWDK+VVwQPsQSmL2X/Q29O9rnvdDzsDziA109YD79PVYA/zzOwAo9Wuswvl4s1Twr56U8h4YsvmGXhz29R1y8YY0Tv3jdhDutNAq9pBMePai9F72hKc++gWSevjJ+tr1UQeC9taUMvc/rTjxqO3a8mD4bPkyNHT48f3s7trVRvh1qLD1eY2A8eNk9vtjMW75KPO08i18APE2PIb39sq28mBoGvih29rsxGE++L4vDPV5sMb5TkWm8vwBEvpmBcj1qh0W9cyfnvRY8qb3Nh/C9HHimOyNLlL5QkcG9w4AJvEmQI76JwsS71fz1vSe8lzzP+wO+tVu0vGAW9byaiFO8Ft7gvPCqRz78z0m+7uF4PUTHyr2HsbC8z2X9vYIJOD1RohM9JcZjvSMIJ76BL2y9PNxfvuTNLr7S3BS9WRqsvsjfrL0o2Um+lv/SvdBsJr4Ftwq+tMMBvsWtiL60WyG+GNFDvl8x/zxIUBs918cFO5jHHD739xI9plevvemSlr2NvU2+b/plvkeqtL0v3R++","uk5yPUI+AL44eY6+wFmDPifnJL62vZA+e9uBPgT2Ib7J3YS+AYSSvd7z4TybMCA9BSf7vSdn8L3Wy58+0OKKvRB5ED5v5Ps8hjTgPiVxV7y5hHc+FLW6vqN4Ar7y52G+W7DtPdV6qb3yngg+sVS8vkpfgD50OM29CVuAvb+omr1I85K8cmsmvpBFIr3nBHm+bR+KPXuT070FsLY+C8k7vFDWmT6NIxm9s3WPPJBP473C2ow8U+A1Pshy/zxqO2C+3oHAPIbIhrwHY9s9eMNEvg9jPL3nQjC+dsWgvRfM2LunRxm+bh2qvW9BPT5MlzG971mrPOw+qL5RsSq+RgawvOWhej5w4yM+JxZdPTuYgj4ZZx495VI2PTJVWj5iYSA+NLugvTcpU72uKiY+d2WcPmXFMr76y56+yAYcPUMuIL4L28Q91FAGPlApA75jRkG7O39mPuweBr0vB8k9ZnVtPYOU3L5CINM9iP0RPhpWij6aZeQ94aW1vWBDjT0XPpi9xZuTvoQRKr7r+j0+yfNHPo+gj723t9U9rA9Pvfg/5r2xKru9HSTqvWwMxz1wbRI9/apovffPbL4OZCG+KcEIvvGNo76Suwm+S50HvaY76bzWO5u9muFqPgqVOj5gbNE9VVMLvhR4C76b/V8+Ds3MuyUD/ruY3IQ97FacviKdb71C5Jy9SNCCPU01F7oOeLQ86N6UOi8v+T12oQY+D43vPEDxij6Fva89HXmGvU22XjudPCQ+b0odvFY8Yz5GBSk+e8k5PiCxBD6BO48+caGEPk1XSz40ZIm8opD4PZDWhz6Zvb69HmezPRuDxD1mAAA+L7JzPu04xT0f+Qk+LOWzPU7a4j1EKDC+U2ffPbR8DT7Eqos915OWPi0hmj7lLLQ983ETPnRmEz52f1U9HOqwPB/BbD6HvAI+2SxcvBO13LxLtTM+4F1aPpr3zjx6fq894JkuPts7Lj1AcyI+Pvi1PPtDcr2PjlY+zYNLPpbNtL1aqWs+2AMfPjM05j3H6Os9","RGsEPh7BSD2aZrk9mGMmvliVJT5M6Bc+lm8WPpQyUL04pCI+SJaaPRcNMb3dTRw9DF9tPQx4OD4+oA8+1r57vUeeWr3zsXM9GcZDPhMSHD7IggK+ZGiTPZF5k73eVH8+UKCRPJwIlT32QOY8wPuOOyFDpT3DbkS+2gsQPgYnqDvKmBM+ngcjPprtST2LjlO9yIHZPSaDhTx3kYW8MiWtPR1CUj4nBG28AxMXPk2C+T2AlUM8TLMUvafoqj3XKhU+O7/lPdgTtrz3Aqy9RlLFvF4bBT0p3wQ8GEF3PDqRED70+8C9Zm5BPuCQFT7Lhwa+ja4UPVZO/D1xC8A8XWuDPbWUZT3e6b09H9nivTefdr3GWpO8nu5cvsZ7qL2eM328UdjMvQrL67xuRQa9jPGfvECzD75pN9U9IwDfupGtTL0vzbG91C/PPcVg/L1DVIG9AjxVvbDgLr0019K97LM0u5rNATyyGR4+jVXvPcaXnT39Gxu+itsPPEoPI70OBV896NQEPgnmoL030fe9vRwLPbz1H77Fk6e9fS6DvRI7ADwv7Kq9B9mdO06erj0JcXy98tJBvCu4R752FLc9Q7I4vpbprzx7duu88BEHPEmOMT7EnKE9SgJSPb2o8D0XgWo9iSx3PeZqIDyCg1k+5PFPuynPUz40yJM8Cm2VPcc/FL61+AK90WQCPYqmBj2cd+Y9sh47vs9Gw724XAG7JOeWvWUGtr3vPxE+Y7ATPvybnD2a/E48mdWWvGRU9D09iAS+P+EmvmfIkb0OrVM+9sbDvTTl871ytxI+RUQuvn0xqbwoKxG+MYKrPeMi6739Fhi+cVmRPViBMT0TTPW8PNowPojRX75lD8U93XuKvZmMSjxrrhW+nOPFvSm9iT7xgiM9uT7jPbOF3rvP6aI9CGJQPRzA572yEAS9BasGvt7isL2cfge9OcriPOg9w70xSHi9gJKSvWcy5L0P8t89CoQHvqMADL5Yi5I9ywnmPYeRPb7k7x6+VuuIvCAlbL0v1c09","8dgAvxEKuL5D+is+rx7ZvdMMbD769sS9fFCFvU7pir76qyu+vUMZPoHHor3I2pc9Yu4nPm/fBLwFqju+9FnUveosO75dwFG+BBtuvs0EKL44lCO9oMVhvYoGNDzSV6q+iieuvN1mur7O4RA+osaNviY58rwVi4m9zdQmvnnGvb0ivTy9BzV5PhQfoj0Hn26+URAwvfxGcr7JLxK+w6TnPf1daL7Hu589wlq/PPakcrsuwAu+9GrIPSZxF7yngjU911q3vXPVlL2AKxe+kiLXvWkpFL5Y2Nq6xhFSvafrsj17lUg+rqHFPZrcib5pzao+OU5Dvs+eQT7unt+97S/6vaHJM75Hpj+++DApvk+RWj6cC2s+/A/dve/rur36UbE+a5ouvTbc7DydY1m822djvlWyLj6LLog9CQyavrVnPD7EfNG+ObQ/PvhErDzJE709w2C9PQv3rz1FKY8+OJX4vaubSrzYBbi9es6lPPMesj4y5DI+KekyPUz2EbxgS0e8iaINP0/OET8ZGTs+efSqvU8KVj0c/rw90lPrPv6sCj6b46m9y2owPm7/3z0IWL27T5OQveVJGz96hIW9a7UPvTbZ4Tx9SUA9iBC1vaJ3zrxLGqC7klEPvuTxzDztARE+D1wOPcQtTj6Of90+v6c2vWBdNj6MjIU8NptnPir4Or7Twoo+Y8G0vfIM6LuyU+E5cKjtPAgJBT5uPhw+dzFaPt1hN745She+Y9UkvZD/Hb5G+1G9EMYMvYAsWj55We68uNY3Pi2LHT6AwNy9l58DPhc1T74GDOM9Piv8vcJMl76N8cS9ktGTvk5pHr19DTG+4M4VPXfRrj3FBZq5bViHPm19E71MeY4+Q47wvQVbjru7No48lFMLPtsgsD6nt2g72pbBPRK3DjxRANo92//3PQKGl748Jo49u2AkvjDP971TqEQ9zLswPgBgCD4qdFa+nEyNvd8XBb1lwp89hwPUPdxII77l5Vu69i4Nvt0ROr4pWSC+ZBGPPJ4aXr1uLLS9","KTy9vqESlL2ZiG6+B/vOPU/g0j2H7CO+/Q2TvL+QI76dseu+ujPhPYI8eT6DqZQ9sGRCvt81j7xPuiM8c9riPb1Aij1N/6Y9FLyZvesmhr726bC+59E6vlO6ML5pelG+JWSiO5YO7bxaPng9AagGvvX50b1mKZ0+ewM+vSpOLD7NmVC+YqGDvnVHPL714pM+gYqrPnfSoL6W+zO9EdmCvnPPIL7dkBK+wnqkveCxBb1Qtrc7a8Lvvf1BML4/KRk+2H6wPNXVd76T8jq+Cq7VvUnNwb6wTA48OKcYvt5Q/L44IPy8U+CyvaOa1j0Dh7O+7a+HvheJ7r6waju9Y+S3vQ=="],"recurrent_weights":["pXoJPfIYiLtd3gE9OjU3vV/3ir1T9tS9DfqGvfS8PL7cPTO+DXw9vX8Y97xlXcW9yNoJvmWXAb77UV09s3K/PAcAXL7Dzpa7bqoFvpoG870bXH+9DSJivZSMOzynz4u9UIUnvtyuIb4jXwW+MGKevYODVb2bkTE+4RmgvYQEXT0keya+VFpivhDFy727/X09/FUVvgQ+371ytOu9uzMcvuw8rr1isri9O/iVvQbssL3LiAw+IwOevRZMOL7vwL48XkzivQbbT74aXSQ9MkRhvutODr0AfCC+56GZvQtao72lSSW8f+3ivVdwVL6QNS0+6YExPbUWXrxRktK9wJZdvVRRpb0TR7y9Yar3u92fV776vR09hfOYvXd2lL29qWS9eF6CvTNIJL6AuEm+0oTkOxwwubxBx1y+dVapPfEHZ72/zrC9RMzCPZLQ373x6NO9yxsuvm+ukTzd6NK7ocjUvLSNQL2EgIi+rvYtPa0DJj4aiwy+9dhGPfNnrr3JsRu+GZCgvFBCfb4vPr27gwnTPTDpIr7kRhq+MJb+vZSWK70ur3C9T3CSuwv9B74Xi3i9v2JvvjBsIL4/Jnu+1W62vRlH8zz4dwu+RigYveXgTr5nBvK94laYvQjhnj3TQkk7JBZOvud4JL4z8xU90+2YPXr6Gj0qOpK93UM3PTaDJr4kQoY9X+5qPeYqGzzqniO92B0PPi7mhDsWbJM97iITvSm18b2IEQI7DCPoPQQrg72C+Ku9/dSuPBtvIT2AwgU9j5aKPdX8Cj2QWDU9Kn/zPWY2QL31Ycw9B4lRPWP3F72SHoq999pXvZxeQD3gDBs+icIMPTZTSzygv6m9/9eGPacx/byssG68/fjAvEDwm7wrmi89mA4mPbpzhz0xhbu9KgJNPXFOr70ttTi9uU+1PetoubxEY609bNqXOk5edD0Lny+9zzB3PhTJKj6ehrC9MmEUvjkPN70Alfu7iLtHuOl6nD1Hd7o9xCnWveQUm720SwM9VGEoPn3HEb4TDLA9","NsiXvWzgwD05i4i9keDxvPevtT2ztZw9jOfBvWAFJr4mDbI8nXnAPbQjiz1UZBO+Y5w0veAQA72D14Q8opUiPZMNcrzjcUQ94Qhhviiz3L3fuwE97CuQvT3llb1Mf3w8QG0BPdjnt70EtDa94e3fvfQB7z27QOk9ZKZXvV8enTxTarW89d5XvjNaoLwed3e9+EEzvJBBPrzMV+a8anUKvo9niL35ew2+S0xpPJrKorwXX7o8tHAEvjNxFLwNNOc9nbRpPKvmd7ulfN69Uew4vZ2Abb2gEVW9ogACvuonDb5jvy2+57ODu4xIz7ufhrm8drELuqqEDT1Z8sy76ug5u2+loz3xy5O8cIXBvWSXQj24jIw9vqcGPlW0sTwfnAA+Mi8DPrqPrj3IOUU9kOhUPCV/CD7I2+Q9G/QmPLIS7zwbiys+coyVuxwjCT4hRrk8VEeAPRJVPT3IjgY9hwAhvZVRizk8IAM+5MeNPcAJiz1LISU+oQpzPTJ5FD6AJdM9XT+APiyXYj5OmdM9e9UavrcDKT6jFn0+c2UfPtOHOz71oSg+BEQhPg/NJD6OemY+NT7UvRjMTz4opJw9f4cKPnKRJz4cLBo+EufbPDmN5j3iDPs99FyUPdNLtTzhFR09/zw2PrwPyD14Nxc+RFIzPZOqfD01K3Q+EUkhPnH9SLwq1Vy9TTcQPp04gz3Uw1O9qxsWPtuiST2q7co8J/34Pc+Q7T1AOCw+VhcmPdmUI7wTO9I9JJcmPnGl8DyVGK49ipuQPX+zZz17k5k9eksvPkdE8b2DPVE+jGjrPUHSED42OkI9ZK5CPnzArT1frAM+hPvYPUq28j3mBrI929eVOkQtej0VbAU8uKELPuEZ3z0Z+zk+UXf4PeHE77wi4j49pB02PU4trz34o909l3tnPvWPozzmmsQ9aX8WPp8Xdj7LJuw9iS4pvAt4yD0vUm491qozPvOQ6jzArWA9s/wEPpYRnrs+Hks9NUKsvJ/D07xqeQc+S0tAPhIbmzx4Vjw9","2Pw4vQmuXD18iyQ9Jcp7vP7kuLxDAei9HlT1vStcIb41ZvM9zrPZPY+hlL1B0CQ+POY4PpKmGD6H1ne+cCutvbdgE77S7Y49fXjdvfyvj70V1888QqbtvRoIfb6xi3M94Af0uwbJSj4Gp+o9+svmPXmHHb6Ta4c7eEZhPpRu8byfqJ49ZMvGvLJ6Vz2j1LS9GF2JvRmfJr42PtW9jGzAvJXKIb55NMo8/61BPg5kCb2+Ut09sf6MveZczj1NosS8+gWUvUxOyrx4pgy+tiwUPsaPND6d9n48doorPqWGRj7WTGi+IhdWvTd6cz7FdKM961jAvDy6p70+WO49cldNPdmNybx96Ec7CeEGPUAmZL3bwhG+Oo/EO4hqcz3kkCQ9qv0BPdClyj2APUM97gPbPAcwObxAM+Y8JgldPbSys737bAw9t7oJvtco3z0RNL693MWXvWjyBTuEH4k9D4q3PBoR572GWxm97jYkviJYZLyXMuI9hoFvPPrHer3YofK9dwiPvYuaXD1HB4S9GHw6PdLT1Dp8KtY746lWPuTEHb1wB7M9VboAvrZLxDxWjD+7bkoDvMiKO73IG4I7e5i5PdxrPjxVQTK9c0YAvn3sTDx+EYU9oqAAPErpijwsUc691KYBvZU4hjxm3hI+t9naPBFo6L3lBPm9FM9AvWe57711LBs8rdi2PF40lD5DSuw8s63WPT7BOT4eMDQ+pkMyPbl0Uz4Dqss9v99AOy/44z0humY9a7FxvSQCe7ulEzw+7s3FPfld7LyOhni8opRjPRXg1z2mMBA9MpQFOgX5pT3E0OA8jr+tPUaRBD4QFz+9EfQMPlXkAj0iJRU+3VCePTGAQj0VdiY+fGevvFSd6j0UTDg+Uvg9PonDqTxhqAS8GvplPlBrij3AIyM+8nTkPVLdSD0TQUs+i5kaPVCEjTz2sdE9PssQPljjmj0dP8M9evwgPkUPQT4n6Ya9dThEPp7mpr15DRc+TfZePflh9z3fAoI8ls4SPjdJvD02f8g9","vithumpTrD230CY+OJzMPfyVhD5a2ji7iWorPX15oT7vFwg+X5atPb7hRT3XyMg9XrpKPjPYdT1zoF69neUyPGcOcj2dEJg9LOsmvR3JBj6ZuLQ9/8j+PRmy2j2ol+Y9h2o5PslJ/T3JKAO8MTnfPRKPZj2VTLY8QGAKPTMGQz4RZ4Q+vwRIPWiXzj0xHdw7MI1fPdGQMT6mGGI9JRefPB/C4T1XriI+d7wXPhXAybvwlEA82aw1PgCtRT5/lsk9jMKMPaKzGT0bqqw934cOPRiECz53My0+0gEvPZxhz7zPhYA92PJBPkxWDj5Nh/M8BaVjvtD60D3glz0+IAzZurGSU731fDo+Rl/bPHKuAD6DB5y9geyCveuJlb2QtAa+C4AVPh61Yz0Z8JK89gPZPUXZrDyP79Y9kWoTPnPpYT3yq5G+tKxMvUQZ/r08UI28WT4PPqd5rj1Led696q6Hvf900r1qPnY9VFSKPenLOT7SXQO+v7MqvcHiDTx7e+W7WRkePTCqJD0q3am8/4WqvV05271+ndy9Ahb0vc5jGT31MAe+UcCEPd7YPj0nZVi+fUXEvUEtIDy+Jq09Q9nvvHSoA73GR5c8cNMQPYzUAT4bELQ9Wng+PuTwxT0lBqY9LhiHPelKNTtPuvg9BRp7PSluez0DGsS99e2fvcLR3b1fQAC98e2bPTwpvT39+Z+9OAe+vZPXtTzVMiw9VeXuu5MgEjyXSFq9rwdsvYNsvzx5K6O8n3FcvT1GnDzXNrO8erPgvfPuFT3v5DG6gyBvva/7Tb6DT5M9TiatvSsUDb1GHKy9A5w6PaUME7xqgma9fobLPV2/MT7LuKK8dcZvPR8+vz2RirY8MiU5vQ/Hujx3fQW9WWxoPX5xzbvhsQm+58wDPXwhGr21hyU9UA8VPRBFsb0W6Ui9CXBwPdj/aj4Rxws9PcVpPfl06r0iotQ9s+qOvWS0kb3NAso8TGJZPf8iCT3/upw8RQiWPWSgJr4bAwS9xAVhvMWFPL0Z8sS9","REtTvYAHJb094AA+EwoEvpqfEr6Pbbe94Do3vXP0Qb1vHgy+QTUxvrsxir1tFOK9jV+qve0rBr49qrC9f8jdvZF2Db6N5xS7DeNZvgX75r1f31C+bccavg+0/rxGPF09XAJqvRYQ8buE6p+9M06oPWmiUb7orBs+7pADvoBMwbw0gja+ftiYvta6E77A3dq7W515vTIGhb78ZWm+7TNtvn2xn70Qi6e8sytDvPoUAb50W9Q835ZPvucpab3OUpu+GLgXvcoyGL1veSC+7A24vYF5372pURW+xCzTvUYV5b2EJiy+PmeAvQRLhL5hmwA+/jDmvWYDH7w9BR++0AAPvipU171MVDw9axctvW+zQ77GbyC+QMmzPWMm9r1j8xO+z/GEvHRQTr5fS2S8jNUmvhYFgb66y0a+F/jZPIuvRj3RGJA8m2xEvmNCkr0kxk6+qtJdPXQIrL3fGQW9uF4CvUybOr7TOOq8b7+uvY72F71zluW9c3JnPeKRRb1t5ru9PKXAOix1Xr5lH4q+A/fLvLIBIr41loi+6Mfsvf+uj756VVe9D0v8vThoKb6a0KK9HCfSPPXwhL3+EAm+NHKFvdCDeT1+LiS+tuUTPbFjaL5wFOK9gX5pvrFCvjtpM0y+46ibvcz+hb0r1qq9bhgkvaPqfrvxzXy9q8dpve2GFb6Wu6Q7P1B0PYZDKr2o8BE9yu3XPdebQD664Rg+mpyWPOdIDL7CNM47JhW2PQkEa72HsZS9CUXxvFRPST1TXLk8NuDpvd7oX72uU5o8au8gPvrzt71CySM9CFzyPTgqpb1A/zO9RQpYvlctFL0d/Ta9OwE/Pt+vN71jgee9u2EVPUEXAL7NqF28QQjTPG/kGj2zDme9BcwgPaPuhD1LzxA+XjOcPaB2w70A0Cy+/HIRPQbK0zzLrP49ql8XvlIqrz2ugim9yNbXPaEwwzyTxdO9NxVAvvIALr4yoRC+JynPvZtx5j0+Z3490JrEvTDKdb6Y3ea8OfNXvJ0HF744g989","HswLPjq4yz374xi9/I6OvC+o1T30sGs8lGSsPdSNibu5/3C82VQkvhcV8L0Ealq9RJn7vd/AXjzZNIc7o6jXu1ZyNTv3nUc+bbyxvUqPDz668bs9Wc0CvIJd/T1MMUa8owOFPULHP75uXmk+gtt0vFH2Q74ghh89Zt0zvPfUTzxko9Y9euV+OvKSgD3GrHa9Hf+VPfeopL3Gbxm9ruXpO4pxjzy3Yyo+4oOWvRMcRD1uxwg9igmTPJwqaz5k6Ry9vdDcPa4RLzxEW+g9NXa6vYJM2T0w17E8CZrxPLEzRj51XRA+JwIKPBRvib2Od5U9LnjFPJoGrr2dQ2K8KR/EPWQkMz1qA+086u1+vUWQlL3uEjS+7/VjvWRoEL1FZkG9ocEYvnsPtjy7++W9WHoPPSYt272d3Ic8KI4Gvg0bqr2Ie5S+l3Q5vmkfcL4/2as6UVvmvCraCb4Rpb+9SF6gvRLaFb31VQY+/I4HvWnJXbySR7q71CPLPaIqvb3GFdY9wJpVvKh+KL5NNnG9+B9aPWhtDb5pXuG9MRRDvkPVzL0Qcee99TE6vIxIr72haJO97CoZvnekf76kKRe9TwAIvgBFPL5N+p+904gjPucQnr1MjcW8zbLcvPhzkr0YVxS+KR1SvQGlfb5Z7AC+oGoavXBrkL252+s7Tb6/u07gH727lLI9R0dDvfW9BL3aQJE95d5dvf+cBL6knMi9afgtvoG407ze2di9iDWsvf3CeLzE3oC9j0uYvbWuEr109nc8l0svPY/gMb01xRK8cAEvvvkXijyKurG9XZuovAU3pjtHbe291eJ7vHtWVL0FRIe9H6XHvTGvbz0nu8+9pL8cvpElyL2leEu+wsdGOkGpQr6bxMu9q66SvvDQ671IWya+USWGvYgLPb7sCpy93S4ZvrNkqD1g6pC98mqIvR3QG77GUJG9GvoHvmPutTxfAGi9QwZCvTfeEr56yqS9MsZLvMyBoL1+IXq+W1U9vL8TVD3ZamA8aHxFvpuayz0LWfW9","kjzJvTMasj1Q2Ow92pxqPQLM9TxD6zg8PyaOPRisT73t9vg9P1X2vfwagT2coJO9e4LPPPxUjDzaBTm9GulTvfW55zyBxV291yK0veAi2D0bkB8+MIdOvTpqwT0tu6c92oOWvDE1Rj4EJte9mbfZPOi9Gr2cF9e9keHRPYD2Dr1DLgO+nvuevc0I9z2oDn28mZwkPQvQzTy6htw9lqYSvQAdCb4kyac8fFlPPVvmu70mXpM9womCPnF9X72SIN898oojPh2sqTsMRlq+LtScPk+NHz57GZw9fp2QvWX+qb1QA4g64AxPPYaXL71bxrU9ikBRPW218Typ/pK8FQAVPZBmkbyETTw9QtOFvZE/gz2A0uu8cDY0O2onLz4d/Zg9m2byvdYUgDwslFE+eVWLva8MgD3vFpI7EH9NPXtbnz236wS9F74tvYPYBr5kGyo9s5bHPJbHuL0hjKM9mcc5PfeKEb7gTz895f9zvKAl5z1+NvS96760PRA2LD6uJJ29qYhcvZMFL76x6Es9/TmNPJ9HEj3+2uy8smMkvgd1S70gFo49pLVTvfBCdD0+WDs9+12ePQLDnLw5iqG80+QlvY4evD1qoI89pmHKvSeuiz3ihNW8IuJ2PXz3rTzLJtE8SBaAvW0+lb66oag9lQMAO+QmbL3dRbg9yzCrPJ+qVzxkvPq9aEuwvVbT5z37oI29LNGsvZGVm73mG6a9KXqSvuNDG70rGie+hHNevcMGar2M0MC99fYuvuOo1r28/cQ8STAmvrNixL2Pk7W8msZvvQEAYb2d4Iu9ybGAvmoLcj1iYOW9Zy0kvR92V73ey8W9j+GvvGjFO75eqSM9I2xDvZTyIL4HG+y+3EGPvcjie72Af++9S7o7virdZ74hLAi+oOyUvWZCAr77KDm+2dGIvV5ZBj1rjpS++kQIvaBx9r3gAxy+P22+vYMVRz30+gS+5XG5vUKMJ778taA9vmWmvojeUL6bd9k8esxEvlEzAz7im6k9HlpHvlSnjLw0QSi9","r+hHvpqjtj3TqrK5LjzKPfKDQb5NYpO9A20hPTo1z719bCG+Rl0xvkhpWb38iJw9af3jvap/u71O5ua99VJUu4XM971E+P87EYFrvboXl72kHEO+yg84veGur73N4Hq9lz51PYVvQr5uQnW+Rg6vO73Vrb1orIS8XdFYvW8Gob3ez5873dFPO2pymb0YXge+p3qUvT1jwL0H9bM8Efuivv6mQL66LBC+XtWXvQIDiL2NipW+T0smvh/Dhr35u3i9iqYHvPnN1jyWqK48owhaO2lVxL149Pc79o2aPd5uVb6adbS9sWh4umJP5L1eDE09w3xxPAz8Vr1x2+A8sPiGvTgSHj5aq2o84tyaPKJqpTrdF0A+1yuePWUfmD3p8aA9+4KTPAAiDj0zLYo9rpKwvIcUCb2bSok93qlpPXZP7rwJ+fU8OtgZvd1d6bzkfaQ952MiPejgiD236AI+wzlEvak3i7xag468I2qAvekByb3R5Q49JxQFPXx7Szz+4RE+CkMTvg25FD0hWYW9ZhJvPm77Gb2jUhY9K+dtPiIJOjyoBYS8+jYfvQd3yz0YuKM96QBzPfsqFz4+Pva86Cx+PUmEBbvel2I9wzn+vdnJHL3gfQS+TPd2uzcwO77qH1m8DC+lPOzMmr0g6Z+9VKHTPYwGsrwyDzc+GiSBvXc/oj0gn1y9abFDvVKvcD3lYxI8x/HwPatZUr65aQa+MKQhvm7x6T36cdq8BJgCvYsCLL3RYjs7FiFgPUx6rz2x6B68354wPgEzmD3CBvo8lQkiPNyAzD1IZZQ95FcRPXkynj3MLcs9J1YOvoWt4D16bee9frhZPQ6tS7wBDV69AgIFvqp1Iz51cQK+WBcnvkEo4bwGx8C9rayGvKdM7709Wee9G8e2PKiFs72pAXm9v/UOvg9IzT0nu0Y+mSI3PSPxCDxwvX488XjCPL02xz1tujQ+ru4NPX9ixb1w8y09t1WeuyMHl72Y+f+8A6hUPY09x71iAHg9GhusPQ5zBz4EriA+","aFZvvAeoqb36rgy+P7sOvrZcF75ajxW+N9rqvcK8w71stV08wZMlvnBJUr4eOCi8gXNCvnQwtTzkriq+X/UevsHaHb51lR+9uvnpvWHdSL4jjzK+rOrPvQ9ivr1rFPO7BM3cvHdOsr0/cg6+VusgPYVxIr6wqTW88fgbvmH5zr3ApBO+EOWQvp96hL4zsP69U0w9vo4TFr4ssEO+ndJpvq/0N73dpl29D8UAvsvyQ75BNTi9O5cfvp2TCr5MZki9Kpghvuv1Er2MH0m9fySevZGwzL1zP+G9LOWEPDERIb72loO9yUESvumGf769gNk7bKClvV6xr73uE/y9xIjIvb+7wL2EYTu+pD9PvLc59D31bqq9lunOveo6w71Xc7S99utYvhmzE7684Ya9z3YrPZ6lnL1+ZAO+jYz9vc4zw70FP169DTQJvm+V/L2JFb+93M+kPTddmb1g0x+9Zp0DvnXYkL2IDKS95pkbvhX4KjwQdLO9JzS/PRXICL5U0ui9dJF4vRlafb0Ykf29dgULPfriwr1ViAK+MEbEvRmgnb3LTQm+LHgkPcfLMb7OkEa9Yhy7vQ2zIrqCO4a92jjbveZRm73165Q8dpR8vZJ8gjxHOH+9sWEpvS4cpT1euA++XhSMvWVa27wnOh6+A42zPG501b0+Pwi+gRo2OuwBhr0DWYY937p0vXPAyr2YppA9/h84PnN0wT3CT9A92vT9PblXy73WrDY9mkzoPdsjcry4PEU9ezAAvkwI4T2BQxs9ryVEPJMJLL1MO5w99/YOPsQRG73caXy942Y9PYdCxL39kFW9gxY4vWlEmryerpO9q8fEPYGZFD4oiBG+25pVPTTKob1QvLi9BaX/PHxjGT4An5081T4jPtiToT3vvg4+Nbq1PbjSDL69aw6+GbUSPsAprr1in649Gmi1vU3grzsyu5899SqnPHY3NrxbJCS8FF3Gvdy6k72O7TS+tpRZvn0YbzyHrxI+KmUQvm5X9r3E0ju+XGiMPXOBWb4mLMY9","m4rCvIus8L2n45o9tGiVvHn9HD7a6Sk9E52bvTnR3Txplmg95kAkvrlbJDyEzfW7N303PsMKcb3ucQa9LmXZPCxRWTzzZ/k9dHt/PETCED6oeAE+Nf00vfDfiT1ShLu7bae0PcjNc71CaAC93ZPnvKVZ4zwzFLw9p1QFPFYOEjy+eMe85hgHPYzOx70qBqe8Czy5PK1H0jy6ITi+A361PI7yQD2gMuy9YNKvPJ6dzb3GhHk9VbKcPf2j7jvhhKU83YHIvXElub14h7+84FXdu8gruT1w6xg+9BvrPD7HjT2dzqU9PDVkPVYR9DuvVuc9NvJTPVwBnD1xNNQ93/ERPXgK2b2bl0O+pvozPn/NoL0epvy9+8JNvTTf4L1+E4m8bjolvmaamL2l75i+bFIHvrdnrr165pK9jIiUva7MnjyA5Ru+VegAvlllu70W1z+8RqjCva0BEr2q0Ti+Buf+vRBRJb07ZcC9IplFvhYrtb2ccc29ECMWPqJyUT3SUqi9WglFvn0qfr5/w3i+mIkdvodoS71zXkO+rrf/vc0drL7KfxO+sMLcvQ5hcr5xQIm9KO75u4A5Yb6IpUa+HAkavb9bF76MTc69NpxvPO0Pr70kyiS+WwKfvSa1rr218PW9MGmhvcxfA72M3cO+nML9vbtxJb7I8fW93Eh2vXEhlL3v7ua8wTq3vduFnb39GKW+GXkHvhj2BL7wgmq8Pq7tvP73YL18Afq9uL91vjkTNL61xxu9PITMva8l673Ji5i8HiByvp0PGjwiW7S9EpeOvVQMFTufft69fSQguVZgmr4ByVm8qFxMPY3ZjL3pHK69cpuaveEnvj0cp469IDaIvcNq3rvQZpq9iOYxvnwfgz3ehhC+aCZUvrygl7yVQEK+4VKFPEJ5Ib6BBC6+ufs6vl1F+L4y3FS+idGxPGI0CL4/zMo9HtY5vpSYsz2pNXG+UBYsvV+dxL11i8I9QfWrvelmCb4aGeS91I9yPDYnaDoOwx49H6ywvTC3tb0egF69","oBVCPlS6xDr+GDK9r8HHPUX5nTzQao488i8gPYHnETyfTJe9g7AavVJV1D3XM7e9FJeDvc/cSzxfi/C7ytFzPpp63byb74K94eS3vXfcoz20Iom8HjdqPruMdj3viJK9VqRkvDfxer3nkKG9a8EKvVOVCz5qAxa8zv1VPIwFRT3McR294pBQPUgcxjy4/eU9zA0OPirxMz7Z1+09LCtZPe0d0bx54Fk84sIKvIbJwD0AYNu8bu2BPmT/gb2xswE+oQmaPVhr4D3CaDO9OvZOva5L971F6L47LP01vtq4Mb5wdgG9svf9PWGcH736sD29hI9Lu5YIYT7ViZY9PeBiPmRqtLzZekk+YmqxvBF/xrx+f5M8mGUFPbKzk76Loj2+CmaAPb70PL28FQK+t73ivc/mILpujao9IyrOPfuYmT2LZwu+t6gpPoRQkb4okkK9VIgLPRY+tr0qj7u8RwdRvi41lT3sJWq+mdMzvSyrKz4x/Nw8kHIwvSOd0b3fCnw92dK0Pa9LOL58RLW9f4Ygu5eLEr2df6s7TjFSPUyS670PsbA9TAPDvR1p0b0EnH+9WvYkvSrHG7zAtXa8Fpb0PYZ3AD0IcSi9G/QSvVKGM71f5SC+NwYoPtkij71B5J48JLITvb7oU70tVeG5c2zWO8jOJD2WTQe+mzKyu30noLy4m3I+d67BvfTIEr6vb3Y+XDEoPuJJwj0Y4Tc+O1AhPhCVBz7B6xs+Tk0nPlm+bT1RBKW9GfKhPbgRHD5D/sO9tceMPqE+wj1xTLM8r4pxPjyqiT1ZxCA+jN32Pc/aST4L0rw9EXLMPYXhVj4xu4Q9RKt3PmdTyj0SxY0+BbJrPhdZ5T0E/pU+zxWHPQFIhTvtujE+5JgCPhbbij5j6Ik+lF5rPea/PD0zXVk+fn6VPWMGfT2KZ0k+t13xPXb6Mj4SosY9+UtMPCN18z14igg+/PvrPZv5Cj4OV4Y9O87RPWU2Ej4IFC09Q/6gPZ3Bujul8Wc9P4ZQPc4SQD5rAs48","cmSDPtxrpT1yHjM8WH3FvXkYCD6U5w+9EikYvQ1WDz1/8aI90o+XPYXhDD5Jo029VALUPRX4xz2K1A4+tzLFvaNqID47beI9XTL8PX9kDT7R7wQ+XCvIPaVHhz0HOvo9194JPo5R3z1toAA9vmyDPUjdPD7s6EI+ivpSPsJLpD02cUQ7mvdiuz80ST4djnQ8rejaPOjRXz2E8Gq8HWFNPohfxTyfO3A9u/HnPV0hPj5L9MM+Ulv6vBbqzrxRArk8v7RXPbFsBb60ao09AVoaPaerE7xpFzQ+99CCPEPTyzx8rfA9JnO0PdwRRT31Nny9TTU8vUAKQz73xFg9lseIPWNfyL3Mj4i9JM+bOwqXob2Ixxe9V/DFvBQIhb0eLeK91gOePrlrxL3u8J69UUxmPa0HPbwR84Y9WPEcvukhcL2MHoy9QRoFvKiE4b0BoQO9xdrMPHnfq73d3/y9gNhbPjOvlT1/dNE9wq6HPRDr6j1c3gK+RR6CvVhvWT5h5zS+eNgXPKMb2r1s7/27Vsb3vYrPMj2ARvq9cmsAvtnavj2OeSi+CkrPPKL5LD3PL5G9yvrXPIc0U74/ztg9vE3pvZOtxz1HOU2+Ns4AvXxDhD22yxU+y0UwPo7a+Tu990g88nYdvOzJbzpT0RY82njNvdXc3z1d9vW9PpqTvVhpyzoJ6VQ+VFbyPY4dMz1qAjE+qgchvumwTz5u72E+JH/bPXkf+Lx4IVs9yoeDPQT8hT2KJgO+5kYWvnbd/L2M4aG9eTKxvZ4pZ7zMPya9TMCkPXpauLqlOwo+u/HxPdMLRr2hOcS9RO4MPoohYL0dWTs9Vy8CvmXNSbzELDI+UJcbvPkJKb4SwJs8JCx1vRjFcru7cQw+Tv5+PbLxzz3J2vE8OYEOvU01pbzyMrM7Yg1wvalpsL00NBK8AXmxvdvMTb7E/p49ZHZYPKsnCT6QwIG9ZDPBPY3Xery+wWg9e1/7vILarTwLOrG87lZfvEptVj2QWwI+jesPPl/STj2Ikaa9","Jv5bPab81j1qhh++Jrr4PfJ3kz3R0sc9ST5mPW84Bj6sL4Y9uY94PRSYKj54G5Y9V3I2PlhH+DzjRak8HNSUu9Kn4D0lCF277z3dPfTKvzxam289Gh07PjheZj01qTE+HticPcGrAz6j1Fo95YD6PVAUoT3ANYG8paUhvRCm272sVIE+xe+BPla1hT72vrW8dC+vPeB1nD7pZms+PrY5PnbADj6LmsE87j/cPTZMej7pf9Y97XkbPnN3Bj4RA5Y95r+APfpOKj5dzLs9XEjIPcGOWz0MjX49S0hmPU+CTD6oGow94XPfPKxcPD7Vl8A9ikWGPTwq9j0NTiY+V8ntPGHSnz3ikle81m0VPRPkcD02FHo+z6aHPMvzUj7cZfi6Tkf1PYJ6Xj3/yBQ+6RztPZo7zj2Pvpg9cxYhPcSghTsWEKM9pYvAPcQIaLw4ePk9Yz8XPVJVHD7xqpQ9XguDPX08vD0ghAa9CG8rPqCqJb23RhE+sfETPW7mZj41sYU9hsEVPGb2Mz4nGl0+dF6kPWdzrD0AgGo+wNTMPcQcbj6fSF0+bmNCPSKqcD37+3I99v8aPdEC1T0SgUQ+qSytPYIKOL2F4bE9hqBHvbRfKT3p/0o+/sJZPh/8ej3XZS8+iQgMPvR9Cj5NRdQ9Xq9YPeYxIT1F/wo+woAZPsYCgj08B769jOu2vQBTHj3Momm97h8ivgxsJb6lgWa+wOhOvoofhb00/qg9gc8rvnTYgL1xRFM8trOjPcWQIr3tGby93S09vFr+zz2zLYG9CjcRvqKJlD0dX+q8IUE6vgAEfT3hP549a1J4PMl6p7x8Du89BUvIvOe2fj1yUdU9m3WkvU2sJz5Sl+k87tBQvAHvqLw4pWq9LwvZvVNLLL4obGY7ODJru8biKj0+c9i8VYCJvZWfMb3tx2a+YMe5PXoZ8r1y3789b1IwveKtQ71H/Kq8t1I7vXA7Nr2l7js+xFWkPfeO5b1ovjO9TgRdPu9HED5E38A8WG41vY7H8Dwucb29","EyRlvZd6kz2MMYU9Y+93PYAgEL1trp694uCDPQJ2i7w33sS9x11WPTCvgjxAVIy8qo+yPQoi0D3aMla9M32DvW6wiDvJhqS9RbbgPTbGj7wa+/O7IYu9OoIgnj2XDze7TKqavLUBpT2WalE89fFGvd0W5TuSswu96Sn2PHe3bz3kDtk8ZIpDvY7jXL41F+C9hjoAPd92W70X3aQ9Jo3kPZUbDT2h4TO+wbr1PT21nrrgfEq6ykn/vHD6nDsBAew9I/X6PY4w8z2Dqsq8Jd/xvM16Hz0oag69oIH6vVB9Yb0/SBC++NYGPQ+hGbxGLTW+1mrqukpfBr2RbJ486BsIPitvtj2FZlm91WehvXtvDb4gffO940YvvgKM57o/fk6+pJz2vbxPGL7c7XG9lCknvqizBb5l/vq9W/D2vJggor1bMCe+fuQou6uynr2ydhW+YtmYvdCpOr7OG9O9lWQFPCqEL75l6T09CNUivqMrq72xFee9GOavPBBKbb5WKoo7ju55vryYhb4oVbW9BvNqPZ1RLr4u1AW+Yt/QvY/6Hr78DWm+SnoAvgPL6L2NfBK9PUWKvW77Sr6Phr69HdT3vW8RE7xffwS+SHxtvPKKKb4yJtu9gySTvfqjLL1pKQG++WwMvozmHT3zS2G+h5WIPXQNSr744aa98pO3vaibNr69vKQ9JMWJvCRLqbyVCtq9OovUvffr1r3QXQC+uJ7bvc4nmL4w2/W8h+XZvFgP6LwwEDq+VT2lvb+Fm71rC7+9s2kaPV15y7ym5dq9J7CevgGqgr0FI828ti8Xvukjlr06FHG+AbSwvS7wBT3+3YS+UlBhvnabFT55xcK9BRCVPPnjPj1eXo+9IWx9vRizRr1Feeq8ypK3vSQdA7yJyZW+8H7/vJ/F1r0JDJ+9nR0GvpBdNb1Puh6+O34JvlI3Eb7/Zv+86ygMvemImj1+NRI8qmedvQIW4byyZ9g9EJg4vd4P072BgHo8C+zKvRNctr1OT9y9HrZMPdyD273Wm2G+","9D/zvFIbZLyOc6O74W7BPZU7tz2vhTA+HWxUPSTW3z0zIfa98ACvvWohRz4YWja9JefWvZXYvrxX6x0+IbDCPC4emjywi2e+/lwIPhRUxj0kGL68jK9sPQxq2z24bJW9slQ7PfNhqL0peyw6N/RDvp8u5j1QQYe92QtyvtFxcj093oe+L28/vVlqPT09pJa8TLAcPW47+T3VgFK97PaLPENOCT57She+g5yCvciPrD3TfIU94NkzPjMLEb29Uy8+LuGMu1HEOT2O0W68IIv7vS9fqLw8H209aIkCvi+VI77bkNc97Gy0PHMGKbuzWNs9ugE0vs5ZI71GXTq91fmePQwFGz2hq3G94RG+PO4B4LyxiuM9f9i8PLIyoT1JaSg+bbJmOyLxyL2lNRc8OYvGvCdP5j0dvRC+hiOavXyCJz0X3hs9/cCqPTMWE70KtNg9tccZPuLJ6bw982Y7Q3NHPZyl0jzuwX69RzTtPZK5Xj3n4bq997WuvQbE/r1ZTq08fS/Tuv0kDT0cTa49FOyEvDSOZz0EejM9O3P0vTQftzxwqJQ9gGs8Pv+6Tb1nAg29ijMLvUkqnboBjJY9YDA0PebJxTwcbwa9vWGuPE07bL33Fc883rcevU/pzz0lMSa8QuvsO0WvyT0T9QE97p2iPBrkFj5wUaK9Hg2yvGpTJT0vfGG9e+jpPaHjJD4uxiA+4EGHPU9lCT76IMQ9gLOKPGVZHD73mSI+aUXkPaKqLj3LDC0+GXaPPmihrz1KYKw8FHCFPineqD3hOcQ9aLe6Pa23RD76Hhk+/xsjPYr2Dr3pIM66qTfNPOfHUb1FPHk95pGBPBEPiTyrFp497DlevYCEWT6GD3I+OoqHPnGVu72oyzo+ps9aPkcxMT5ORok+a5adPcpGNzy++fc9NmW1Pdx3mb1QHz8+BPkcPjiICD5IiFE+sfXlPa1upD1OMDU+6nIFPv8oIz4xstu8063WPTEjATzS7Bg+lvIZPiwF7j2c4ws+UQ2tPCGcBj4IAwQ+","/XdMOqcnLr0mnIQ80x4xPAsu1T2cvV48U+/XPawFpj02w7I9G4BYvWNqwrxlBi4+M4k7PnE1Rz1s5aM9UqD2PKJJc72glS+9vcQXvEArTT6sPTO72RBMPvknmbv6zhI+UwRFPf335D3LQcY8naMePpaOFz7I37W8NsX9PUfZajyqWr89t1uMPlcx9j0sDlK9JPkkPihFID66LTo9iQkZPj545T2b9qs9dCdLPoJELr3etX69TwioPVqx3D2Gzaw9vujyPXh3MT0eShw+G/cEPqEkKT6u3hI+KyuNPU4AUT5iLZe9n7nQPRy6drwKupI9jxVOPXNTsjzK0X075XuqPQLGFr4ycWy9jW3kPXO4E77FhLG9TeizvbamSL7KLAW+I6mbPTDXt70ApgC9IgTxPPgm+7y3eu88HqvpvXTv0b1YF2m9QJACvn0A7jujXwM7koZUPvIKwTxWdy08VrGrPWtF6Ty47QY+owJQvRZugj2xtaG9A031vA1SCz4aP6I9k3DUPWBNpb0Hwn29CJsbPqt+1D3bkJC9IHKyvbqHiz34jV89di9NPZz4jD0vVBC+TkFTvX6ElL0HXI09grpvPS1yAb5fpFW93OvfvGzwMD4Yl+M93R41PHgtRT04Tuk9vGm+vE6AML6sqNA9fhgmPmfi3Dy62+m9EhygPIIAJ75tNwO99JGUve3EvTsuArw9bk3QvSlko76aXxC9XFqzPQKjrb3bm8U9LB1XvWSSmjyaQuA9l14ePXUkCT5LhlM9I7NmPRUg57w5x5k9kBijPBlSs73AHAs+cPu+uGi6G70TQ9E95Fn3u3IBGT2Uxhe9XyzUPIqOjD0jbYA9wkqvvCtl5bw6hoc9UpYgPGfLbb44Ee69CskPPX5sQDzqgi4+5VxwPZ/Go72OKAS9AOvBu9q2SbwOTVA98o2RPHFkyrwxc+I81vDFvSE5rTy0NlI+6js/PW97drzKKBM9eoMfPehIMTwt60U9IoF0PtvMpj3YTfu8UvASviMJADsalOq8","nVUUu3Rahz1M4Jm9ZkUCPtkioz1FR708ZKqmPdlh/juhf268nINdPueEDD3Y6qk9WvYrPUiUID6Cw2K9pdPsPXUghT2yGMs9+QOSPUTBDrj6p5Y9wlA8PhI4MT5pTT+9ot70PWS2gT2GLUg8zJoYve0yVD6tj1A9334LPoCTcz2UWr89U8eyPt06cD0WRIE92eDtPXnjvz1KaEg+UOqzPdYCEj27iBc9sW/WO1xFKj46twU+I1bBPVPk2D16/Mw8BV+fvIpZHj6PoMG90ucfPitkbTz/YDg9+9bTPaFu1D00P9M9LAtGPVjd2D005IU9HcRcvGbarjyMOBc+bLZKPsLyTT3treG6sGv0PZzHbD0rM9091xIHPfZ2J72SSYK9L3ImPojeDD6E3t09gG4NPoxlGT3LVtI9S7nTvBrYAb2VaY66LZydvUdDqjwwDKY9vJCavRSh7Tzh4fQ9VIFGPQAw+j1LT4w92BqVPQPhnr2UBQ49PdADPVjzsj2/FcM9d2Juu9wVBD5Suik+EncmvbmzLz4tvR0+R1tSPTSmXz7yOcc9zKwyPs7b3j2CXiM+MwN8PgmyPD7JUDk9qWDYPUDF0jzCfZM9y/+5PdDBlT14yaM9jPEGPkQmB7xAfPU9Ei+uvFTVfz3pnBw+Z7tlvA1uej1RFAA+XT3sPaaP2z0FPAS9yCoIPc7ADrwHt7K9Ml3qvaoEzrz2yRG+/s5WvEHkJL0UQKc8CWYrvRipgz5nLIc9/y7JPdyqRz2jZMu9mAiaPVf0Vz2md5U9FiqWPQdl3D1a4KG9VKvFvX9UIrr2DS4+JXICPU8hiz3ZUVq8LXmAvfBQbLvaVxI+lKZvPRj37j2y2cY8Yzi3PZEjBTw1nPU8AZiOvbQIHL6XBy+8iV32vTlrwT2Tz+m8bbKNPN+Jdb0m//69DkdmuihAhbwNB/i9YyOCvWeUAr77PPG8od8oPK9Cer3t85A9b9IvPt67hztcY6K83BLBPXvWTj7pXMW8D/2gO4aPLj5g8li8","iuMXOytvNz0Cp2k82q3DvRHsgb15LnY9yJY9PpNG9D125WW9oFRaPY8KizwIs6K9hO9XPfqkOj22nIO9jW86uxrrJT43reS9TJXSPV/3Dr6P1tu9g+sQPp3kA75X5os85p/pvbdTqz3VBSa9NG3rvXbV5j3tuFq8A7YYPlHywjxGf5s62vt2PKmZlr1dnAg9BWFTvdtALzwJL4I9pdgVvaUR3D0HGzC9wrn4PUt86DwjF3k9UNSnO0DdYb3GwZU8UL2IPFE5Az0DMK89a+HRPJ11hz2yRnm9n9N9PPF4JT6jUBq+2phYPrSGtjqAPA29hqyLvUzlkT240sA9LAbJPQrHrz3HmDg9OXm1vQS/5j3TAAY++z7VPQxoCj1iGEs9/qXNPX5zPT4gorM9EGoDPobpqjxQoOc9thEMPVJFpj0vTiU+C/pCPZr31z1/sAI+Z4w3PrjscT6yjwU97/fOPa1mnz3V9Bs8HycdPpO/2z3MgJc9yK8XPZfzNz0PkK29ASk2Pji5oT75/EY+nUaXPU5ZIj4GO/E96TVEPuMRUj5hh4E95/0PPj6387t5XQw+s5qOPd/TSz5bfgc+DPbrPXjb0Dx6BG49EPnCvca6az3e69M95QW5Pc8Y4T2KZbs9iEPUPbZL1D3/Rig+xt8qPlf5Yj0NkTw+vGFEPn3i8jxQ6Ay8c/+7PSMKUr2Vk949IPBxPsBkEr1q7109J8VQPeD+Oj7uMdU9YLqwPRYYJL2hyxM+51mNPeQDOT1njE0+q8CXPaxR3jyCHV290WwYPgepLT1BatU9hz9hPBhB3j0CFEY9YS1TPt0O5j3XfLA99t0KPn+0w727j5c+CdsTPVVnGz48E2u9ZJMXPmJkCj6LRUs+Z9iGPltNTT69aLw9MurwPZnACz4pfJA9GYrEPVvPFL7qUww+CKxrPrqicT1lQzY+8VGFvFJB5j2nNdM9lZtsPghnyD1jwKw9yNL0PfBKtj2MYK49HIytPWKsJD2OV7Q9cu5yPYlAHj5/DIg9","HjcAvk0GEryNrJ89ATqXvaS5U73JYTg82VsMvbokAL3VAiW8+W1APOE+Dr6CZBs80tlGPkPYrz1BDy6+owYSPR8WqDyZ0Bg9pXI6PGbxAb4Egs68T5WRPXTyy708KmI9ldOcPdBmzz2wb+08vpOlvG9yxjyzQ4s96xyePejFVryFzCs+d6kRPLLNSj35FGM7o/NJPHJrXL4qN+C9tlL9vGXAjr3tC1w9w9PLPfsZYr5Giwu9jYl4vbEjWz4WilK6SK4svYp71r13nwI9bt8BPqCFjT0dOdY8hYwEPsMlRj7spZW9E3M8PfhLzz1v+/48PgipPfqxbb1IU3g9J3j+vZnZMb0ECXm9dYbLPTu/Xz0chzm+EZrQPLODSr29QVu9C6+UPFv17Ty3Zxk913TtvNYQGLy9qc49w8CZOnc4T7oXbNG8iFLHvHYhLr7ytvi76PQfvr3NRTwvto+9ack2PRdyor0dDfk9PfP/PNlmVr3s7g8+ZTFPPeTSCr3imY49oX4GvmMGjr27OHC8rVXZvVwUlLsAQ7a9aF35PeBGtb3XVgW+qv9BvXautbwYj/E3fPLwvcoLub3IMhe+i1B/vRoz+7wr+2I9wbbqPPkcrz1uUhW9T7fbvar1YT0EsYw93BJzvSvAnT2IQHi91G0+vfsKY72ubt47f5dnvbQURL5K+p09VhVHvfHM4z3i8aO94gLQvXkIBL44V6k9iJYUvbIe172g+Da9HQRXO42LL76qJxu+pJVXvRabEr5HJOG84Bl5voOZIT1G1q69g3/fPIt4XL5GdwG+mUEnPEWZm7xLvSW+6byAvSzQDL3I4sK9oVbxO+jWz70UCU2+u7bsvbH0S74ymoe+a745vvrX2b2Dz7e9JkAuvgXDHL6ofpm+mYhLvTyHIL7t64y9gUjMvWueF73xYyu+3xp9vqdvob0UfdG9+bYjPOkh7r1Ccyq+Rmf9vbFnzr0liqA8WZYavoBmsr2wsNm9HHh/vg53qLvRxJu9oIoEvlTHCL4cpTw9","h7gMvjGcCb63elc9/VYsvc9MR77KDIW9EvyFvbfiarsYvK693ZSlvXceVrwcoSm+sO4ZvIYzIr48N6O936QuvUV0fT1winS9g02OvJGZ8L2XPAk8yk8vvghjw73ZJOS9kghIvkimDD1xHMG8dIYhPaOddr5FqLe9bIQxvqYFo73ctaO9yfgzvfEp3L3OdCE8bsXxvQOxWb4l6Y29CrFZvslC5b0LRny9mMYUvXQbl73QmFk8840mvQB8Qb3pWgS+OnXMO2IeE76p3xW8dF3XvXN5270c6LO9E91+PKVnfb5ELsy76+a4vSflAL7UPbq9KWffvIt3Nr3jkwC+76oTvX9SqD3Hpq09cxLfu98XBD4RuzQ9aGDtPRuJBD5K+R69w9sgvquvsjwyYag8Au2WvSYGtL0j6gW+pvLqPeurkz28WoK81IC3vThnHj59OVk7hKnsvb6O571VCyY+E6wmvTKoEr0+YEK9AAgIvqbOjT3ustE9TJYgvYxO8L3776q8t8cOvR8XxTztHOO8hC+uPQ+K1D3iOJY9NhsUPfZcjrogDlo+cimjPXcuWr2zr6U9KicQvSXSBj5NXO67qEoTPS7Acz0Ynlu8hzxgvXed7ruc7n+8brjxvBN3Nr3uaNC9vlCEPeX4BD1TRNC8ZSDKPFtM+r1kzck8HuINvgcdST58PuA9OLGqvZOi9Dzo6+c9jkgsPsAtwD1aiwe9HmI9O+3VUzyIsfk8qqZBPTclej1JenY8dmbzvY8Tcr0XiQK9RzLXPaswYLxSfsM7YNAVPUrpfj0Ds6S9KpvuPPdMCL0hMgi80DkqPbWdPT3JKoa88CZDPRLxLL13fd49Y15fvT+7LT0/rd29cT8TPbfkOzwF7F08tOSlPOq4Sb67oZc8EIsQPQ/6ED3dBSA9xrOvvafdHL18aM89RjTUPCe1sz1P7IM8xLlBPfjsnT2gdsc2i+q5PSLH6L0uugy9YIGSvecTnz3m+FO96Akxu11Ey72oRze7BQgzPchHZb2gwUQ9","LcPovWDe8LzwM8i8CeIKvvPkEb7no4+91mllvfU+X77lerO95BEjvlQBvb1xsvO9rJfNvc6hAL47vSO+3K5MvNApXT1pxKG9Zaaqvk4xjLzoVcu9kQyrvAjZuL3gwJI7FZJAvVGr8j3IGh68JBgSvN6Pl73QjcS6pcIPvkH43jwf9hW+a5Qyvk8g7LxxvxA8w26Nvcninb3SBJa+ic8Nvs4rqDwzXZ68ahXCvcwpyr2MCPK80XiavZokPL4EbRy+E5wMvuMICb03MoU9BTWTvW1g3jy1rwm+Tyi6ve6xj734mPo8RWoHvFhkLb4MWxW+AfgnvvPZAL4F5kK+VojnO/ZVv7zmztQ9W47ZvVKrCL2FUDG+nAu4PHG/Mb5uO8W9oBHNvWSArr0N3sk8oZ73PBUzzL032TS+vRI0vV1rcz3K5w++cXlTvsTyT74SXOq8vDySvSR7or1yK269DTDzvYCs0r3Z+EW9IAAVPagPBL6FAi2++VUXvdn9Fr78boe9bV0MPRKLc73ktw++OyjeO4Dri72Ml729oXx6vZYSnr4ui629kt6Kvfc8Bb6v9ge+MprGvQb4RL1gV/m9S88Qvp12Br6uG0W9XdfJvOQCpTwhCH+8Yd0IPdVDDD0gghu+jP95PbSVNr70LB2+JePnPbSRwL36SdA8/eydvWL9kbzRvA09QsGZvX97+L2lx+k9GrtNvM1VKD7xR1A9y5txvOErpL3BgO89OwCAPTU8H7xXKVm9EyjnvZmgqj2q/Z098l4BPl7IGz2/RIM9SMfwvDo6Zz1tuzm9/Dd5PeTaFz7rLK292TQUu6962b1i1VK94Mf+Pa11wj0k1x08S/8vvPXnhbvBqE09ayv/PPa+EL2nvCK9n+8lPTvUg71SppI9RMWqPLsp7r3ciJ89gRCfPZ4c9TyJuL09DNMKPe1b4D3Ic589YHJQvWfoGT1crNu9xAXEvYxcuLyh/Ua9GRkYvV5AKD7ZG1E9y6EMvk4m970eTYO9clMRPsmVsbwjx04+","9EGQvM28Sr4s8KA96kpAvXVPlT2S9Sm7BApFPYkwET0ZphQ9l6OGPXQGsbyhENI9BjMevY39qzyak3i9ia6ju3XcCT73sYs8gGHAPBoDhz0JoVU9g49LvJ4hoDxBlLa9jLVNPauSXj3s+AU8vYIJPtCxeL0qsFu9i1advUdqgL3OJz28MUzZvO4EwD0S4qQ9WnQfPXCnrT0t53i+g9jYPbOdz73erik9VHGrvdL3wzwYtsY9P1+UPH7J0T2UVk49xuSjvYjBAL2uxds9u4wUuzpRsL1UtG89Es1mPRxfiLzRurc6+/+PPQnp0D2W9jE9+RYWPXEFHT6SAsc81psXPXiYuD0oRgm+hoa6PTih1b2tD1S+fTPqvcTj2L1zm9G7Gdg5vkf0zL1TNy2+VSUcvv5Xqr0zXq29ClLkvcT1qr10MNS9m70dvoQE1b26FbC8JsI8PHj+OL7D6ms6ec4CvsOf6b1HgTw9Fe7jvS6IzL0pCvm9cn0mPpULkTnkxqe95isRvr/am76RRGW+mV5evGyiOL67cUu+JrWEvh4pm76TqrW9Py8MvlqRb72ePxC+c0s/veZPW75xTcO9NX/8vdjGGL5qdoW+y372vED+Ab1+4829Y3gGvo3bez1YBbe9odyvvTDdhr16kvG9kEjlvegicrwYD8K8A1zlvWn4Ar5kHkO+5o6ju+V3NL5TdyO+w0yxvULBH74tOh2+0lqqva9IEr0lAXG9am0LvoynAr6YsFK9fWjJvd/Q5Lxy9Hs9+8gNu9mfXr0Wbae82m9wvu982r0qGiu+NArAvaTTlr5XPie9rDgAve0grT2EOwi+9cFKvlS2GT7scMO9WTZavbPtVb0mNKC+o8qCvs9EsL1KKQe+7eWGvnCzBL0jTYG+pqlEvrqrLL7rJvC9XntBvsyNa74VAf69Mr0tvqsqKL7BZwC+9iiivhg4h700YiS+ionpvW/qML5FZla9yZdEvqKpnb2n5IO9wtsqvZ0bfL221Rq+ECXfvTj5T77nV6+9","0F4oPXQrk70QWpE6QClPvbRnVj73XCs+nE04PpBLab1UG1y80uiEPchkoz2wJra95ll6veKjibw1X4M9UIZVPQb1c7xAAyu+SuEePV883T3B4o69QSIIPUxpBT76HKS6aoIpvdOlm73Ji4y9H5zfvUOhST7QaPo8BOniO8mKUb0WEnq9HdR9vcyOhr1HiEY9nPHsvEdTlj7Kqky9inZvPUskpL0eegK+hUoVvlScFz5hNwM+aNxsPt38tr1cyrE+nTuyPQPbhrz+9TU7ktO6OygUDL788aG9Ki4dvTHlrL195Tk9K2U7PkugUb1NBGC+apLxvSX21z3TeSG+xYbsPeOEQzwhZtg9w4HxvUuygT2KaRU+n/IrPkcqn73wEqu8Zqw/vKoOCr7tVSq+DsTEvJEkcztL5kA9QplCvZi7e73/ABW9o7tWPon3GL7adqe8dTASPsbu972GCCo9AdrzvaIQhjxF1eC8ONCOvbF+871unb+9mWY0vSI9MDxXKVa++1nwvKuGsT3lmQY+H8qbu/H8l70VDbA9TServW85zr3O/Ja9EQWDPLiAsTyDSos8qBZvvkgINb7O+AA+ef+JvLpxAL3P9qE98uQFPeGh371dDbS9/k+HPna/Ob5bQH49MftePTh/qD29+4S6iALMvO6SgD4RuLA82wAlvmovOL3H7iK+WBADPqvyKT57ntK8pJgYPhPr2T0cowg+vDXzPQpyFz4NWVg9iulTPYkxIT4y1iE+sDOwu1sf1z2mNRg9j2dGPd1NC7705RE8Ss22PO8M0j2cHzw+/izmPYnhVTsczug99EaQPflTHz6jgBc+e2pOPlt9wT6Pn7g9cnqePQJoaD2hbCE+nb7EPe05UriNBmE9ISujPZ4VOT4V4Bw+OcXAPSGgiz3WL9Y9pwjRPBqZK738DJg9vyzsPfvCjT7rgCo9bRH8Pexhcz2sPYE+86CtPGAErLzG9+09QmzDvMANE75o1qQ9fNs1PhYDCz7Jyoo9AKRaPrYO+Dz4Jj8+","IWf4PNOBCD6rJtS9GBtEvo09sT00lR4+jXFmPuJPCb5ZGjk9hp2dvf+kHz6w+AW98IP6PbftUj0ZwBW+mrM2Plmo1zwwD8C7DKY/vFJr8D2cXga963IxvJtHDD2/+SO9+3YJPsmUGD179QY9+DUEPm9WFT46UGI9W7IUPR0tLD0o+os8GQp/PEVBpb1e10g82hYLPnyUob0KD/k9cj0aPu2H8jywzcA9C/IrPva0xjzalBC+F7ltva711j3PGqU97ondPXOF7z2Tah89mmbzPCH1KD7a6eA9DPXnPVmPDTw0hD++Vpg/Pv2CAz6fqwm+gU/5PCn+lD5MXR8++kVKPc7FCrvfcLQ9AL99vSEK1j0D5WG9AbCoPSgnZb1SJpE9eXLFvXsFcL1cEe88G+BRvdV4tD38rPS8pogKPFf+Qr43GVY9aAcbvMWdOr3HwW+9n9nKvbkEGD7LfBC9M2d9vS7p8j1InEO9vvTpvarI8j04S6k9+bqwPfYoKL3KzqI95aVlvEH46D3KcYS9Ez69vTDP4j3kSwy+LL6pu29Rozl+I8W8+6n1PFqn3bwcHHa98ouVvRy++72+RqM91TlCvDftmLzQjyS+WvhHvrJPdb3hkpi9tUTXvAU/CT0ZNZQ9SSGoPAL7szx/6DC9qb3QPZz3QD2Mzzq+QJBxPA0CoL1dbaE7FLIEvV/4F75GUE2+LvStvXmzJ77fDp++Pi64Pamlh72w5ky9ReJcvlqo6L0KTNo9wqASvX3ZHj4FrOo9JAQEvsJCrT4I7NS9b9oBPlkUgb0pLwe9kU0SPqt/tb0dPJQ9vpQavpiTgD370tm8FPXKvMt9bj7WiiQ+CHQYPYiWED7gIdg9luyAPXciUr07jUC+Ha9dPvySnT4A5Iw9eWqQvcNMrz0L+8q9selTvVxt/r0RebC8OQm/PYFJ8z2tKjE8veMkvuNtor18jhA9khVQPq6BBr0FAyg+iXz+PHGngj5z39s9za0pPL/fHD4H7VM+vKDovRotBr750pa9","fZc6voEW27rK1CC+OD0qvgzaA76FzI69hFg5vYICKTwuZo+9R804vsNWab5WO8e9hXEXvtvxar7ZWr+9cTTkvbdF473d6yu7YZIAvifFAL7m9Cc9LXVVvuGfDb7/fBK+H7MPPGzA2L1LZOS9Nz0gvdN7F74x9Z49CIfsvaUX+7zI8O+90GOjvn3ySr5PYhK9bQgmvizHXL7/YdK9aiYYvnO1Fr46OeG9+hAnvg5AKL1B1gc8Tl0OvrxJ672y8zC+T1tFviAngb4me887LQhcviQTJr7pCDO9ztbHvDTzgz3mPLa8/alCvhSiLb7N5oO9nxUIvmhNwLm1RFm+xpCHvlF+072rm3a9L+bYvRtpd775kB++sJ3BvGdMML7OE/o8Y29KvOo8b736Jee8N4/ZvchDDL57HR6+iLGgvQxGfD0s9zy+HTV2PX6lw72AsIK94V0gvsWenb0rd+O9lMlSvmazW71JJ4M9+TCFvW/n6DxPAYC+7z7JO1tboL2rIWa+/SOsvf0pZL5iBa294oK2vEQdoDucWXG+3KyhvaKikb76ERG+Rkvpu2Zvc77t7hC+i/UivpQ8V73QchW+1+lqvnNED71C1vq9/PJDPTpFEb5LV0u+SZwHvrO3o7zloOK9svQavsW1sb18+CW+OQQOPEvFnr1wrLy9g1lwvu/QgryJtzk9+DB9vVap/b0Ge44+fGMNPZr8Rj1UlfW88BOqPWzta74jr5y90zeOvD1Okr641nW95BPpvbNTND4JGdg9L0zGvePGgr19zwE+d0UPve8ikb11TNE8OUboPSXIq73ll+k7tCzXvW/erL2lWjW+/UGGPkuXwTzKcNi9PQc3vcz1HL5TyDw8gAAovEKuob09l1O7CaOqPeBXzz2DYYo9/HaYPaInyb2Bq4++IU6VPaBt5DzXnGo8HKVIPYOe/Dwfbak9JxamPWw8TD1MbxO+/FMgvsE2nr6CQz++3yx5vc1DpD2IOPE8S1IPvhKqCL3WFTG+889qPmFvIL5+iIk9","lLNaPbPCzLlWglW+naxTvVhLJD4Gw1Y+bRZLvpZVr7wTYQY9GF6LPSm6Y73/7me9VuisPAzTXD1q/RC9hK9gvSIwXT3SIEs+B+ACvshpILxHNTM+bUOUvaWj1L1P/E691r1CPXPi473WzGU8KL3avbXCxb0lUWQ9CDUnPkaWRb3Pm9o9AQmzvMHE5LsoMVy85OzXPD9a1DwNs9k8QHgIvkSKQDyxObw8d6fbvTr8Nr6O+AG9/kj9PLfmCT4aApy9IYN7vJeb3L38K3G8LI2WvIldlL3tYjU9uM2EvM8+CzsIEuk9pEJAPYOMXbwfDGe8YNQGPtrARL24XTS+V8CwPA2JDz0HSs69tnNbPnwKRDz8/6C9dd0IvWClCr3Klla+JbIUvhh8M76shR6+TUjFvdghX72Ts+08t0v7vZAZPb3lnA2+SWptvXNpJr7iJHK9wTXUvdwvSL7lYgC+UjyXvYvZkL1Upfc64y64vE02C7188hG+ceTbOjVxF74kZAa+PcaOvRnRP7747ZW+Z68mPYexJ74N60q+VRY9vjmSj76eHni9evzqvYhPo7rxlDu9nw0FvssqE76rDQ6+P4/LvXmpSL28bbW99r96PaNbMb1GcVa9u0HCvaG8ID3w1oG+1zEwvXc/L74QHaS9SMmOPe7yYzwCbAy+BQFBveYo7r33Y5M9f1AZvefGZjxkaW69CXjRvGEiDr14G929Bi/9O2NVRb7U09e9A18ivS19W72DdRu+R08PvRrivT2xeuA96LHZvQFgOz4I+xc9Wk/uvPoiMr3LS829bDF5vdSVzLxFTke9J6ARvRNO9L3RzVu9n1ZVvkRCjT2znFE8FNA/O4Sgkr2Gr2e+/HtXvsd/SrwNDw29yZZmvqrNu72HTZ+77Mv4vb57lb3rWFs7Ym1yvWWAqr6C5vu9A9ytvWX5ML3dLj6+DPYdvi8hwT0AzAq+tqEdvquEHb6gptI86yktvt+a7L2uda++tHsLPARrtDuEwNQ9WU7+vHTd9bz+EGq+","BSorPTnSer22bcY8OBS7PUfsXz0swn29SIeFvLntcD3ze7a899wSvaX1uzz4uRO+ugwlvjKRvD2IUo85FXTJvbW7Aj4bpo69JdQmvFi+xT0qW6i92nzPvId0uTx/3a298ZQavidsPD3/pZC9sahVPdRmGj0ZFSS+WF2Hu/9sDj11JAm+cjwyPG+Gurxbf5G9MyeIPawxJDqlNDg+qHB+PC4KxD31Azi91XKYPevunTvYNKK9wX1DPtVwpjwYOFM9LM3JPNhrJD41opg9vwQ7Pfem770dF9u9EnJDPcfQKL4a3yc+R/E7PtRby7yo+Tm8ozEcvGCXQzzj2+W9lnX/vHLH4rzPUt49aC4yvdE1xL3+MNo9cPepPOVD2z28bBY9Wt6GvfYAgrvpph288b2gvUMe0j0vvYm9Dn/UPW/f27zcC0+9Tx0rPnpof71shyw9Fy5evSz1tr3HRqs8Je8VvWiQDb0g5Pe9vPT+PImyCL434ie9WAodvQ4BZ757WD+8NqnVvcrEpL7bKVC9Gd9BPaUrP71VZq+9/bvKvfZlT74ft0M9io2PveVNNL3/FAm9BkOYvTkcMbz2s2Q8yOcsvZCPUL5IuX48PokAPIKitTxWlXk9G0cTPdV4Nb5vxB09zJEku8Tc0b3Csui9yfTdveZFQj2nCfo8zIn9PEPGHj3GOlq9d9inPZPVM771ZYo9F7ACvLtJ5b3veBY9HiuYPZgvsDxsU8g8zcpDPSDLpD0SU4i9ZgzCPOoEyrxD4kI9Jli9uzmLML35L+c86KZpvb0eUz1m4So8l3ZyPdh+7z1vb8u8uw6xPZxakr1MFPW95ZANPr5s0D0mzZy9fB0CPjBNFj4Ta7O8LluzPcc+cbtI2gm7RVOzvOPc/Lx3Wyk+HAoLPs580z39fD69mevNvOQHFz5hawk+3dWSPY6N5T3MIbo8LDOvPRpTTr11q8E9iEISvO8kez2WwaI9B4zavS8pHz1Az1k9lysYuj+dC76xAmI9bY05vZNJAj7xjoC9","Zo00vaSw0T3E+rk9ZIsYPVZ8rbzRufQ8RV29vDwAsDxbcy4+0mrqvENTrrwLDuI9vWiJPeHEHj1XWhA+ysBSvGeB3Twdm1W9lTAgvSOsDD20eBs8aMnDOp4YIz3vcq483pp5PfVNjD0OmjY9wZ88vVWN+z1EeWy8/x6rPa2pM70nIpE9e/CAPtj/5jwIWcM9CS+3PPIiPLxy85U92VecPFoyNz21NEQ9NumbPHDZKryIrwc+DSGnPUoeBT04tWy9JdNKvCZXhD17YfC9sjJiPjga3T1bDTw9Rj0Zvbnlnr39VyU8U0h7Pbj+CT0i0Qy6lY8tPA31UDzLclA9d8eAPVgAljumuQ89jOtcvWKl1LzOSO29eiDGvZ9/xDyf1wk8/crqPdWaqz2u4GS8Ooq0vbjyBr3nI2I9RJc5PFsPvD1gWq+9sL/IvGYBRr7L4ui9mFEuPgaNbL2SSZ69IhBcvdF6TLz0lxA9CFSwPb19hr2klWG9y5+9vc7DUL0kUy29o8MoPdj3wLw6GgO+V8dovA5fvL0Bv1M8ZEU9voQ75j38zgs6N6iHPdq60bsNB8C8VkkDPNzt/rz9I8c81TcjvYle2L05LgK9iJp+vZRf5L22o7W8jTU5Pc1fqjwBkwy91o1uvPl7Lr54g9G8q6wSPRidpD2vILC9Rcs5vXuoobxi8CM+MM0gvkan4Tz+ee49LnaavQYEDD2MrLK9oWnyPQG1H7yDSY88pzffPYRRYzxxSrY96oApvgQC8TxIQRC+YEBnvS5jp70vIGE9P3w9Oz2ibj3j6cq8VW01vGA0tzwSZge+HmaTPWkHFj76QF09/hdZvHpblb3YxZi929PIPNZCwb3MGas9W/4/PZ0lCD1aCR68z8VGPfZ8dD0qhZI8lqslPqj4jTxGfDG9IS8nPomclz13V/S9Hgcbvok2Pb3OFrG9pMQVuhlYyD0H7Gy9nJHHPbEnAb1eEk+9/Z3bvMt1573hS0o9iepAves4qz0ySxC7kBoBPcVM+L0Inv69","83YaPUq7pL1RD/u95W1Bvu88CL4oxSm+ExI5vZ8Z57yjpxa+0BZEvc3ozb3caYK9hwA4vmCu3b3wIJO9rJBrvXpkWL5WNqS9Ybe/vYBqXr1Bxo+9Iqq8vRrzz71XbKe9s0kBOmivC74ARXO9LfYGvuxck70kk729kcvXvaXkXDwaTsO9lZOivmWgJb7088+9F1MXvs/4Tb7haxm+XX88vhFyK76VvIm9SUR8vnY5O77ogQO9SWYYvgsR3b1EMF6+sP4GvnXa8btBuJs8LuGtvSU7HT2+pwG+VRsDPAaie775V569ys0JvkRB8b3r0Xm9/lLyvSywkr1yfAG+DeOLPdY6wr2WtSg8T365vV+/wb1xCeO8J0wkvLYAXL10+Au+rYSRvZwtC7764YW9Sp+TvaiHm76mmC2+2YkIvZ/+Bz39opK98bGqPJFL4r3yADK+ZlOnvXMtFr7rJPO9fo0RviGa3b2Dz0W8kIu+vagdqb1krgm+PBVWPW4QM76XsLu94ssAvldWYr0kO46++PhmvQFBx720gmS+Xu+aPbf0ib5/jn29OeSLOzbDgL4LYCa9qrFZvnBn9b0hkDe+wgblvSDvdzyZHnG9imKWvOe8x701YDu+b3CpvKHtJr0LuIG+Sa/gve4YjT3C+Ea9FrRdvWl7Er7f8qC9GWtvvrjvBj2XzNu7frFrvR9eFj0EaDa9TLCLPSc+sD04txU+zogWPYakyz3AF1G8NCnhPQTyXrwWonO+ou6RvZO3Mz70z+U9DyO0PVJ/Yb32gGS9ebfzPEJVCzuXswY+2QYiPsmwKr4bNc48yMaPPCEnTjzCgxI9sPCUPRSlaz1/wvM7Tx+BPc33sL0+64e9hFJmO0Tq4bylmeU9CGCDPcVOHD003Xy7CM8BPjmlWL3xHxO9iCslvbd1sjzbGmE+bHMgvjXeMTyHxLQ9fYAmPi2hCjw9qmW8zoWwvIGoOb2fkia+tokIvn+nRz3HTbQ93JZrve5ZOr2MZdO9E5lfPV/+WT24x/09","5qaCPUCxsbz4IqK9vdTpO2Y10D105rK7DcocvkvZ/rwFe2w8QxacPJYeCjwr7Zk8LNU5vYQl9TxsTDc9c/d8vHrNl72lMpg+5KqEPZOK+T2dnbM9hu4GPS0dbz7lldq9yoTVPYfidr7wgXA96N8xvRvxEz0WeLO8i5CtvJNQ570DMb49BlZHvREesryFuBS8pPXSvaiIBb1wgAy+LwOJPdytIL37PvU94GNNPd16ITr0v/87D/0aPbw01D2kmym814SNvZHfI76zuym9MHptPQGYhL213789EpRZPFumAT2rrzs++8AcPbPr6Lx53S8935KpPco/5r0bkZC9I/NhvFvaLb7UWiy9/YaJPfCNOL0E2G69ZG/hvI2Bw73L4kI9NWrXvcDOOL6Ao+e9lO3FvViiQ71kIhi+NX+cvV+DG72CgLy9Rgi9Pc0sJLzd/2Y9/JUOvoadoL0jLaK9vjgWPOUGYDwIwwO5S6KUPJLjob0iR4C9ZZZtPVaq6zsyGx29cC5CvtnVi75L87S9E6OGvVoRUL6dMg2+0dKGvmhqKr600pS9IC3OvWufjr2qKEu9KL44vZ19Zr71cwK9HmXtvZggEL5f8su9AaWrvQ7MGr3l4BQ9qbmbvdYpu72kXeO9gbhrPWfDZr4tPWC+0Uvju4jbgL2HAoy97AEUPQ/+wr3D+wu+3XyPvSAx+j29Rwy+pTXUvKPzPr4BW3u96pOSvVbe572JmLs9/LUvvmt0CL6Vh7S9eI01vny3RD22roW9lUQJvU8+M73qI4i8bFbEveXj6L1YERq+aBHCPMeaOL7mo6q9aSQovrB9N75ixg29JSUIvgRBAj680069+pI3vUCcB74vn+2+K6ahvS7ciDwlS/i9HE9+vXG7ur0VGBC+HhjjvT1SB77P+gi+1FUGvo+lkb1yKsK9H9vqvas7xL0eRgu9SeWPvqK4mz1V/329nCznvf6GnjskORE9BYIvvsq7nr1LNsC9vlGlvT/Dcb2uTqo9pz+YvXxXKL7ptYK9","3e1kvTnlBjv7/gM+zB68PFJg17pQmtM83wwhPu7S7jw37r69WSkSPBOxQj2Vw8O9KV5RvU6Ba70WCq49P37FvMcRn7tipzI8on9GPU2DYr3eoza9NnAKPLo0hT1eLf68I/N5vSp/Fz2crhK+bxcfOno617vAJ647FwGnPf5b8Dx510i9JYTFvWibiD1uO2E8yNqgvDSj4j1Wc8w96ZdEPczjgL2qVEc9SSmDvRRs4z0J5oI6xfAMPuDb/jv0YQA8M4iavMQevT0+uAc9WlwNvBCdIT1zTkM8cIngPDPhYr7iROE9ZPlqPmWT/b0ntFe+R3dZPIxsibw0wFu9LTJkPciKNr299KA92rwRvfDhuLzudga9BZ2MPdZ+fj3p7EU9Ny6WOzlAdD0KSeE7aaY7vk9FHL5iy8C9HOHwvY+t4jxyz4K9OXAvPdzrZb1fqoA6XI+BvXj4lb1JSqg9TNj0O4EX4D37xHy9F90hPSvbx73QjR6+EsQvvem/CL6ziAY90CMcvayHsba/Nwk+hkhuu3JJuz2T3CK9R63QvUgTk70NBO69kRtGvUQykb3S0lM8GNsHvuoHD74HF3495xWKPN+FO70s2xS9wR6BPXvJ47pcfZy9zxA8u+IKdb7YW9G8C+EfvRuqm70Ra/i95kiePZisoT1mX5e9TPAhvvhMnDxdR1o9iNPevVXjCL5hoxg+psFTPgOuSz6lRya8hKYjPSDzIT6C3PI9VbxQPsR2Pj7/IeY9LYyMPUVA5j2Z90A+2c84PgonGj5w+ZE9O1eePYcH8z2MwP09rMYOPkCOSj7Fs6Y7pWWwPcxPorsc4ZU9T7sWPvJOw70SJRQ+SsrNPYV0GT7eMbM9ECsRPj04wr07Q0w+ShIaPlqLND6Q/oc+vszbPQK1AjzgpFU9QN8sPv5P5rw8Wxs+HdYMPhwtDz6OtB8+3IPDPW2TGL5Akhy9Y39KPsEyjT30jQ88zdoYPi+ZCD5i6ze9wRNwPl/TW708ews9gkxOPSr9kD2oGuY9","bBvxPaNRRj1eMp+8UCJKPmSerT1CXcU9SeHvPRtPrT2XFAM+/6ZqPVnH3T3zfv85YMVqPV1pRj1E6CG8DG7Qve/dFD7nbtS8Vqz0vcgaGz4QG5I9Y100Prilgz2wpzM+pEBAPhCy8T17QKs9YdGWPfVHxj3iEp29jgp6Pc4eZz7kHCs9g/jQPetPsj0j02G7nDogPsd/NT6K1PQ8EMY0Pr6s+T29iYo9CoFvPvf9UT0WAVI+XCSFPc8/TD2K40Y9Nan7PZ3yzz2ie4m93q2iO4iBDj5+ag882aeuOvBKjT5ZDHA939f3PXXQpj1ejxO8suQBPmD71j0bzA0+VE7kvPtm/70/IQ48qOoKvqBZKr7liqK9ffzHveU4aL4Jp/E8PWX0PZBnM7vVWnY8xasMPfFkwj0wcgs+MRVGvZryIL5z3+29R4tsvNFOl72rXLY8iVbNPY86dr15mTC+vkg3Pv/Apjy1IOY98OzLPffBUDz9xQy+B/qDvHIAaj7nHtq9qREVPhS9qDyghMi9qV1TvtsTALzgzCu+7XDevcDM1rykphu+49ANPi40KD5FFEm+O3GMvAoWjby45oU9J8fPvI4A3r068q691uppPQ/hZz1zjuA9TbEFPpS1Kz76yOw90gR8vVpF2zxAkn89X8qevIU4BD5ReaG9foSwPafRrTszyUi9Nxu5vBqJAT4MlKu9HZepvfG7Ab3a32c+5wuxPW7WvT1FPAe9yz2APX0LFz24gz88sNOZvUicXD2KPQG9JeDfulF/KDzyiBA+OuTVPXhCPr0HKBG8PUoOvRAoM7yK5ke+IHFrPuLrGT1UBzS9ZrCpve8h6z2rvyY+ODW4OwHTDL3d86m6iNNOOoHJML2ecRa9Sb2gvfRk1D2bNUc9aQ0ePfdLSb2AJxY+VD5NPgSfUzwI5Z87KmLRvcBdXb0M4Fw9GlU6PR1Ibz2Ree89Z3jmPa7TIbwG/fS8QJGTvXDp+L1hc7S8ODb8vdEGIT3h5Ps8KWG9PYZ89rzWBpk9","/cdvPIlDqD2B7UO+epWHPfMIET6IxRE+DfOlPc8eIT6nbIc+uQwCPov8zj0qfpY9HWK/PSgAFT6PnBo9ptkKPjPdLD4eglY94dgBPpChWD6NUqY9m2vDPdmTyT0gqKI9ZcHkPebeAD6H+Ck9rEcfPtKW6z006dW9HrEMPaU1Qz1pwyY+/KU6PiUYHj6pG3s8fwBkPeo0DD6rRBc+d5X4PUxabj1SagM+6QHdPUiGFD3KAUu9nYCePkp7Gz6l9bI9wb7CPWP/Ar2x+zU9VlUtPmABMTxefYQ9nzOLPm5kWD7zmZU8f2cLPv9yxD261qQ9aoYqPik/6j09kIE9emqsPAvubD3RVcc9AnmiPTgjND1T6zg+cpocPhicrj3Xi0I8pHYsPm4qNz7I5MO8Zd6evUtzBT5uZ2o92ev2PYSmmrx95JU9zelEPd3VyD2r8bA9NGiBPSXnCz7CIg89dx6aPNt8iT2Sj509+5vTPXP4zD2rKoA+nbAkPNLbUz5/7lA+dYqQPjvrEj5KthU+WVSLPZn4ST39Ny4++sxcPjA3dD5lHL49OyeKPZkUxjnI+2U9KesHPk2YMD7q93o+XUDYPfBZez4X5029BCgCPqRks7gipuU9wPj3PM0vvz0kMU8+UumTu1pM7T3A1wM+DXktvA/hRz7VoTI+e2dbPZBebz05kHk9UPVsNye/1jxXqzE9YAUdvYLj770RHty9fnE0vedF6zpzrl49D5G8PGVYzz1sLiw9oO5XvPMXCT51dnA9jFYSvlSHmT0PHCa86fS8vfVRurxWIcg8/wulvYCber3WQpo9QCgrPerSID02t+89ruvFvWzFhDx/sP27NW26O8Z3gDwOKjA6YZtAvYGuub3qbKi8RKQIvh5I6b09u7M87UngPI6axzogXVY8nl4Nvd/jGbwgJUi+hMurPYFykzxfEOi9vwC8vW8DXr0P1rG9GbmOvDWjfL1VWgQ+lmkLPmMoh73dIwO93a2DPLhjCL1BNpI7c0+8vAM9/j3E6F6+","NrkBvkWy2rtjOoU9TPmmPUWIEr5dxGM8Q7ZEPcnDAbwiDxO+kDUZvQaI5L0UQBa9kWQVPdqIOTxNNvI9b2cVPHkLZTzOE+u9mSjtvbielTv9MJi9+v8mPZLLHr2qjme+mBpfvITYqbyPSui9Ou1TvezIaz3olFe9XrhRvmGKU7xmniM+Vf7uPPpXwr1GQsw9SvqwPSaP4j37XwK8Nk1rPPHNN70cuhO9Dd4GPkFxuD2DO4Q9b89UvujzwL3T4pw9qdGPPYMATD2Zf5A9ewDnPe9w47023Oi9iN7HvXwjiL3Y9em9K8WkvfVIm7w/q3M99g6iPet6lT1ZkmE9tmTBvDyMVzpQ5fs9VMeQPfJ2gj2nXGA+o5zQPZOwID7esds9Hi3BvM6K/j0vBME9DjbrPWBA9z0Me00+9Y0GPsMEpj2HwjU+wqSNPUzT8T3aWRA+zf8HPgFMlD3H4FU+p+wlPpQMRj5iiQs++02OOtaUrz3r95M9dcBgvRhaRj4cn8w80A0pPmuDfD4gHtk9tLcPPsLGDj6vKBA+l28tPuTjSj4sZi0+nvFcPZFvJz1Np0Y+/i2WvYv3fD4eLC0+fSIbPipBsjxrKdG9IfigO4OeQz1J+pQ9nOAIPskHPT43gDE9EI4TPg6EjT1kWxA9mOBIOxU2Ez7lnpI9ecMqPvrGfj4p69U9KaJnPpxI7LzRYcg9i9H5Pf3i9T1rJ6U9mcsDPj4SLj6N4AM+7G2gPa5z3DyD7EA+YC5tPgdTB7w43Sc82iSzPUW89TtnKM28oN0GPsjM8Tz+LTE+N+bhvYJ3rT2ayNo9iNC8PBelyz2jjEs+IwAoPmH1ZD2mYLQ9MW0TPCe/yrw+bs09yZYoPmwWWTw+ukA+f8/qPbTMYLtMgSU+T0w6PuXTxD2RcxI9aPqDvd1Scz3SqbM95eNEPg3qiT4COtI93nAVPXFfNz1ZqKg9qGyCPk1ywT1g1n891uSUPabLpT3JhvU9GAmEPR8kmj1cJ809xt9+PYgHcj1SLl45","7E/BvdJIVz22IOk7Rc33vEd19r2A5AC+G0MzvgZ2X71MI5E9K6SVvXC2FL6xfUQ+bX8BPsAVhz5nZi6+qbZhvS2F7TzQgN09hcb2vd8DIzwfK5c9AGd2vHu9ur1TFKA8OxspPQo3Gj70wps9GQd9Pt7li76trCi+Jvs/PuOYAb67xqo9vo6UPBfSQT1izJm8DsJuvcC4k73u0G69Dlk8PdhOt73kA4k9Ibw5PqsWYr1f5h68145DvhBi6T0mM2C+l/GXPYdi370K/bu9PXFIPqUlSz5lOB8+mpphPoVx4T2zuAi+VEbMvSBijD1f16g7MEl7PRO8mb0MqoM91HtyO/7AMr0MHpc8S1k+PiofqDtYbIe9IkwUvZSS1D3qghi9LS1nu9VafTsTLDQ9ZzS3vNuiETzPyYA95oNgvdBUlb2H+pG9GRbKPNcH/zybbLI929xkvdI6+7l0ebW9ZD+Ousou5Dz+NMq9AvRSvWQLvDyOs5o9WJg8PcWesTwt0W48YKarvZTMDz6UHm68lR8OvbQ10bwbxOo8CLIwPjtT97oH2WQ8ZsR2PcsdyLs5j008EPI3vddcbL1Qfim8hW5gvdGVT72lZYy9Ffi0vWIN4LyePtG9zhVPvEhezr06Toy7pdHOvW6GJD2uRZS9FROIvSm9vb21lgq9CKp4PVy0i73Wnem97VMzPT124b0nvaM7kevwPf84Rzx2MLU7TZ7ZvfcKizoGqpU95owdvFP2rjwB9Jc9MCTwPEUsJ73V0Os9hKrTPWoi3L0vg1Q8dr29PVmiyrtRLwE+SBklO2dqtr2GkiE8V0RtPLNyC7090o692MiTvfTs8r0+Poa8AWUSvuVDVL3fwOU9NMDyPQcQaj1djbS9Yc7fPR3yAT52+VA8tFxBvLvHCT0QEFs8Iby7vC1CmD4z1h4+6x4/Pu5D5TweXdI84o1GPYOqsrzyTE49+TAMPoTWpbykPgY+m6b8PW0unT2N/m49WYOnPQoiSz3wv6c8iUfQPWiSxj0S6uM8","S9/yu3Ba0jsjjtC9Hc1tPcJYOjtTBgq8r4nbPXOZXT3lXR298Hocvem/1T0/jgY+gWLVPCpvO70DzQI9KQGevoramr2ZKyS+WOSJPeGp4ryLeFw9CbwEPkLvnD1vvRO9TPaMPSYKIz2E+Ss+qiXJvQpt8r2zCmk7GCrOPWDhQj6WAAe9lDsAPVZVOj2XFbk9Df8tPrm0Gj6GqVY9lxw4PeY1wbzkhq09wwAPvl+/vz0TjHC9TG8kvWBE+TyfTmM91jLJvJDlDz74ELk9ia5qPabp8z0IkBu9lDfmPYYud71O4gS9YeVRPk625D11wRw+zNh/vZn7mz0j3EA9KKZMPlxX3j2BkdI9cNUNvsw1pj1x8t69VhnBPd5tIj04vg69swttvaEEA772rPa8kpJ5uvtf4Ds9GEW9RN6lPa9FQj6uzLI9jK6sOwuK4T1/zj89pF63PWPD6b2zzLy85P6COxchHz2fa9i98wEXvcfQzr3AEYI98BWcvd3c0jujr469A+C+POoVh70dvQS+Yg3NvZKcqbzRzdS9OVakPQ3MKD2IoDw9DC0YvN4//ruJgm89LaIqvcGUkL1qtLY8wgkVvBfbTj0GOqS9q8hqPW/8Y72mRTe9spzSvS/A/LydAqu8S8VavRVdJb5trRQ+EU87vdLVRT0aEVM9ne8fvLcspbs2l5w9Pl7KvJTzBz4fCuS95TS7Pbf1db3xoZK9aL80PgH90j1YTIa96aRIvoFiYT7G/Cq9+8IEvlWR2T2RowG+7bOxvoNFXT0x3629FIdZPuwxs7yeDJe9DZwEPrnbST3BBhs+2/5MPevCWD7YaMe9RHqAvvbQ5D15qpm9xzH6vSkB+Dt3aRQ9eXmCPn9bDLx4bC08AnACPlnHs736ymc+G1KDvR0fOj4Vmm68cWURvaIx/TnffTy+DJiUPL0PTr6viyo9pj3EPGPzeLuu+4C9WsmcPT2XdL7e4Ji9IXgNvlRHlzymd9m8d8qYvjn+ET7DQsS9RKSxvN312D1JWje+","rBMJPISFCj2otco96xuuPVtF2D0YveM9r6FRPpcjxj1hfp49pM64uNpZej0xcTM+m8ObPXt1LT0LKBI+805uPWB6Dz4UG2c9Ti4avXt4+D0Y4SS85BMFPXF3kz2YZY89QdnNPRfT5T0I4ww8pwXqPXlaMz1uR5G9iGLEO4EbXjzG4TM+HfKCPpGKhD7TZ808lFEIPnnqvTzogdw9p55rPjjDMT53yIY9emE6PdKaKj4rc8s9ebo+PkYS+z2xuAw+kSljPXLL2D0MVWK9vlT8PUmJmj0lASk+rmBCvUODzD38xsc9+TqGPa+1ZT6WBy09Y54xvTWKJz7DP0s+S9kVPdgHyD10AU0+yCzSvSPqpLsipgw+cHDrPdU/nD3HNPQ9BykBPpTydT0gErU9g3YnO41XHz5xYrU9KW8qvaj+Jb2aGMe8BDf+Pblgqj2Pcx8+4nh+uqsXKDwFWl89zLg0PY0N3z1UZo49UP8hPTci6D1K/y8+QuYjvRkm1T3IE0I9ad8nPcRVvT3sVrA9O0g1PQyR8D2JZYE9c7FpPnohRj4pXeg93oE2PdZx6T2bO8s9orSGvTxgkj1uZQ4+45mwPaxw0TgbUiK8hs3cPVtFgjyxet89gXJ4PjSY8r2u7VY+NVadve6KEz6zLvs8hwgIvEQrwT2FoRk+P5NXPXsc6D1Y4W88dGygPbeppz24VRa+e0NzvE3rt71j8le9O9arvQblRj03sgi9IkAjvpNK3z2m9aI9K5H3PRPMp7zdEgy+sc1su1T7jj0GuaY7pKK4ux+Vuj1I+MO9HN6uvTPRLj7jS/M9+cSIPc5x7Ly8aOM9wBoSvvdypT05XH4+rvfUvINskD2lBG28l1advSeL1bt4seu98sQxPYQJ0L37Qgu9FrI5vSoSFz37AiU+8RmDvaec5TyecA2+i7aKPfPOAL7Djxy94aK6vdvIar2yVTk+FBHEPda78r39EHI8DFyaPXj0O7wYVJC9VtfDPVXJSj2vICo+iII1vuaKCbwv3qq9","GJPIvD5+5zsPt6Y8x55KveyImTxsprq9E6Yuvcke4DtrSLI96Oi2uiSRBL5E2Ti9it0gvbaSo72Aw1A9sj7pPbtKOb3Ksjy+hN2svf/fAz20xt290MBSvBBD5725yio9pYMhvj1HSj3UkIu9JkKJvW2Pjz373Ts9ZrZZPSpxKb1RHTO9ZAq/vPLVMj3Kdzk9rbDLveBKVL0MnTM+KOR4PHRktL2TETa9mDoUPuuG2zyLlFw9cphBvKiXD77RNN8914K5PJU5vb1s4CK9Dt2Rvfd7Q732Vwm89MudPXH1hjwBquo8xsB2PbUDMD7rzbm8MMZNPJQeJ71Izuu9e9AEPXxV3byKt2M8r/qaO3d0fb2s+pK5OCeUvaqIUb0e27y9MmdSvb4t9b27u+295Je9vb1YIDtTg4a+lsEIvuRHmL35FJq9ygIJvjfYzL3C7+a9AKJlvR+BE74JevO9BMvyvfwv8L0RJcg8oTx3vIwCBr1wBTi+Jv/zvJmvx713/2i9+/bkvUlsnr7HMQW+WvSVvXFoLr7RjVK+hTQwvh+YU76TjwW+HXMmvn0+F7401w++Pv6UPOQuf73v+iO+Hs8XvfzeHr6iEeO8r+NBPUsPTL2F5H698GAWvpiUfLxcore94Ib/vfosHL6/fQW+T1uWPHx7N73eq9q9KUUzvengnb1kuC+9NfyLPX517bzuouC9EKR1vnyhx71za7K99SCXvSSIcr4bOta9mqIGvm9wIbz8oa69lJGovXhhqb2i2Ks9Ui+RvQTGQ7i0e929/gcivhc4Hb3wawi7bG7LvfFt4zsPawq+cGOjvQ93I77xOki9COt0vUZ1mzwOAIC+LQCjvVMaAL7w31e914g9vlTaXL2GF+C9xJa5vUowzb1Jmzi+xviTvZfT8L3fQwa+k2PlPOnlJb1s2bC9P0BAve1XMb6LWYa9r5IFvTIHfz0V5S48B+yQvd7nRb4BjgS9VC87vkrdLL3InOA8u93lPBlS0T2jS+U8KxoRPdfX8b0Vlki9","3hSEPZwSsr1DNf28oXkNPkMvkD17vp49xnkOPqswhT0IbfC9gHoSvtFHSz7bxNi8MhgVvQRC4LwbGA8+rcgWPTYd5TueU5y7bQN5PdkuRj51PWq92DEKPgPcJD5CaTC+y5tVvJ2l3L0C9FS+tIDevUA5+D0pw7+7UK3rveAOvzyoewi9dyQEvtTzRD0nsrE74XsWvazgqz1W+d08GZk/vXNrFT6fO6y9qX6lveODkT3Gi7m9o3gMPqP4u71jwc28kizJPKWBn7xTX/49C+8+vu/uub1anmG+2d0VvGPiLL6siCU9JYiNvfBZE75Ywua9pDWBvYU2Kz6j+h085RMMPVv45jrjRI08PKclPeMwCj7Lv9E9thRXvXRJFr2cpcU9wmGpPA4ej72pJ1k9vX8jPdfjCz5BhKi9fdW5vakmfj06Mx09hSpzPJOt8ju+v7k7gmUdPb1YZ7znFkS91DPSPPwtNT6oBwQ+jjxcPZf6cb2+9BO+jEiRvehqtz31ds+8/VHqPYOXj71gZ5k915yTu48sAz7AS+y9Wav8ve55x7xIlk08zGtdPjeQiT12swW9HlXQPParCj2An+A7TkmdPA6imL0yGhQ++AsHPlcHF7xMRD89y+l6PVRIcb166pc9y7uvPWRM5j3Ia+o9036pPfKd8LyEkEK71N2bujTONb3bD7+9esa4vS2YBT2LLTm9r5sZvhrQzL38yIE76OKivvQpPb5YWh27i2GlvdTKPr16unK9MTM8vpBtfL36vge+sF6MPU3pCb6in22+rkFuvizcRb6UAiu+UCtOvtpW3byz/wy9YKbqvCCrk728ggw5k9j4ve7BHb7RBq+9BT7HvVTSxr2RDJ2+PdruvX16Tb6XCsS9hVMOvrcEHL6wUSe+yxNYPRkNUb1HX7K9RvoDvZaZiz3Hfiy+UEVCvZ0nbL6VNQE8CpQ7vT4p570fsyi9jqM+vvtfhr2sp7077L1GvvF+tb0Cztm9WJE7vuSVI74sq5+9mK7FvU8LqL2WqU09","JV6nvcEjFb3VnlO9dk0tvTBsSb0icau9dLqrvckiMb7lIBi+AL6fvNB1M7zr7qw89YetvXoCE77l4qI9VqhVvvzBD765ARw94E77vflE4b2mS+49uQuYPJLg2b1/C6G8Tgc2vrmpmLw0rNm8d9YhvrNUIb5jNby9x7EMvqibgL3COW+85PA2veuFWb2c3lw9WuCPvLbBsr0mwPw97DF2vrSTNb5ik4y+p/ikvU3qlL2YeFE9Q3qHvbEhBL6Lvte9bSQEvWWjBD2RYxa+0wdMvTd1671Va/K9CYycu0/Hg73oJ1M9s/+hvWMP3D1y85w9c0mIvnJzJ71Thcu9cO5fvYQa0D0nXQ48Pp4WPgCIBT7q5wo+RdNWPfvU0T1U9uo9RSAGPmLdx7uJlRg+OkYmPPmKv70Q3GO8eyeHPe6Awz0TLDa9EwrIvIK51zzroLk9VKJQOEtSF74CxXY9jR5VPZ863Lm2WXe95DTJvXIGBTytBu48fDWBPO3Nx70RpeS92UfgPHgeEr1LFl89Cd2bPevBKj0TEgA+ITpLPQvrmL3HkOY9W4x3vXT4a70AMig9Pom7OvqC3T3fFkq9jemWPQIG4r3/i1w8Umn5PdwzAj5NjiG9BV/mPRUId7t2e2C9vv6dPTB3ez2XoV29kMW4PIt8/Dxl+iQ8exl1vRj1xTzWOCg9dnUPvsI7Vr1HBIY7OV5VPVl5kb0ryua9VeVZPZbWoT1B+7g6J5AIPnHmTL19Abi9lwYGvpmscL3hHzQ9U7MwPtEjtT2t/yI99riWPV0PZD7YGK89ZWqEPb238DzQf1C9IUKfPWCIHT7vFAY+ndkjvR641b0bVxU9vS8AvuvTMb0R3Js9cYrePKfwAD0+MmA+dlxPvOnKKL58I/Q9CDIDPlDaeD77o8q8U3vvvSwlID5BodY94hW7va3Tjb5Ps8+9O5EjPnRafj02m1e9fWnvPTG7V77IqFk8/rOWPfU/oLzQLu69l+DLPAvY2j0y4Be+PhoFPmMpIz41myo8","XwecPN0nnD0q+hU96FxnPagPET4A5IQ9tzYNPoczAT61/VQ9oE/PPa3ABz4S7t48vhrSPVCx3T0f3IM9pFI1PZd2Gz5ssp694GAEPl3okj3AN2g+z092PoVxgTxles49Ly+MPXK6Aj1PPfy9z+HMPdtUKD4rjzg950hXPmdE0rxmMiI+fTFRPrpRRD1sy5+9SjjuPYnL0j0IqnE+AiMlPgHnNT6dZKM9SH5EPkqR0z08jHG+/xrRPZ+MNbyC26w9iFvrPZSdUzwmUiA9KmgcPkZFxT0uWMw9FDSmPFwb4j0NbBw+VvXqPZAKBT5toQs9cfslPRk12T0qt/o9GJQtPYXywzxoPN09wHSXvMSlsTyOV10+DnpRvONj5T05CFw+C60mPsT9xD1nL7w7WtVyPaICID6Go0M+j4CQPSmMQr0ndhA+CAt8PZHt4DwT5/c9jkCGvVLPTTuWxuE8RjsLPvZquj0XxgQ+vO/XvO1lpTtSwA0+1AVhOy6iaD5PNSA+JunzPf4+xDtp/GQ+SQ6sPYlYDj7Us4M91UxOvAFTDD42Qy8+abYePoPdrj1vc349KrZIPTmSoz0OSlE8hbmvPR3xrbsTSq282ZHTvPWotb2/0Ko9PcSbPc092r1iF9Q9fpgEPpAFAD2OFgg+pFnpvZP3zrxw+4e86B9KPngigz3pldC9Q3X9O2APRT3x4+29ok+8vT0zBL2VjDC+5iUlPuk5qz00gra97Io4vMzsUz1iA2k9JTDyPS7JnjxHzfy9LjqKvd3ORj5wKL27kCWNvRW9DD33bPK7aHATvtwTaz1tQwI+i5p0PtqLbz1UbbQ9SWIbvk0a6DxBMi0+DI/avZnn2D1HGb87NiKjPdPJsL1isky8gDjwvePRibw1+Pq8FP+0vckVUj1Ahx0+HVTMvYO2iryJT1m+WdbQPWk1s71QPWS936aJvV1DkLu6kIk9WG0rO6GJRz0cus89vGQEPkTyQ74qLSy9IDewPat60T3yDkk8TRPovRR+GryOgt69","7d9iuGh1s7tbUno9zjN5PZCiAr5pGFY9h9aSvEecQzzuzAU+UP2WPLhCALuP4/s7+SR+vFftJz4yj509/qwKPlsPj71dL9W9uYUeO/DXcL1itG2+H0blPSzro7vILw29U5levZpqJDszUjU9GkUwvXlQhj3l+qs8M2MoPdr1Zj18KK49jrpQvOC3rT3okz29WEW6vKxJyTzHTuc8cjPGPSFfqLwmJkA5u2jVvW4dkTqi8mE9tCQ0PSSnhr2pnYE9ONfDvJGIdT2A3Yy9zRVEvFdWdD2L3pg8g5WrPPMVZ7zUTRm9EA9dPddZAT6uOo29ORAhPO1Phb01mgq96M8PPfzlZL5YEAm8y9W4OmW+9bzeAdw8YPx8vcWItr1GG5c7tubqvc9WCL4ypKO9iBpIvCYKhr3WWBC+3ef6vXZ2Kr6VCIC9DlCyPVg6w77dNdu9kJskvs4MEb4V4JE7O6SLPZ8VHb7UPns726i0vdXOm73Cg7O9KPQBvizpFj3AJMi9nqTZved3W75SPUK+YxKgPd9IzL33YpC8V0uGvrM6xr18q5m940gsvTWCZ73OUhi+EPVwPm95aL0YIvG9oGmpvTfbLr7LNOW9KeccPY56qTptaky+z5iuvdZ36z3a2Oa9qJtUvlwcNzx/lOK9JuFVPeJamD0DXYa9GRq6vAaPUzne0h++nLqAvO5N+L04ph09EXf7vZyV4D09a5q9DBmmPRvu5b3C0W28ifL8vSaW8zzyAek8MwVEvV8LMr57XAa+bee0vUhEMTwml9O+09iZvatFlr4tuzW9pEIVvqXeF75VIjC9mE+bPNJi1b2XXO29XwpYvV+SAD1QrQO+/74/PYeaxL1ohgC+ZbxIvbEWirytUnS9Cr8bviyKN75jjZG+EUK/vRvG9r0oRRC+scMwPXRHDD9weA6+/XQtvu/8cr3ZxOC97XIsvmWF3TwjRUq8mFUhvWG6Ob7vhQk+9DeqvdCrq702Vcu9AWb9vOnpebzjHw095dAavVQLPz0hmww+","pRq1PV7aN728DMU7116LvOzmXz4gkSg9etOuPVY3Or61DAe+uLWcPZ78Lr2ZJoU8n+fkvXtMlb2pk0M7MS5KPvEtED3Pur681UwjPvkV2z3UvxE8movoPWeePD56h+a9sb7RvVDxZbvGdjE9nWVevf2f6j1QbvM9Z64HvSR/pzyir5q9yDvbOwWszz0JKTO9C97PvKoeXDygEso9qIgFPVtNEj0qk2q9ahwTvmKASD04n+i4qx4dPsY+Ar6UkLE9rSDUPBWzQD7jMwm+38WXPH/Ftr2nJPy8BYGUvKv2xr3DmSM+GGOyPf6FeL2YnRw9EVzDPGCGez6iLMw9CYNYvKehcL2jCxA+pyOGPCYAHr4yuc28v12YvJo5Tr9qLhg+v+zgvbYfG75etze81Sbfve26871VZdg8V0OuPU3+0j27Pxs9YY4NvgsIwLz/OoI99qC5PaLiDj18bWK8Aim1vlzKoD6m7c++65RsPTamab6Se+69Y71JPQWFDb41ABI+3/COPvw1Oz5sSBm+gxu/vJ3CbD2JXJw8L4fVPJxNDD4bqFO+mP0bvXbmHr7r9ma9/aT4PfaerD1tmjG+GKrDvDf25T0XBBU+K6nevDf3iLywsBa+5QMPvC+rAj4A2Qq+YOSgPTlHdz6a7bo7OHxovT8oNT4wAn69t6lKvlKw3z0CBAg+0jcXPqJOBL6YxAU+4OEyPqOvST5LTjM9IP+kPVGhmT3nY9M9M1LePVluwT2FKmQ9qgyjPTefej2z8iE+c9BIPowHRz688FI+lIWCu+LQ8T0OHZM9p1y9PWCXBD5UCvA9/sTNPa9xDbxeSEM8vQAjPiT+tj2ykKQ8erYkvYam8z0dgHA+yaNPPgLl271SmVI+hHgJPtlTLD41C3o+H6MBvSSDFD4p8qc9roKzPStMhTzha4Q+LlRsPa22+jwefCQ9/xXnPcsCDj0wxYa7q+8/PoHL0j0c/TI+OkApPkFrkD2lAEq9AvwxPn9xtDztG649Dp4IPtgEIz5KQwg+","2DVNPj6EJzz9Sww+LQZ9vWxQqz29eyQ+fOqAPZ18Tj2YYjY+XlguPnAfDz7r1HC9i81iPjJf/7sOtxg+ddJwvXpxEL1OCwG9/nwiPVBZEz6zcgy8IY4+PilbCT5rOVk85VC9PXGDRT3ll7Y8IAzNvQX58j12W6W9iYSoPS+ZmrzdkWU9VjBfvE5GaD6/CTo9m0MBPbbm67vsgRo94YhkPsmamT1Cvn49m1wSPucGOz1QRNw9mqi7vQ21iD7rlus9dbSDPZL7nD1HWNy8Jk6OvOgOhT2bvmk9ZMTmPFRezT1JbL08CPnoPZj1Hr1/i+M7LZeTPE173j2dqDw+xC0BPlF70bw7wLm9SxNgPdGXpLn++Ze9zIz5PCFqKb3/wm49Wvskvc1oOT2iORk93xipPOiFqLx+Upo9O/4DvLI6ED6S0X+9fNBGO+QnVT1Ysju8L+ABvuI4f71itki9/8JlvX6tNj5+rMY7hsU3PuAKSb3XXYk98BEdvmJKir0fmCy9ZXYpPaa6LD3bz0Y8WtT4vDe3VL0c3j69+qalvdtA1L31MhA8a4y1uys5nj07z+c7i587Pe7uYb5uuMg9xcynvdW6qL2yGNC9U1EWPDa+XbwhpjE5DzuSvdlnBb25izU9Vf2BvT76Jr27Geo72MZOvaKNkLzd2Mk9xVDIPQ/8EL7oXg+9iSRAvUlVhTzPjCS+9vTVPexmPT2U6n+9TN88PAt1iD1yQcO9HvUOPDu1kr3P/hG90rFOPdN/y7yOqMK9gpGEvYjP9jxbR4a9eRuivQeNQr5voQO8V2l1PVfQh71REWC9Df2PvIgNFL35PN29dgVWvd+0GT4cvIQ9ju4Wu6KWADylsT48elzPvTj9X7wl1CC9VsGFPD4oDT53A1Q9FIbMvZjYdjwTF9S9YPWbPdKNEL1MhqK9bbP6PQt2Gj11ysk8uc02PZnicr0va7y9uj2Tvfmsl7tpb8I8+OOPu2quIz5w1DY+478QPiBH172Fmiy9ueR/PYV7tzz49FE6","zFjTvbATGT4V0Zq9zNamPKpcfD7wMLw99SMsvuXGM70VB5E9AGWbPGPJjj0k4sg9xI0IPmesvz3um1E+fI2yvaQKoLuz9u49HEqVPIBFtj05GMm9/xgePVm6lz5Ba/48kRgAPpGeB70+2Xs7sccVvnl5/D0oAdE98hXoO1SNkL40aQc+X5ooPrcTlz2PsEC7mXi6PXzQpT5GItQ9AhjRPNqNRD2b4S8+Brv/PU1BaD2DGtI+2CdbPtn0AD68gCw+YEAdPdsGMT5cqiS+k2CivenoyD10MAw78vaDu8dWYz63sPA9tugIvf/ghT3aQHY+zM27Pt4EcrtVHlU9osAAPu0IyzqUhwE+EW+5vT6R1L3OjhQ+f0fAPY54qD1bSrY9sMWtOyafIr1sLYs9Ri3evMSunz3M9ys+Ic3WPfxtz72lfY89sN43PgOkVLypF3A+vQcavgR4yD0pOb0++O7qPR4ksD2N7fw8Dm70PRQbk7zh3QY+7bHzvUvj0D1kC1e+KFwkPpNNkD1Guio+s/JWvQSbFD6WGbE+FOsovliUVj3BPNg9nlG8PTZFYD3UMpk9fiSxPeUdnT1EzRM9C7LFPWHVmz3uawC9SUS8vWeGRL3cuCU+CMYjPjYLgL5RyTU+BEgIPVYyMD1Yns495ZrDPe0XFD+6riQ+JfYzPLtXTz38Bh4+BhkAPZeVCr5EPjG9ATqjvfiHIL1s9zs9Gi0BPkfxUL1SF1e9cyIGu7JFu7zXgGo91fUEvT2Jgb2vHQ4+ihJaPXnhKD5QDkc+kMaSO/gmm73tUpu843qsvcYsNb4bMAA+6kl5Ovaw9z3vBca832gBPs7kkz2uCaq8FPEfPsJR5D31Rjc+kgtNPU52wry0QzU8W/Q5vsPGjT1QyEe+4tL5PMUOrjwmdxM8cyPGPahZ6b7rwF++FCW2PVILqDwOOZC9RQyJvPGA+rvfdR+++U7fPK37Db5jV5o96zsPPszoDj3VERQ9ttXFPbkQpL3MLvs8bVFsPbBF0b1NatO9","2+29vg0yG76YBDy9m8jzvBbRirykVoW+9f96vmHP0Tx5kB2+C10HvuN3h75/CBi+o8v9vP30Cz0gbC2+JnvGvKbXBL1i3OE9USeOvDPzLr5ugPg7LD0tvfNbjr77kGu9vU0vPWh7Vb38BlY9aBJcPJTR+b2yA9Q+akO8vcHORz16UwU9fgx3PSHjIb6dAf09UQKLvBKzBb3Xvr47ZBMEvR1UGL4p6Ym+bUwtvmAggb3xtvU7FPG7vfMk8r1PS20+NyVIvV2SuT13kWK9wwK6vYCYQL6mf/A97wlUvtqmkL4h6ZK7xdjoPf93aL2Rl8++kTdNvSpqDL2w08O9eA0CvMCuxj24v5w94HJ9PdCqTTwDSRK9prCevdNMuj1jqLy9gzhIPX3Qk70Q4iG+o9+5PSph4j18wWM93wKBvA4bzzvmc5a9kLpyvB0IRjy1d3u9aR3rvTLY8ruTWBW9lB68PbNICD1rbte97O3evTpNgjzOj2S97uNxO5Zas70/ya27b6ivvaEbyrvN5UK9am22vIJNozwl2eW9rL+JvpiRuL2DGny9mHvvvRUBLD1iKQ89KD6PvSHncb44Jr+8AjWkvJ6wk7vL0qg9ohV5PHp2Wb38i/e9zrjTvS8VXz0OEiG++Zemvc4Y3r2ajsW9KNMpPnG7NTwD94q9rS26vUdFPz3bYQS9BpPVvVUDZr2YOcg7lnoGPYEFaLyhRYC9lIPoPcSZfj2Q4gM+HGh3vWyYhr1aXRa9RVDMPTz4IL3z2CS9vDiEvIvvJT0ZSlG9rwY1PUsbNb7iL4S94ZHsvNEyLb2sSrq8EhN/PSjUZb3JOhI+kZKwvQJDGD6KoFo9TwDjvUmQK7274a29NPiOvd4/y733w9+87DNXvjjuZr4QKUU94EOKvdrFMDyDsuI744ZOPL4E/jzXz8a9q0/ivcF7o70ReC07408KvqoRCzxygcu8a41uPPdg0b3Dx889cs3hvOuuGLxEJI69gcZFPZQjRD6AywI+xIMXPWT/jL3ah4O9","9fxVPcnJCz4OBQQ+F6ykvDeykzxybaM86rgqvRcyELxfqaE9XDZYPRuI5T157U69LFrQvYIKKb3VUjm9TMIPPa2Zmj4Jh6Y9n6jKvfyPyb1Eayu7GzmEPa2nIL35N5o9SKp0vEmQYD2mOSs9CJRJOyiaAL7FG+88c0jcPTnsrD1R2Mm9JPQtPaAmtz28JDq9+5ckvsgDfTxlia091rQ9PTO/s70+4ma8zU9xPCeXNr2Er1q9irp/PfustL1Lf5e6QI+uvWLZhT0zvBe9DWI3PfoAa70b7rQ9UXTUPYoSDr3GbSG9l9cFPsaEQr2ESQM9KMR7PYMcvD2114C75FpMPNDOkb1FqL89GUcdPs0vSD0F/co9+o5RvRTiAz7RUjM8YhONvJGkdL03Ao89/3jXPKSEYz7vK0M+XFuCPUjTirxpaKU9FZY3PI26BjwY3Vu+r6ttvKh0KT5yQHY9L0s4PgWrIj5qG3M87pLwvL8GIDyDVi29ZfpWvDf7tb1CSOQ6RulpPgOWtb0Pasm9AkK9vdXce7xJOzK9QjAXvKjAD74AmcI9MGsOviNUDrtFDC4+QGA0vv6CJLxfr029V75+PshLxj1GNIE9A2I7vbLSEj4uwBK9SWgcvfYh7zyBHLq8KXMdPfaJ/b29Zo0+iGSTu8xT/j3A5PE9mK40PV3ktz1STYe8NC8yvQgXir1La0O8FXZ5vQK8QL5UY8O9H1fNPXAhZ75s3yK+7HYOvn2Fqbxoqfm9J6Yxvoqr8b3XYPG980AWvnyV2L0t0SW9qYgovWwAg77Qj+i9qCjavQdnGzx8RWK+xpUsvcdky70/yYy8dMTxvLaiFb4JSrs8llG+PY9ERb7ZvsC9lOjDvXBIUr3qqQq+MPeAvlAkE74iqIS+0MyWvY/j17yVndC97WlBvnVpyT0pexy+b7bovRirMb6Kj/E8nStKPcOMiLvbNTG+hv33vOQEgb3mYgK+TRoevkXwqT3sia08p99Ivqx3b7yNk+u8dD5GvpDoQL5mxAW+","VTf9vNyhkr2Oi8m9xBvmPTJxCr7ukKS9XcjCvX/GYr4oknO+eAIYvnGkLL7jzR896twbvhxceb7gqlY9i3YmPcAsBL3NyKW9aXULvgO5c73y2UI8taeTveao2720GAC+7d3jO7F77rwjAM889lcVvfGCzb2qw968qfdavjNDSL0XzDa+XIdvPYLvBr61lmu9CQw5vrcTh71sxhC+t4yPvvJvkrxii+274UDlvCplMb15ZWw9Eropvp6wAb7Vsu29j5QpvBgUvbwZRNK8U3g5vQf/yr3VVra9VRK/vUQECL5N7AK+P7QJvi7/jL3D+lc6UFcevvD1cb17wdG9u0zJvVnOYz0xcaG9rVmnOuLk7D1wm2I+NFj9Pem6rj29OoM9c85VvJFzb71L9vk9SSYBvPih+r0IsvO8ovW4PR0okD0JUNG8i1iJvZjumz2Z1qm8VnCVPZ3+Jr1O6Bw+gqWCPdJZHjzuvha+XH3Jvam7O73DkwY+CgqvvJPE172xoom8SEz6vZwLwr3Q1g4+qENUvfz7pr3Mxmw+lC8avQm3GD7eASY+LROoPPUnTr2fMGs9i0bdPQfa6D2+AN87HigUPrBcMDz+Tn68mBwKPo4C+zwSLUe9tkuIPb186LzMeE69ZIvgPZNwbbyvsm+8tnnXvKY/uzylhHE85/6pvedXpDzaQA0+ZgiIPf9qaj2bCUk9fXq9PUd5Zr1smO88Wy8mPVbsCz2B37y9Eg+8vRCB5zpWBtE95Jc4u3skjj02eWI8H7UvPm/r1T3whQg+1bH/POG6xzwVseq94jjWPcWARDwewMU8mgWTPLmDmj2udMG9ty2KvsC1ibxqV4W9ivAGvowrgT1vOYS8W2JUPU+F2TybD7O8kZ8hvaJXbL6/N+m90axPO4hTDz4B1wg7tVSNvVH7+j2GKY885tqJPXTnIb6si0u8C/5RvPUlmj1mFpA4gg7XO15vjjxzAcc9qTiEPcaI8j3wfFU9GVXCvHOnkj03HYq8/a99PdVDxDznwsw9","+EWAvfX3Br1Hqxc9wsNrvMm1Jr54YO29nqkxvm3Snr1RiDO+LnY5vaSOi72PreG9a6//vSlHx70dLDs92mLAvf7H2b3MEGi9iaIave/PsL1hd3S+a9wWvinPDL4rERe+K3gtvJQ7tD28FK29l+svPa6iULviDZA8GBF0ve0sEb43hw++FfqLvujkFL7VebG7iFdSvdibPL5oHQi+Ylotvlc+mL0P3ri9tAC8vWgpnL2mytg7rSz+vQpe/L1tgJO9caYJvfv2Zb05wdm8jmUPvlZkEr7pduC9KhuwvRzsOL7KTzO+Lm7tvWG+b71Y10O+yFf4vbYBB767E4q98/+VvSOHPL2TXpW9TIQ4PO6tSD3YgDS+329Gvt11rjyN0ei9p+3TvQ/FQ726wOC9PQkNvuAJMb7Y65S9C6rKvbEGoT1Gre499pg1vUx4ubsdBGe+e/QGPf+O2r1O4La94bnqvf67B76tTI46kT9svV60rr0mGhO+Ns31PZlLDb5iayO+2npIvZ/sSr1eb1S+Q9fAvQQAyL26kja+XpdkvZ/STb51OAK+Fk7/vGDser5wQ8m9P65ovpN+CLxlpTS+t9I6vfBBHz3GUZq9nJUavAQxlLuiNSa+tZ4tvqnFtD18Fxa+aNOJvac46b1oDiC+Be4hPtQqPr4s7pQ8bbJ3vYpnr73Z86099K0TPXB397z8KAA+YZh1PZ8Hsj05jgm7b0ylvC8NjD0qKkw9WPwbPiazvr1oh6G9K2uPPUv6IT4Layu+PzcsPRf5ubyshM89QnLBvaJ4uz1Qpsk7TyCjPeroCjyXHDS9NL0MveJUqjx5FRC9ShPTPTxDwb01kyG+7QuJPd73R73HRwy+zyv+PYEhgj27e+W8p2RRPhb74D3YhE48d25vPdBAzLyZK7q9e8c7PcbLDD62mZE+yCjvvD1AhT1SfS29jecYPfDXv7vjjoq8WrPuvSixXbzQ0BK+zO7lveIkaz4GLL+6X2UuvkqoibuRiiG+7WZOvJZLzz2XNFI+","3bI5PoVXj7zJ7Ko9/pHQvMdokT2C2uE9LPa4vfQUKTuJHgc9UZSGPax+Y71WMD68WsCRPdArG7x/UP+95u8+PVgB5DwUEu49yXc7PUrTGzz1+pc9YPJ1veNigT2JElG9onq9PePomzyVK0I9TDDtvNEyC70Hb+u8iBVYvGbyxr0OLhY+N6IgvOC3Gj0oWF66n2oOvWLmYj1q8xm+FuuEPCyeG71tEDY9z/flu7dYsL0XZlk9OywQO1640j1IcLa9L5XEPC4lUb3En6Q98DhDPJjbhLtFWgA+yl+lPeRZPT2vXm494THSvTemyDzN6dk8X8gfPt9Asby70/U9Evg+PXmcxzuZgwi8QB8qvQXq7r2VGi6+Ij8GPRg9P75xXSK+20b2vcahFr4CCrG9Ugf3vbOCDL2J7Gu9CFcKu/4v9TuYsUa+OKnMvbvEe74GtDy9Te6dvXPZV759rOm9K9gyPd6NdLw7oRq8EhLcvZ+8Kr58Ahu+BkAfvKrRSbxr8Y+8uE0/vmwMgb7r8V++MsXMvcLHqb1tYDG+hHuAvtfrQb4Mr+m9nB4fvUEcMb4En4O9Kb6lvSsxf77HuEG+XDwZvgvd872AqRO9ooUkvIRr472NXDO9h3YxvY4xDzwTBE6+HDyUvReaAb5DKfC9YucDPaRpDD5Nvq+9jfgMvmjdJ7yshGY9GjvIvZyxvL0aWU49jV2tvQzX7L2EqRG+8JCsvWjIiL34RzO+adoxvtyYH747W+69AthkPeTaHDxJ4r89WQqcvY8mz7zeVw6+wtsVvm8WFj4geNi8JZKCvT8HCL6D4Ma9S/nevW7AOr352uo8vBdqvmwQ9T0ABUK9hoF5Pcy4RbyNF0m96+Uevj+kIL3ChTO8txSkvZ4SIr7e5VG+d8XZvZnzq7yi4pO9UBAoPTnnGj7HWuy9s/u2va1Hpr0TSs+92nnQvQfaGby+Cgk+BQOJvQcrK71IRp49FHgBvlbJV7yj/xS9uvzxO67LdzylXt+87SeAvvY6NLv1sRy+","YS9rPLPlZT3jwDw94IEPvUTbeT2fyQ88AAcNPMe8mz3SYkU9YOVLvjbQgz1mwAW+0TsrvX8utT123Su9FJ+CvFioXz1JiSu9bQ05vSXuzj3zvyu+ljPqPEhkUz2ccSE+lEHpvPjDsTu54c69JLGQPcABYL1GT1a+8oNCvUhEWbwGJPS9b9WOPU2Moj1rVB+9kjmjPLsRLj6DK4c9pvdHvWWEo70TBho9/TcnO34zRT5Kz3G8ZLdrPgJOOb51BTe9zZuMvU+Fjz3dSgk9n3ewPVYCzb281Y+9hE/7vQZJm72t3d89gZVzPQpz1r27oSY9r2HaPRGNwb11Ej69erAWPO49Jj6PTCo9Mj+Uva6+mDxtd8g9unbdPT/AtT1Q8GI+Te+TPODzs70z8KE9SrC9PTCjz7v/Ido86rHzPYB+Zb0y/qA86rsyPuLljjty4tu9OW0uPhSkdL565f096hGlPXw+F70VSxY+zRmGPnp/cT5YAyo9pW6kPe10u7rRN2y9VsQVPQE1A75qCjs8c4wFveejdT4JgDc9rk8zvo0lTj3zrwI+0cZ4vQhkJD10qwq+WlCJPIrehT1+k8w9UFguPUNzDL6sM8k98FgOPQ9YXz12rI48YQiYvt4PJDyH/as9tTABPn4I270PWe27fsBmPS9QMj5814M+MM4HPqSiTL1GVtu8scb5vfbv3z1IqA8+RdMLvvQFOL3jxL69NYKhvdnZyb13x4a+XJ4JvvgYBr6TgRi+hhAmvPd6xb19fIu9gUPtvRPSBz0b9a08T12nvWg3Sr7hqg6+4LT2vThz2ry4qQG9SuNAOwcGWb2siUA8WnMxvilCo7wo79e7sQnnO3s0rb5HFG2+r3wcvq1CIjtS5FG+h6yTvmFsUb6+Fo++EFWGvY2KXr1197q9qDqvvW0n+D0Mbr69MZ9DvkguOr4uDQU97P4Xvpbxp71RQ9+9FL+IvbJr4L3pXlm8hZOAvm9IO77CUP+9LadjvhiAl7z4Cy29yaDHvcoUTr6V7eO7","0B8WPZ1oXTsuHpU8FzVjOwW8Lb0hMLG9wLVnvWmWBb4j+GK9ANsMvsDhhTxfPYQ7/SDVve9YxL1dOfg8ZBEAPFizwLzsFQa+Jlu7vZNNZr5fNiW9swucvfTyxz0g9Ry9DLhIu6+jjr3udxC9UyHqvWNqC76RTZo9ZPErvomWdb7a2zG+8NAru9ESPb6VhA09HvGKvVZnMb6cdL29fBOGvhkxAr62N929YaE5vlxDRzyflfq9M9mavJx/Ar7Rrjq+zmkDvpcWl7z7Pj+9EKjNvVm6or2RBSi+w15RvRCIMr0kgpy7rldvvVDV0L1qDwu+mCUPvucGYb4S3Ba+zc+kvTx4Oz2F2oG9zQ6fPGSpqT1/UzY9HDURPr5QJj7I3vQ8J6uavcw9xL2ZMc89LXoHvvQ9Ar4NbEi9CvyNPep1/T3xLGw9JDPRvNRGAL20oXc9J5QSPS+w5D14yuk9ioyZOpajn73H1XE7GoG+upWe171eIXu8HB/DPefTVD1gL4M9O7pzvv6Gtb2b5ck9HIzRPObfyLwTKnw9J0xrOFH4mb3/HhQ9BVsCvSj1+rx1TaU6yu4EvvyTWz7ItKi9ezUVPSa1CD7E/gU+XD++PGCvsz07t8u8y4UxPWEpGL7g1Ju9wj62PabjmTwEN0a+DR1KPOgiab6x0Iq8bRGIvcHzQz2Ci0g9lftQPKzkqz1tx2e7gm6ePTaZprxHtMa7Ob4RPu4PtjzLHXE8kG9IPTV+7z0PXYO92y2HPR8bU7v37yM9gvykPSARcjzTqwI+iLuYOvTFij6UOSW9p8dxvUwx6TpadvM9LTK6PPhLrjzr1cM8uN+qvOkbUr0rVps9cmqovWGoBz2YaB29vI2aPfhJAj44vx29gRXpPY9q270eiw09cXXsPLNuhz0/eaY9iwUTvf+6vT0jGBW92kKRPMqtGD00+w89v65LPPRsrj2Da1G9w2M1Pfpcjz1WJXo9KKOGvNUsZTyjmY68zg35PJ4NBb1eygo9NTVhPAk2qT3+Gz09","6WcNvg8IMT17TXE+bK0+vs46Bzw6P9u91ZA3PcMaI77Xss+99y6JvtgXHr5znUG+g3gbvg6NKL1e6TK+iPWmPcvv4r2Dp9W9MZ3nvcdXybxULm49/7IzvjrCn72mSCa8/s45vhHtPjmch5S8Nh3KuizQB77Uvfg9yAXUvPD54LwyIU6+O3FtvoPfEr1TwJu+bcOWvbxaTL7xAjW+EjClvbHkkbsKGVq96/TfvErpA76f+4a+I/wbvtCjBL5CJWy+NZUevstn1r0i7LS8UXJhvan/sjw3hdm9fDM8PYYZor22sLw7RsdUPHFFoT3WgyG+birRvTM4Xr6j5Z+9UUBNvnjoKL2WjJy9tekfvYEgJb7LtYW97fvovd3fpr25YMs92ZbOvWeBz711MXM9aWMlvly/0L0FjZm9jQY1PfsqFz7UV/c8j+skPnGkCT6FQSC+PaeEu7COIL6A9hu+u7F/vaYmsL0mGNo8Xm3Xvf+srb0pGoK9Esj6PU1c7L1fitC9z66lPV9kUL6dJTm9M482vnw6v72VpOO9DIkNPPahE76Azpc9g367vbxNd720MSi+YZArvtwg/zyxq5w8my9NvTtge73z7WS9ncIRPk9Xqb3AZX+6bcXWvZppPbz4TP69rkwPPATg+LwD6B4+6BoFvvgL772F5A+9myEgvTDGpL56eqW8pPUVPp3JqT2bL/A8pVOQPChus7xaJvg9owiZvPUb57yvcpY6smOAPQusD768+Z+9QFCIPcEL9j3kEwK+Qd7ivNhMXr1Doaq9o/1sPQkRar2RQDk+fKtEPX4HqLyDDQa9KwJUPHCrPr65Api8kHfQPL2/MTrTUkE906TFvJYqvL3iqkQ+fGQhPD3ODD4r2XW8mtQEPhGWgr3mW4M+7kOTOwyHbb1ELdy9fvXDPfzOY72sh1E+zRCUvTXyNj5AIdU8CQQLvW9t9L1tdBM9nD1AvY9ahD0x1869D9VGvZVpnTy54Vs+SLF5vS4cD75g3L69qHTuPegXA76Q5FA+","CReMPqEplT5eQaw8CbGZO9zmKL2aJ0u9x5sLPqzbMz0iKFK9VKAkvX6xCz6VOKs9RMshO/7ZhzzACYG9EeI2PJq4cb3+JeY9doL8vOYihD2JXBY9UwanvW47yT2oG129PdOFvYbEFL0KVg2+AizVO3FKx7wfiV8+SACOvtyrNb5ak1I9rcEfvq/wFT3rcc29F8VPvhNsqb12Ote7Zav9vJvBlL3HAz885Mwiu9DRebxJo8e8AQbbvLj3u73pU7s8SminvfH9xr2DHCw82pE+PaSngz3IHLC8+RGJPDsIhz37a9q9mfEqvhPiZ71eSoC9rjD5vfYun73KL9a9TomdPAqdur1QHTi+aQQbvlvNWb6InJi9/vg6vq0CljzmvQK96SYavdLoCL4Vu6+9kKvkvUd8kLzHNB++Nl8OvYnvkb2VYxa+4+aOum38Zr1BsO29T4wBvsUH070WOZ+6xFMDPLD61r1bj3G9nsAHvRyghr2DLU6+8KcCPcwFAL6fkQO+Y6Ayvjtdpb6uQGC+irwgvb61273wdke+V1hqvozLhb7xkoQ8qp8mvhqdAL4ag2G+fCzBO/Tfab7jGQK+9vRMvrSOBr5FsLO9bAnZvcYBZr3lxN+92kqLvZ4m8rwXUYa9YxOJvI2+bb0xKSy+kvEmvZ0Bgr3EvCO+bCmsPOpKEL5I/8K9gxIuvdFPrLwdegS+orYSvrsjIr1Tqas9SMpNvV1ObL03r129gbsPvBQVRTyCMLS8gE8hvqupPLtjMX28/MwRvbK4zryNKp07g2gLvhS4dL3dqwm9ZiMKPAu4i708k2e+abnZPMfZurymZIW9VNDmvebSlz1zJJu+iVCgvXXJorxN7SW9KsdNvncTuL1S5kG+lx7/vfA7uD3zwSO+4BEKvjvfar3j1Tq+napdvkXmwr0UKYq91MTtvfUF/b2BibK9U14Jvbip9bwCs1g8gNFxvb6ZCr6eem29j7+RvcxgVL3lHfa9zPdRvZiKTT23usW9Y/TSPP7Re75xuqW9","lLcrPOuWw73npQ+9UEcCPkfueD5d0jY+F6RDPpULhD1Jrk+9/5wRPKIZ/T1R7mq+n8qovasivr1fqtk9Y8hFPQH2ED5dYLa7vRS3PVgADz73oKy9NBDkPDEHXz3KNYq9cwPEvP4X2r2cOf+7Yth8vjg46D0w7IC8u9ngvQFIlzx6KMC9CI48Pb6Dyz1fzZA9gZWVPUybKz48Vq09WGeTvH3ICD52uMy8qvUBPDitFj6m1fS8YATJPXr2b702s+A9AJ9WPX5QHz5Kd9o92EBEvv3YBr7DEiC+5xa7vf5jBb0v7ww+LNYbvSChor2S6sY8nYBzvctyCT1KnFk7K6qxPE/G3T1tfLO9rDy7vdEwvjxzlR0+5+zZPY7EIr6Nzs+9DunRvamb6z2itPI9vpMGvTUiQz0ZhN482fUDPndREj6Ypgo+X9IFPqqb0jtD63Y87WE4PpZkmb2we/4812rgPElXrTwj2Tu+OG/wu/CjGj2RrjO+iqfTvMs0rz07KjY9EEOgva7PVb2yqpa965z9vOw2gb3uKaU943VpvrewrbxmqaU884+vPZuUdL123u+91uuWOpaqn7ypCtC82mzHPH0Vnz01tHa9H88AviBH7j2Hkqy9sk2IvTKBVD0Kjdc9fpSBPW9GD721Ri89kOOwO/J7Mr2Mreq7uKTTPc2g6z126yG+Q8X3PZMVxr0aviA+LLFCPvADJj0sGak9da2lO05ZNj2guZ09c0DHPXzmRT1Yi2A78iO6PexcubyMe9Y9ZWFSPiLgprxoqlI9BuKTPUH1GD71Y2g+HZy9vRCnOT3N77A9q7bQPSQYEj7fm9u7mQS6PebEfr0NVsE9Ov90PREPYj4X3Ao+mKu2PcoWIT7bumA+fQYyPj8YlD5srCQ+szWsPewsnT0oMmI8dWmwPNCIZDzFroQ+mPeYPYFu2D1bDGI+hr4JPihR7j2i/0w+3GIpPY28az2Ecss8QOMbPq79Dj6Zrc67P9CEPRCH0z0+tMw7wZdIvLJ9jT3rdiE+","trGzPVQ2xj0edVg9USm6PfLagT2MKTc+A7DCPfWnsbxGQek9QsSrPL6ggT6o02c9BNEqPk5gID4GW/u8w3YqvQmyEj5se869TkiBPYrwDbwXxGY+WV01PrFdUT0hkpO5F1UPPlOPLD2KGB0+AJPgPYoqkLwECKy9DmcZPm5fWj6DxW093xwuPmCvi7rw98C9eTckPqdd2D1Kf+I9iSAiPqDz9j339Ks8+Q/FPQhpfj3llUs+srUjPRf90T0Y1aI9LqoOPFfebD4eVI89EE6WPrda3j1eLXc9KOBGvb5UHD7z8OU8TS5KPgPcErnrjp+9rrMOvp+fjTwiCpE88V0yPncdA74f0Fm9JM61vPHwAb35TLI7SPGcvae06r1Y8Nw9b4VaPVbF0rxC90e9BJTSPTn+ODw/7xO9NOL/vQdwJ7mNOBq9my63PbjPY7w8V6K8TN6MPSZxJb6UVmu9L3KBPe74ej0n2aY9yvXYPc4VGj3CjDW9PxBgPRTQHz0dge+8PvkdPt+D6L3idb69jskaPAKBUDzx5qG97XAJvr13Pr0XRyW9WAfGvM0jbj38IRG8Fmc2PtB3273H7D8+GcAlvoNIaTtycTC+qGwsvaJjWT3/RkY9iUKhPTNUgDznlJA9USw0vD+gT710JHY+XhamPabKnT2Qnwa934DIPT34Kb32pU4+XwVCvnA5+T2oFcc8bwyXvU3HEL6nqIs9RzQFPsTKwz1yTSi9GZ6TPNnEcD20Ot89azkHvs7ZNDyrA5k86nDNPZmsn7xGxoM8gL4IPvQ5Sz18CiG9cL4AvfPlkD3mn569j/HwPCjpaD5Qz6c989u9vboc6b1LdAS9Z5YQvqS0wz37qSc+Ez0iPuf7/LxmVae9h8uJPVMj4D0D03c+3wa4vQCQHD5X5qO9ApgbvPXvyj3qKtQ938Y5vY/me73AMbc9YlPGPRVoQz0P1Tk9HHtOPsy/zL2DSpW9OfDFPfptebtNz/O9J6HnPd/5Dz7k26m98sWMPRzx1T0dNPG9","qXPAPRLJ/rvYt9K8LKkBPT9adz6Pr789gpWRPSyQZr3mYHs9CJEEPmGk7j2f9V0+rX7xPLfR0z2lYHU9RE9ePZysiT4z8QM935lvPQyIUj6ptQY+3/WrPa0plz1JTY09HQq4PdSz3jzuAIQ9TVgKPDMuZT1TgMg9OPrEPfaQYL0HXUk9KQODPpWU/D1+1vg81hCau5DVcz3kn1s+gQMKPlOanj1WhjA9LUQRPocHUT0mYR2+9KsePhRHMT7jn2K7mHfuPapwKrwWQk09qvxrvH8hAD4s8qE9/FKWPVIVUj2BiIc9DcrzPB3rMjvYViU9KKX8PffuDz5t5SM+s4QPvXx7iT2uIKY9yb8/vXPy7jyfko89xNyUPd33qT2+K4k8mMQTPi5gSzz4Rnw98qqbO3n+kD0Q3fY92OKwPVf4aju6/KA7IjiuPcgM/j0SRSU+R5NzPca0Vj6LUl494tubPOBiVDxOT6g9rfiLOyc6lT1hUmA9oDNsvQ4WZT0z7Z09oFmvPULqUD1QL7s9XioIPAXHfz3gjik+T9pWPZOHdT580Mc8QuzVPXxKfj0gWs89wFQGPPnsKj7eRz0+XngzvcL9mT2pOfA9P82uvdiQVz3CgaW7XkXxPa3KTD0WcYI+IAXGPfpXw7vfXzg9zqEnPWIdGT1zOfw9+tWqPcWJZj5C4Aa+IWLqPVNvnbwiiA6+6NEJvoaKRr3UdK29UNRVvcOlOLyCX4E8sOghvuRHqz2HZDA9rZWUPECa6L0sw5a97I6/u7X84TwCYoy9f8cDvaxslj0jK4O9h6yTvn+8dr19G6E94KjvPay9BL5qz4M9aYtIvhg0Br4TLGC8UrmYPChnET7OLZ88N+efPPLUtr2mzPw8kDYRvtD4C75WILC8PQ24vXiKBD5yoZA8u/8Evp55PL3f6Nm90NuYvFI+VL3/CRy+QOkfup9mzL13Kok94FWVPbRitz2C7hk+tUwmPjFF+r3IPJA9s3FQPgGDkDyVKg8+qm1ZvYjohDusktK9","cTjgPDf94LylpYM8E8FBu6ONr72RLYq9Q47huy7ZSLxwjbs89ZwIvt28jbwoKJ28DqR2u7+UPb36v7a9Id6WvUa0xT2uDnK9EAJmvfHzoL1fKqq96COJvUoTdzyFaQO++YrCu60nCj4K3Ae+TuU5vU2enj3f4uo8q6LHPDt/Br2V+aY9SAEZPJrNQTzhxy89qBKgvJwFwzyQWeq7eYFavQCG2b1GXaO9fe+nvTaMr7wKCWM9l5sqOwW0nL2grge95FtyvB/UFD5TDqi9RxIXvraxOb0et3o7nfiMO2v6jL0qnom9M4yJPPNbmTygfty9ePozPVHWD77neGs8Xo3MPBjgPr0xzYU9BBrbvf6wk73enym+a9rivdupCL1BRTi+jWK1vbaTBb5TIGm+1ioGvlooLL1KhUi+ABkevVajJT0mCCK+DEYePVfZzL2VhTe+0tbjvWkKaL3yjIK9KrqvvZBUX753XBS9oCpdvdo9c7pWZC69e1/rvcp5b73Y57C92xnjvcHWab4D5lm+5zzeO5D8D77tWyC+a4Zuvg5aC75JNDK+Hm1dvCfsy72rwCS+RP6pPQ2oQL3Sb0++77+uvQp9FLup9zW+xVkjvo/6P734Pz+9/30HvG0y27w8gwq+a52WvZWmCjwaHhK+raJ2vYPPEb3TSna953+GvHH8Ob2RZO688lInPAsPpz21toO9AotMvh5AmLwH7qA9qO+rusYuWr45QD6+BdyjvUO0Br5o7xe+Z2TGvW7EEj0mSG89AZfVvU0L6L1YLAK+5cVUvukBXz3vbDq+PGwEvTjYD74vnwW+b3mXveZGzb2fMv+9kiJXvqKufrvqavO9cG0xvnFvZr3IkgG9fOXNvT5aNL0dK1E9uyTzvf6HPb6wYVS+v/6cvfOmAr5q6KO9Ivx9vaffyrwYqsa8vfQWvojcAr7T4yS+ti0gvtfrMDpGPFK9dxeOvVhuW73VTtO8DwyBvj1zTr7/AwC+elxVPTGQ4LxgT9q92SeLvW6BZ74Zwsu9","kMq0vOulM7oC2Hw8HXQcPl6lPj47uwU+gTUEPnARHj1AQ4o9z/25vV/I9T2vKKO91JPhvQy9s72nxr47fvZGPUA0DDxTqKI9qU75PGOdpz3z/bW9Fd2suwitiD26B6a7nbk8PEUYAr6cC+69dGELvcQ8Ez51k5c8t3KLvdD9rDwtc/K9PvnIvYBBjb2uNwE+hZ2gPA2JkzsAAK88ekyFPb2FvzwNugI9+qogvh1Rbj3P6iU+1URJPZzhsL3MZJU9TSJ5Pc7fe7wuuQk+KGawvCAkm70aP3y9GxqCvRGAdD2MOIA90arsPIRhHL6cS/A8aNzLvP51ij0Jif+8eblPPVPoij11VI48oGU8vV3rc7vstqk9Ud8VvYug6b2rggG9KVIcPl+Q1r1qwZ08CizRPchhiLvqxc29AwMCvpK267zZbj68bOLlPbvzYDuArrE9VHAwPl0UHTxdbxs8LRiCvTkGHD0nwiy9GJIvvIu4nz3b9by9frUtPAdqNb0gIdW9VBqrPfoDSb0DJyq80i4DvIV5CTw8iJg9FSPevVjljL1uQFk8poq5vchKZryz9Rw+D0waPHeGAb3pqMA9MV9Yvc+vnzsGijc9ZYIMvC1hr72iE6g8nIAvPdjyML24+S25u6sHPtBujb23k969BwT1PLab0j1t2GM9iH0KPuVVST6miOo9W3cjvSxEYD2d5qW9HfN1vlETNb4e2MS96titvdq8pL1r8i29/6QRvszpab1Zc6O9ZiaEvYS1gL57HBu8NKfpu0z1+LyycZO9uNotvW7tqL058Um+MDgivp12pDzCEEm+BGRsvfQ1yzwX0569bUgzvsWOyb0ZOnw9PU4ZPkkrGr4wU5K9VyCzvY3GF7rE0CS+fMk6vsjsiL093ZS9vWqwvJ44lr4X7q69APYJvj+Hpb4s0xq9ecayvTWQvL3mS1O+ZmDgPdM5Ez0tPo29qAzxvbRFDTweE0+9kjAdvo5Eu72nt9Q96i40vQ9ddb5Olai+RMWEvlMqSbwpXju+","Y0lhvGOKADxIFu89McqQPYbCxb2BdyW+tljzvblyA76pL8m9IU6mvfOi770eWYy9ExXZveapgr05wu28KirhPByLST2s1s294voqPVBl173gfqA97oJ2PT1RIzxGZDS+eq0nvvcDwbz6vIM8htKGvQukB75bdng9kx9HvRp+yjyvQG296vkWvVuZyL3QdxK9/qOqvY2wDL5x2vK7xRKjvjiLOL6mB4C9QIhgvpKfgr3eXKI9yCzXveRAvbwE3JO99LcwvamMPb3wkTC9JIYBPhOQtr10Hbe9qEgRPug3Bb63ne+8i43aPE2D/L3dGry8E0KZvvDg2b0QH6C9CgKUvcZat7xGPWi9HHcDPv3bA7ts16s9LnAHPomlVz1X3SG9IE3evF1D7z2gMOw9rSWVvdO7J75Uauw81CBYPkbFSL3KfKW9SiZLvR9d8r10+Ug8qr2uPFJLZb2AQi0+F/3ZPeX7bDqs2TC+RRosu7YG1Dt0IQO9q2WUPQUYuT0q6NK9udEdvsB5+71WUli8w/c/Pa42lz2VdAc8X5CsvGWvrD7qpiU93rDFvT332b2hApq9cK1IPnT8dTz1wQe9wF5VPTX5Yr14PcO8Qvy0PDUpijvkJn69sQodPvAFRbzk63o9w0KgPfBv9b3mMKS94OxzvYzcr7ree4c9pgFIPbQF3T36djU+U/s2vZkOFb7swik+44QuPk8j7j22kMA9+IKUPTiODD63y9A9FActPoHvrj3sMec9+UiFPVzdkb3w2pC9yRdyvZQdcL0+2w49+FkgPlT0/D3NT669VFYXvWdJkj118PS8EK4kO9dMm7xVeok9gfgLvrgbA74w+Rs+dcSoveh5i7ywLSk9yEclPryfmDylPOO9t71ePV9Ulbxm2L89EW0GPkZUaD6o8Nm9W80Fu4c0xjx4Nyo9Yae5PW52Kr53WZk9xM/YPBwYwT343Qw9w+hwPnkgwT29nGc+ifdyPnZMkzwUCO29gb4IvjYSLj5XuYo97mf3Payssz0naIc9","v3XTveSCPT2hLF0+cRgtvrRmCb40zWq+Iw9EvBZfhLyHpuy97JYwvbPk373VKs+8mKmYPP36VD1d5JC8lH1VvcAH4r2u5mq9sAs5PbyNcDyXije+MhrqvbUjAbyDD1w8qbBmPSkPvr32jHO8Zwlkvcz1Ab5XPLs99Zg+PU1rwj1kAtS8e6+NvvhLbb78une9D3tkvQmMSr5/HiW+IW5cvuHZWb2vkvK9VzEuvk0N6DuDCYA9qttRvr+ORb7COKK9/sw8vbtZ1L0ah/09GDqOvGyjir3BH7W9PJfOvSqrAr0xo4K+KCB6vcW8Qr5BhLC9YSi1PdtMzjroy/e9aXnSvTG6N74ei2Y9kkuiPRmedL5ISge+HJ1dvnAClj2ktaY8Sb0Bvk0N7DuZ+h6+VHG0vWPOtr0Gb1W9QAsqvt1Cfj3WGt+93ZGjvVWqib09f0++eCJSvlRM+LsZ2D69P3KMPQtf5Dwq0V099Mm6PNh3tbx5gnC9WA3rPLnYDz07QBu+A2fgvKujL774kgy+7ZzsuvJYQb4Zkgy+H4k4PAEI2b3DlXe96IqBvTgzs73+sXS9pvnIvqbfdL4Vi567gmSRvUQTab0tQGC+5+8jPVfaGb6l9OA8AYGQPW93qzw/7QK+VwOEvqJZK77hyb89e2QIO1CdVz3fEsS9q/f2vQyQjLyPiaY9IYIOvPSu1T0Sfao9uToWPkQMAT7JySA9r2vzPKJBND3YmDA+1asiPetdP76MN667C7cfvIh1IT2FUuM7DNFauxqB4TsGPxA8chCTPUKcSj24BdA72RzvPcT8Oj0JXEc62RPXvVGqsb3dVzs8juW2vX7YhjykUvu9rh2nPW97+b3hsjg+ln5mPXh2jDwEKS69yylZvGPq/z2jsYy9Mb4avRzPsrodr8Y7v9TmPcI1Lr1viCo+gPWgvehc9zz4eYA9BLHsPV3OT75XTwY+V7+WvPMYjDvRTAq+SZMDvhOsLT2i3ig+V3W/vZIt1L2qEKg7xC7mPZttKL26YeU9","OUF1PdmGNb1zE0y+KJnqvLnIoT2FMBQ7huNyvgexuDyF9li9FJjmPIRZED08EoQ9iOWQPQhcGz6DbDk9jDNlvUyti71PE14+RWOXvdMSrDt0d/e8vpMaPaTQAT2vLBS+YjdRPixFY726nby9REAgvaHm5jyMNtg96gvLPBv7Lr0rPxa9eIdKva5PB715w6m8QHnfvW3cZj0qqDG9fw1XvkIOQbuNfZI9r9p3PG1zgz2EWqY6MtuYPTFssT0tors9DsiFPW9pIb7PYDY9ioaQuicHrL2qsSs+tntyPZl2Z71A9LM9Y23yPfXZrz0AKRC+dSIyun5Rxr1ltsW9TVOpvbs3ED7JWsY9ntWGvaNmPr0oEm4+nlAbPdDpIj4HxHM9nfeSPdforD3ENaY9tqsyvefSDT6rcb48aqttvQdpsj3bM6k9ImEFvXwLTT0MhDQ+CdXnPWQZFj5nGme85T49vDk7Cz4IiNU996CpPXm43T0yA308ayW1PQhr2j1iI288mz3QPatxdz6dwaY9b14KPbhYGT5lft89MBNWPvxhNT7fIDk+Hrk/vNktTj1w2Mw9sFXjvE4PNT57B9k9dV5bPjE8Lj7g4ek9OG7aPcm9PT40HhY+EkgBPn1BBD1GEwc+z1oHPgQtgrpPuyA+VafnvcsS6z3xTjY+3OYtPrw0dD3eXwg+//d9vbUVxj23JHQ+RmMKPv7+FTzS80g9Yz2bOzf88D1TcQY+fkurPf5UF71NHuY9WsVfPTDTuj1vlmU9DW4EPoQiFTwTKSI96VCCPTB1kz367gq9fnpYvRcnTz5jY909CkhmPjH5MbxEIpQ9xnrsPeIzjz2sC989jw9DPa6wgzs5s5Q+X3qUPgHopj3gmwE9zoEzPqdGvD2rZh89SY8APlmgsD01d7Q97vCwPZrsHj5up/I8jJlQPe1LuLxWU928S8Q2PVWqsjxu5Uc+LAlPPXeaAT4XwnI9s0w4PvZcYD3uzcM9zSuHPEDrsz2DeQQ+HinBPfiJOL3AtJQ8","Hve+vW3S1zzs9Sm+UHmzOnKwj72DgRe+uefgulw+6bqWTz29pcn7vI+iQTsaRTE9Ay+0PPE/970kqBO+rv0Nvjm9m70XGpw9mAtvPJtKDL3OVos8gdYKve8ElL3DUGE9+g07PtBH37272CU93ntuvbO8eLwr+Fk9OxuuvYAXi7xiW8S866TKvS4IsL09Vaa89RsTvZOFoDtH8la+Mnh2PXJfmr2UJCu9RA18Pe11X74PE7m9ywQavn2ZDD45aLe9kvkfvjFKNr1CzSS+utUIu6+L+TwSXiu9ZDQVPsGDLj7mRvW8kbfWO39Nuj1fdku9U2hzPWahYzwGjEI9jHEDvcc6YrwwXCG9YECxvcDYbj3Jbbo8gEG2vbUmDrsqWOU7GAmzvVFzBj4Qcgc+Yk9oPW8jPrxSKVu9os53vQITi732Sno9Gsdlvgw7Cz6hLkc97GtNPDpVej6b0ge85vp0PSV00L1dgoU9m95RO41CtT0QP0S8rEkvvUh1FL76lyc+LxWivd7t5T19KB4+ojCnPBBN+rzFRIc9bnUgPv3+Aj638ou9F3CNvGR4qT1y5P28FrO6PChTFb78Pok9f+FZvYLm3Ttd+Au9ZkI5PduLwz2VN4496CgKPYCmL724Goo9nP9ou3xU7bz0Ev89sxKgvS/myzzHnNq9FmgKPUpoWT159KA9p2VtvW7tHb2uQhS+K7kqvtnujL5uSdm9wBm6vJ6bXb4OqoC9tRjfvevqCb2lJ+M8F6P1vD/UC74YPPq999OUvTJfk724+J697TsBviGJtr1614C91RTDvQBHNb3F3RC+TaIRvdMAHb5sq0O9Fgg1vj+Alb0cXjW9G4YpvWMqfL5c9JG+so2HvkjpUr3Bxuu9P51pvnruDb5crXu++bNsvcm9Bb7h6EO9J7FevQJKKz4q2oi+3J26vVpTgb4Wgxu9ghLEvRPYGL4kleu9zXsWvfZz373Gbe68Oe0Hvl11U73Z1TE9qiFRvnisW72TUBi+R4oGvuar9L3Hf629","yoQGvAFikbwntMk8i+D3PBVpGb6cJ1u9n8W9vTNL+r1Yb0y+7RNhvvTRlL0UKFU8hyaiveipTr5jApa9P72TPU2rwLwi5ZO9rTyFO4JKZb57kxg90mJWvikVIr1MPcK93D/ZvTBpFLzYE7y9o12hu8ewYL5o1Iq9wX1mvun/G75d1gO+hCPBva06XL6uTOC7hbLqvRdRCr7LEHq9CJ2jvn0Sqb2pegi+LT25vYJgLb1fuCm9/oyevPXQH77gsES+yg6zvVNDQr2ZJXS9mpIFOgDGSb5uHYm9QgKFvXVyHL6GzRs9MRyqvY8LAr5qepi9thkRvefXTbwwJ6W94E/JvTAxdrwo3389vmjcPcJMmT0pf+o9AfCaPbZQwbxi2rg98Qc0vYJaNz33TUw+DuIMvqDLA73IY5G9NsYePYaiy7zZBp48m3KiveDKj71NKwI+/3DSuwhaLz43eeQ9/1v9us1cxr2syZ47LeaRvMd2/ry5/Oa8wgr+vUOQtL0RRyG+Xqvqvd6NT735fIo9vwy2PTe8Vz0Njsg9X+gQPiGY+jxocuy8x/DevSE4XbyNe/s8zMQFPpomHT5f8/S9FMrgPXv44z1SknE7T1m3vajgj72dKl29aAlEPJVCZr45nI2+fmQ5Ppxckj2/1wW+hKDSvVOQG7sE5nE9x46HPSg91bw2lSE9wIn3vF0iNr1HVFY9OwQrPmiOhDwynam8jFXdPQl2NDyuhmk8ypgwvVe4Sj0Sius7MFhEvenzyrxCBv48dVE/PaHSHT5JXEM9yKc0POQzSTwbfsi95Hm0PLCaWLrg2mK9HHopvDncCr3kRzY9fS3dPOWaRL6MgcO7juw5vhTgLD2AIko9yLcJvf4kFr1rs929uOcOPW5MebxQZSA+s7oOvYEAib2qlI293D0RPaHVbD2/Nbs8eEO0PcQ42L2Fpq88n3vOvJJD6j2njUw8jP/ju7WnQr1zNgc9slU8vb9gcb1mfaw83FkcvbbLhT0a+eG8rcI9u+hutb0Lh4K8","J/bHPEoRQbyxEbs944aJvqQhj7yIPpy9F2QtvvslBTwJx8S9Ir82vnIyrzwpMjy+WXokvrgJjL0ZjVk9Tke+vYUOCb4LJMu977n2vXBFmb1csa29okUevpiwKr74Eqm9A5tCvipBc71ihhG9142yvQYvRL6LY969BF3CvcdW1LyQsxW+oUGOvpllZb6wuTY86bHKvbDUir1aAIC+qTIBvu+cfbwUxUu9liUtvm2T0L1q2rg7qElPvvPmGb7mRo++WTNKvsvy6LvKpQ+9vhq8vFE0Er6gzPC9DidUveMZ87240AG+ly8EvsOzgb4BSwe9IsKZPRn8Sr5NcBW+OYiNvYdt0LxCPlm9HpHBvarImDvttA6+PP+HPEI3qb2Gu/i91pE7vj5tSrzkCFe9VLXqu7+EEL4dWOG92rqovdcJez3oqBS9lHV8vQaFFb0VRjm+q0BEPMPw2L32dZq8MLfSvV9nGDuKvey81Z+tvdvHbb0hKTC+RMyzvTLWo71mskm+SLCfukPc6L1ecC6+LHFbPeRy5b2LndS9bYjfvScxB74pfUe+Ip6YvWAPmr2lXBi9D6PRvGlhd7x1HDe+jjPYvUAHRb0usc29829GvJ2irryDmgW+ObEQvrYYTb17Ktu91KuavTTJEL6FODm9wMSVvdQ4Vr3FLoe92K8pvnYEBr5wX3W7rR8kPSNqpDyDQEQ92CXkPHLGFT5tZxs+qKbfPBxcQz3a2n+8zv5lPDQXQT3s3BG+yh2gvfLsuD1bShG9OQ6hvdTd9jyFuZe9l4jIPfF6WbnEwfa8VZYMPgEFFbyyAbI9sWQrvrO2F779gAm+sSuoPXjSPrywaj69fMOzPetAGr54lNS9D0fjPCCYgzsc/9892kHTPZ74nr2/KEE+Upg4PiGVeb1m1OQ8JsLvPW+8lj3ysWE+M34aPL3+hz3TTE49Tt/CPZGZ2z2wQQA9FPsePLNHvT3c2Da+71gZvrRKAz5u16a9r0kkvgkWaL2pF4y9C1QGPVZ64jzXGW09","1/CZPSw1g71KqH895VpmvY67bj5TzKy8YJvcveJvLD2G0dQ9kOEVPvDlrj0SbHO9CdG7PS0tsj125FO8tMe1PXrpk71Dmwg9RE6OvOGH9rwDW6o9YHJDPetznj0rD6e8oeuEPbHQjLyTa668hqDqPKKp3L2oLQS9ALxTPWCSVD2CTkY90APTPQ0gb73P1jU9LQEBvdaXm7vIXH69sn2xPC+Pjj0ibIq9XhiPPWElOj1Yldw8E0iKvP8cFzzYD6o8p73uPBy72LxLE+E9Qb6/vagjKD1UrIc8Lb7aPOT/hbzSbey9wfMCPcf/RbzjoZS9sGsYPQQZIT1PBLU8B1coPGAGTDsx2AO9pSwNPm4SYDxarSC+TPf1vdX5szz7hSG+QMfPO8e6Mr7v+2O9nTcHvcZODr08GMS9v9h0PdaiCD1NYy2+weumvXjaOL28GxI7zna3vSS3Fb1eRii+H7vwvbUXAL44pyC+PMEBvv66UL0aiEq9hHl/vcNKqzzIeYM7tsGTveM4gb5rG0i+sIUmviyk773V8Ra+SMqUvhinUb5INRm+30P4O6Gxg72cAAC+i5flvXVID76FmB6+DvVCvmWGhryEaoC96YNJPXf/0Tu9MYG9jDPRvRSFTb3vWkO+QKuZvfCG8Lw2Bhu+ajfVvIw0Cr54/Qq+htu0vayfE75bYvi8ucczPOfyR701FnQ+I+CyvVU+BD6rIWo8GBwAvnOU0r0eaAu+VwAwO70hPb09tpC9XJUDvihLI75Z/ek9ocsKvpJhWjwEVyS8o5wjvW2ECjyWKCg800BIviPYE74zZia+PHh6vRJ4ab2/0QS9OItqvPMFTL2dKdm9hjWLvSqO8D2hiyg8uH8svjlUg714y9q9QArlveyH5jj5tAC+BbtQPVKq273OXxS+a+mDvXBnor4jA7u9uqlHvYJZwL4QW6K9Kbq9PQuNqL3+hBc9Z5civj7JHb64byE7bWFTvlep672/QAq+E61EvFqvXb3bTjG+HlasvR/NzL30fG29","b64nPVmJ8Dv3kei9XUHVPW5jqjpT//A9BeaYPTEoK73lDfQ8lEaRveHfqD0Mp7E9w9IovirGcD0EJXm81TdDvguL1zyAOCK9d/oZPvsnP72RIo49ZteuPZ0vFj7SDKG95iTQvTImL75hDuq9yVQxPejILb1pgqM9TSWevDDZTj4FMoS9dRN7vV2dIz258yM9OQrYvVt6sD3cBTk+mQVSvVzbgb3yQr+92zKSvfUZGj6P19K76tcYPpb0jbxFHnM9MtiYPZdANzzBnJO9vnp3PW/qL7yFW6+9+X4lvShBjL2X4lc9OvS+vRZmfD14EjI7qwsDvcnlyz28Lhg9bR80Ok1Auj0yHdS9z63fOwYuib2WJyo+2LD5vWmOEb7S1Gk9C1AhPVOvGL0p75487qlIPfFR7zzD5z48IIVxveC5gz21BLC8F6OoPuDUL70FSjQ9WzKdPQwq271Kk0A9gMzDPVE11DxEmfm9+scVPp7Jwz2N8Y69hn3BPYdNpjwYjBO9zPFUPa7oIL2rLwy9pwX/vfLkgL2DNX89Jw/vvc1hJT4pY1U93OXVPVsGv71Psbs8etyiPQ0AAD0N1fE9r9B2vbbZ2jwqna+9WqXAPSHxdT5gKKU8bipMPWVnMD58swE+jnG+PTDCIj7BfWE++dsBPni2jb1igeK99U4/PVNa3rwYbLm8w8sivlB32D0+7kq+X4sdvokPCr7KriW+x/mGPfl4E759QNy9v2ohvqDVC76yHhm+B3dsvVbSabtZxGq9h0hFvjVdED4537e9KItSvnuo4L396VC9eoGXPG5Apr3YZms9xIlFvlZUvL2tbKy96REJvfFmyLwyPly+GiB6vXOr9r15aUi+u2j2vWOaWT3WxJu9+c8/vnGNJL4RWna+P7PhveqNCb4PbiG9reyXvTzc8jyeLlC+LclPvvyUWrysMUi9dpb+vW0xED3o0Zi9Vi8KPQMuOb5F+mE8ptthvpigv70//h++hpg+voCdCL1Qhn29OzXpva/H171aDfO9","dJe7vaKvFr1wugW+Kpn7vXTf4L0iVS29BrCpu8Zuzb03lvG9U8EyO/VwQL4Ach49ygy7vbo4q70CcLm6PtVaPREBFb5QeEi9lzgAOwVvCL7vkoI9rWmhvdWv+L1AmyC+PVthvNlVFb7RjxW+8JXDvYYDfb6jw4+8sd/IvRZy77ylG4e9ex4XvnXh/b2CHBY8L/gDvhggCr4rhMe9OmOQvs8Dw734BD2+HOrNvZ9QqLzc4DK9H921vVx7E76N20y9+aF3veKE8r0iad68S23yvf8IGb4GN3e9CeVEPfIfIr5IZwm9LIsnvqq627ufhQQ+hBLBPHdpR70XTgO+h1FGvtariT3Qr4m9FdMhPYd0Pj0vWAY92Wt/PTzMfT3H6Gw9sPR+vdd/FTt9xvM9gDR9vVtIXL63Pwe9E20FPvbdTL1AfYc8yU0wvrHarT3jApI9DfmsvTd6lz1m0Kc9E/OSvS/TJb5V5dq8QfQZvTWtDT1JTWa9q8prPD5XAb65cwE9itLxvSM2Cj4SOwY+J+gAvat1JLs4SGY7NTLKumCbrTyYwaM7cSmmvddjFry0G0C9LkCkPVVUUj6fvtm9VsWyPAy4yDxricG87jxYPX5lxzyWFZo96qNTvd7UOr7UeVi+gYWQvdd/KL1bJJM8lkrLvQVi/r0QB6Q9IuhgvpoBbz1z2MS7gA7sPbQHqb39pa09JCv5PWYkvjysAjO+aVmUvVijhDtEW569EqGuPfe9l73c8qU9cksYPb5EXjyp1bc9NePpPVC8zT0qi5C9JCppvfSIAD5h/L683hSOvASgL70+EcQ8aNx8vRY4kT1V4wU+QkqwvBu9bD31pMC8FYZJPb7tpT1SGbi9w13nPf1EpD3yQIi8Jhp4u8Gpxb3eQa88zUwdvVSrpzoyXj09xc0wuw2lh73qSpu8yTITvAMBE72/G9W8kdMXO6+0AzxpKEO9XJ4IvaoEJT06zr89sIgGPoZprz0jD5q97XRjPXms/D1NyBK9cXucPGcbljyrXd09","7gRDPdAiJbxDEWQ8S6bYPZpOob1rAw09YLAXPVWWDD2hFlk+FKYHPgYaRbsPutU9ObklPmrBFD7VHhk94RZbPfBLxT1GP8c9J7u0PY/byTygL1g9kuQWPmRDIj6HY9O8SwatPRDXzz30IsI9+aUWPvMQPD5I51q9dh3WPeVE0bu/wH4+1Gh4PgvBBD7ibIo9gJw+PurlYDwzKzs+VpcwPpRuuz2wxTA+ixxyPRdPSz5sT7C959ZSPvxsET0TIBE+Iy0xPlA5xzzTj4E9jcIdPoUqlD0n2349/4K1PNtPFD7/R707BlvKPHoGeT302Zu8FbPqPcjKoj2l9ho+PLTRPdzYbj5ve6894PnuPWTMSz1prtI9BpwhPvSjBz6ISNE9PCoqPvFO/T1pXIs8Pwu3PWBYMT65108+U02VPdPYOTyR51c9QHzEPac+mj3S3vQ9mt1nvWvutj3k4lE9+fE/PiiRjD7ncEw80UnBvWa5pD2tCFw+j7LjPCztUj4ItJo90pypPTR4qz1+MxA+G0covZ+Fu73497g+4VjcPfLwcz7cHLU9rVwLPglYyD3baGs9yu6+PBkAGD2/0L49ZEwHPulFeDuaRaM9CwI+PJ18ez0/eCo+HS2mPbOqLb18o28+o5WBPciBWz4R+809y5DUvbY+RD4ejhU+yicQPtGjJj71B8e8YxwUPsnp2boZx5C9zKX8veE7nr3UIgm+7BdwvXNhQz4171294qD5vXK5OT6eowo+tl1YPjwNZL5kd4u9VBZIPF0fxTxG4sy9RsHovYrLFT7ncO+9CoSPvVW6gTxeWOc53gXePfdMpDx2ex0+xJ6NvcO+qzzVLww+E3RVPEFMEz5xUBa9lrDCu4b/kr3tyKi8LRcrvk3NUr57Sj89H4+DvXnjET7RXhg+Ad0VvrG6iLxd3Oy9JE/HvETqBr4t1Z+9HYievZCRU73JSoA9Zdh5PYjAGz7il/A90LVWPiL8Y70Ku7u9fLmbPc5TUj2Wb4g9oIbQvDq/BT79Zdy9","xlEJvSZPM73KJI49b2plPUn2Ar22wlK9FLIYPsfRrT3A6g8+jV0HPuz84rzAzz08aCAAPSqAEzyaaQA+wMD1vT7vh72VELS8PM02Pb5ZAL3J2sm8EvWjvQb/ST0LAzM+kw+LPPn2Uz71JxQ9oCtRvTJb3T3IqAE9pZXTu/eFK72jHt699ed5PTwp2Dvsog2+CWR+PFMXaL2Lcg49lNgHPuNpib1/GGW9OpWsvCYtF73yMMO7uPrCPW83LT11W6S9Jus3PSBiHT45OKW8I07vuyGW+DzUEga+GvucvVIBCL60BCW950qkPemnSDxLHVS9TKHPvbvfOj0P2HI9hha9PdI3sj1Bks09rEBYvRFmIT50PTE+LPXdvVZ3ij1o33O8UbmGPIM/GbwswIA8ICosPU2xmD3ZnvY9MywwPU+ScT0ncUM+iN/mPae61DykhAA+gGqLPG++BD0MX/M9FeoAPuFi4bw2HTO9GIedPdK5zTobrBA+bt1LvBCc4j0/gvY9eINzPpc5SD4Safs9NAeKPaRqQz4NpZY+x78sPof8CT6cyQk+W+vlPR7rSb00uJc9PNqXPSQlwT2zKb89G7zIPY5wIT4Lz6o9Gd/8PNx/6T1uuXc+OjoCPnBZAL4rdKc9s/R1PTj2nz0Qcws+nCl5PXEyOz2nCxo7UVpdPlwCRD5KXsU98rl6PetRKz0zBS4+mlcOPvDzLT4/nNQ9n32RPbHJST4DMR0+oBKBO94EkLsJMJA9JLYHPgwitD1Ps6i9qNtEPlE8dTzAWR295wJSPom6KD5PWoc+h5IrPnaA7j0OH/49ycaPPePGMr1M5rC9MTEMPuUdBbxR/g8+Ro9UPavpD74VpDU+kkroPW0wFD3TVzA+IioSPoJIF779Pjc+TvkhPvD/9D0lzMk91BHIPQSR6T5tlZs925IPPhkKTj6cvSM9n5dpPmougr3Ty8c9jwkDPmka4z2JiUe7r3IOPitvVT6BdIk9DalvvbRuxr1ZiHg8LOAPPUhkOD273BM+","/k5avXbd9T2Bn3O9uuAVvg2ZvL2+MFa+dazEvODmOD35z6Y9WRD3PF3szL3ATcw9H24YPiS44D3aZY29i/VXvov4Sz0vz/28ffXCPFxg973LmIE9WCGvvSnuO70Orw8+lDLKPdEWWj5t+HA9EJRrPa29Qr4Nxom9w/CDPXtSJ72EVMu8/3SfPbdOn73d35+9lQoFvJkiUb1tYNa96R7tPJsqYr3sXYo94koHPkHZSr6Crds8G53GvV0Klz2Ldzi84QntvfvTLL5/ERg98lEQPgnN9rxow9c95gPePdDIQT5go6O9s6SevH0ckj3N21w9E7gkPn7j6b2Xrwo8yQDXvSJAp712dfc9+DQJPkDvDDpLeVu+aQ+DvdcMez5ZkLw9LBOvvN8uGD241Y297EOpvcr6fjxJxAO+jPA1PciXuzxbjHg9roZfPMiyZz1fXYi9ZGIevuNE1D139BU92z4qPkXiPL4NepI+SgRAvo+akb3gyUY+A4aNvS1XGz46FPC9d9aHvce7OD37mLu4QNGBvRtPmT19Pem8B4kLPl6jPT0bnzk+QLWdPYgOJj2k7i097xNHParewDyxY9O97g1rPWQhMj5Zixs9wr6mPSoOET0hFmQ9IMTLvU+qvjwwP0s9a7YhvrTIijzOSDC94N+tuj1MxzwXUAU+jLePPillwT0chB49BFX7PMJmQrwzByw+w1APPvnyhT22bRc+5yjJPadc4j1ZpeI9dFTUO2ijjDxl/tI9L0UIPr0MBD5Qt7w94SNOPqpKvjzo1DI9LemMPGiS/T0Q1lU+/sY4PJHofz0LpB8+5/XDPU0k2j1rrkc+egd6PpV6qz0xjZA6B7ugvbKPNz5WwHQ+vGltPnBeLDzOmgg+Up4NPg9hSj7rBlg+unf3PdA8gjzHTYY+ddAVPfQjcLwUwBs+P2ztPQoWGD7lct67W9kDPtylxDwWAVw+Yz+lPEuFrz2s/wu9cENhPpKO9z1rMto9j3XpPVxkLLyVeEo9nUATPjp0JD6y0649","DfPCvAT7ozyeGRW8d2hgPpJ79T1dk1A9IDE5PoGloD1Llsk9bR2fPUS9bj2qBYk9LDIEPm6Dgz4FCng9he6Vuj2Fuz00AKQ9zIEgPAW/OT4DZW+9kbjSPVfMcD1M+xg+rSkiPhs7973eHrI9Tb8gPj/GKz6h+q+9B7sVPdTgBD5yZ0k9pcMzPuPqmz7yZ0u7ikJiPjKLbT4Qja897dhJPjWIDT6/HD0+DbW7PbOVfz26cTm9Wqv0PWvRUD4EZbw9NjyKOrXVSz2pFwk9bQnhPdZgND3TQgc+Sp7wPFhnbz7EPQ8+aCY+PoIKXjxuLI+9SLAKPsUxWD2fk5u8MJc8PSNvID00fvE9C+G0vIgl7704i+294m0bPA1IIL78qUa9pZ+EPsgbaD3N0WG+4ZmoPTPx+D3BSSk+HVTWvVs0kr3LOOe8bsBhPZWUEb5se8K7pyTOPG7n6jx6AQq9EP66vcLGSD2vxaU9IBlfPUG4ej57FKi9Cr2XvKyNHD6r06e7kmSSvAoVnD2J9/+8zhegvYjJC71FGHS9SZOWuhZU1jsxMGK8W6qRvbzAyD1zOF698JGPvIlUiDySmgQ+ws4BvrOnoLzMbDg8+cPuvREsXT5nX2I9dx2rPXeIID74MbA9Q3CivcMW5b3YPTM+ZlAKPgr21j0ynlC9vZUiPlkSK74CKVw9sUBnvFTAdDxVNVU8fD2ivQoGGL7N/+C6uBapvcFHcb22AJI9zR+NPgpp9z2o5mY82CIWPRJCcD0Msxc91TQCPpVV+70DkxA9NjgovQBGKb6xlYg9AqSKvcRr/b28ykm9/y3iPUn2r7l4Epo9eDlQPm92sz1X5ak9lfZSvdeHEr4LepU97tC5PLHa3Ls98fm9ENSAvUhWNT0I2/K8JG71OwZmPr1TwIm9HVUSPUQjnT3Yzwq921C6PMBBsr0G9yO9ye+IPeTKo72jUpE8mp3wPBLIwL2Wb+08q7qpvZSpFb6kAcC8KZ+YOUTzyT0buzo8cdElPOkzZT0R96G9","OQE5vM2B37zLEZw82z4vPoTqaj1lGJI9LjgJPWkg+j0TIrs92Sv6PYx/az1t9wo9wmr+PQXJfT2/i629AJYwPZXsfj7auBK843bbPZtU2j3ibYE9jejSPLMurz1zNNE9f3qHOvFDBj6r9gY+FgQLvakDVj4+4+W98Qs7vSos2btsnQs+x2hVPgKKpz0ndjm9TeiPPde79D198eE9lnsDPg4VCD4LWTA+zGUNPUcg6bxmUUU9RdHmPdn62Tx8Vck9rtPZPV5tCz388lg+51+fPZGYJj07HZ67hBDXPLnD6T0VyMS655rOPYc9nj3WneO72QsMvjdN8T02r/28fO5YPqxpiDvuaog8GUkGPiRbVDzQkm492YYQPkPoHj5OSoM8a8cePd1nSD62qoo8y3kiPsKMT7zmuSw+sETUvWzLbj1t4Z+7IRV5vD1RZby6Mzw+zsnNPdJOHD0h+bI7yMSyPT6ztz0o5TA+z8AIPc0bbj0p5Sg+ZKIuvg0cdz20drA8aYMLvW6sIz6vHhc+19A3viRhOj7hDyc+uYYSPmN5/j10DOs9uuTCvMiy6LrVFaU87XJKvRVUOD43yeE9kccrPkwCjj3cYKU9zzEFPi2WPz6kiZU9H7p3Pb5po73Sgo0959qAvSqcMz7zer08ji6Yujg+GL4P6bg99CMxPnwRBz4HaCy9R9cVPiPzgbvdcfO9ETQFvlB5lL3k3xs81FetPY0fmr3W84494M8yvhM9CT760Qo8zdJGvfp61b10YYY8mNHFvUL/lj2nfDO82qkNuDa12zvzFKo9fbMpvr70Iz4XQQ49Oi24vVB/CD5Jx4A8qzTdPNRNibylBoc9O26TO3rQQbn51Js9E+YhPS3DHDsUpuC8uvc1Pc0PtL1fk/q909yDO0yDsD3Anoa9xnASPHe4br3/0t29WyoBPrziIT3YWyA8kl+HvUAYb71kfxy+O6vzPSL70bsTFyk9YJHcPQPc6r236MW9yh6kPeIKIT4NRSQ9lJZdvXHAAj02iTq+","9hp7O2Km471Hdp689eKqPXftDz17jZ895vabvFuoI7wwa4g9gLxGPRF4jT0MsbE9wphtPfxltrwGmu28I0IIvKx/Bz34WSS9YYKaPWKiXz0rqYm94CLAPeSBGb0Knli9DJSbvfRBHb0QAB48kJuvu3rXOj7Wbko9MtdFPZpj4T2JBXm9DmmvPrT9jT2WT+S9b2ZwvaMvhT3iQaA9rDODPhETsTxliJ09qeyhO1gOXr2j//M9q83gvFYSqr3TGM88D0y8PR5Y2bxwjdU91xvYPTJzuj1dOBM+eIU6PuorKz3HLJ+9jagvPSpi7T3yhPC7C8bgvOl32712SKe8wcC0PBAoET7ACQQ9rfIPvrP9vz0a7wo+qVQhPTbQZz0o5mo901GqPLE/Qj1kctI9+yv0PUJnzTx4pKU9twgxPBDwaD6A/Uc+1SIKvmVGSz45aqU97HoyPttEXDxllh+84zvRPXX7Pj5vwic+vYrbun07Aj162co7rmQ5PUfNqL0aG+E9z+tbO9GEmj4Ny0U+GVNhu4v3WT7GV2I+7CaVPsMLWj6wy1U9rlzDPQHLDz1rBNs9A93QPWegjD7nOI49EAnTPXrfgj0UYgY+BMbJPdGmRz4tVoy9edeYPQeJR73tIu493BeRvZRFKj7vjR0+mejSvWCfAj7pYsS8YBzqPI9gFD4gXgw9j2u1PYAxNT3pQV89iuuCPSf0Yj0pxLA9S0AbPczUEz4BVRc9HPH0PYIhYb0BtCk9FjXqPMPxij1/XhS9vkb+PbF5s73cVAU9Jm+iPYIjGT5VIQM+UqSrPdX8Rz39lBQ94RE8vIn79T0WYyY8+EQjPuRZeL3/SKc9lJ+MPbD3zb0CITI+SVacPURyIL03xO49JddRPtsFO7wwZ2U9SDk1vOT9FD5HmhU+AhETPX/IAT77uos92auDPTqh6zxOLT++o84GPhTVBj1uUUQ9gllvPaKqSD1i+KC9/EmUPeeVkbymRPA9pIdDPZ58Bb76tA+9JuIRPj68dT1F35I+","QxMbvh1XYD1ZKCO+NitsvIiIN70XJhm7yIXqvUwhGL6E95Q7TeGLvQG0Jjz/0R0+6zxEPkONJjwDn4o9E90lvse8/72qTlo8Jxv0PXeIl71q/sw9oE3YvZzo270HuEI9TQqfvGHuEb3TvHm9GeB+PevRAj25VY+8bFg7PfDepb0qAik9QEHjvD0F9L0+7eu9meeaO+Ak8r1n+C2+oooNvUISyz2T+qY7ZYYbPUrKrb0IVmO9lb+DvigykD3yQl+85vwePStVQb7SGES9zxcSPZejLT5915u9zle6u+G74D1I3+29hA4Kvq6pPj0xOAg9jOHmPQaJxr1VvM49EaWavQ7xcT1UBEK+LHREPTi1OD36KIg8kveiPFlff704z3U97lIBOxcm+TxPJtw9qLuNPCgv1zzuPw09Va0mvSILsrzjOVE9Wi5WPbNoHz5Grr09pBzHPUhs8z0rTOw8oZc+PV5aQb680NY90wS0PDpQujzio4M8kIkjvgW94j3X88091o84vcGtyj3O7RA+1keiveFFrr0/6tI9VVtuPaUwJz5VkbW8C29bPVfd572NBiW+ZrQLPN4A5b2KWxE9MHeEvSdaybiPgIG9VWwgu1ZG3z32Wtk7/Jq6PA9WLz5osCY+nI64vGfzWj2GXX499bJfPmUgQ70W2U49ZD+TvGXzDT3Yy6K9WAGLvTd3Pj3Dndq9SNCIvUUTDL7aOAS+ey4Jvm5+Q76tXAS+i1M/vZjOvr0NcBu+3AgRPSSDIL4txOK9vj4qvtvYyD2bota98ZMQvW8wF77jaEq+GwtjvWLnST17UaO9fJBJvg6w5b3jy3s9zFrkvFRTFjx0XZK7jBxCvfFrTL5aDrG+Bi+CvrR0Fb6J4la+bvZvvi5Y9r1jWNq9dPgsvSoIITzL9kq+E1FLvCILET4TUQK+TDYpvmiYvb3+xK29IQdDPd+xmD0F9Z697YldvaTRyL0FCKa8yObfvT7N7rzI32e9Vs4HvrLN3L32gi++lWAAvR7yzr3n0mO+","PbTmvOIcCL1Zg4280ACWPC1rIb6B/nm9WlqtvV4Rbb1u5y++hLwwvYjbhL2DGtO7dQWfPNI8Vr0dLr+9qpXLPPC6CL1vJWi9utOxvFZx272JEzC9bYPPvZUOCr1QEfg8Grsdvtwtgb3wFaq9wLkGvr5WS77Qf3A8puZGvr6KBL7QsAc99M/WvTI7X74VCN+9w31ovbVk6LwU9wS9w7YYvtEK9r0vIN29khc0vuntNr2Hyuk9ySOdvcjXMr670/O9xRREvgRH7Tl/zr+8mff2PO/dE77HSie+X/msPJwVLr4W1609GnEWOoc37b1kXda8wwFpvkvWCL5d6gq+Tb45vmdEhb22ZGK+Ea50vYV7AT2jWsY9SGoVPivqTz3XTZU91YQfPR7G6Lgkv109vD7AvXZDcLxXdLs6Vj9rvNJ50rv7WbM7Do9sPJw+lb1gFJW7jw9XPRlGiz1wZ/w9oJrBPGq3EL1b/6+9sVs3vW+eOj53zX490FakvI+2DT5w1/Y8yF+svV/J5LzBgHw9e240PEkFJzysFRM+cxrqPbEozLvbApU9JbowPctwJb3RywY9/ZZqPTYEjz2sfwS+GAEhPjNLmTyEIf09mvC8PVdF1LwScS687PKCPdLLP76efIS9hZtePadkBL0xe9e9bIF0O/Nhx7qHwnO9Hr/oO2mkEj5ze0E+YgBuOp+corxq+ZE9xsVpPhoi672eTHy92yIwvVrhjrtAK289Ds7CvIo9OL3lbJk93QGcvF+X8L3nef+8e7QAvYDOjjtCwDI9kdodPuao1z164Vo92fQaPQGDxz00Ay08Y5lhPLHhTrzsnF89tUZAPb/bCr4TNay9SKVlvIRhrz0hP7G9Z5TMPVMohDzDY+s7f4bwPF0UF76W08q9s3nhPGK2kb0ODQs++d6oPbuWZ71fc1c9sZi1vNw9tbytm2Y8oHt3Pd930T1vTEU9vpJSOUpmED1t8MS9FHJZvTKyvD01kAi9mYRhvRv+9j1QQAK+QyHvPS9Jbj2i9mE9","+qW1vAVqCj08XyA9W20hvrkK0DubNwa+dQufvayDqr2/xtm86bMOvto8rb2YPpa9MHTyvcDZz73YV4G9ksY7vf50Ur6cU5G8KdlYvQgVPr7tWCG+dCozvSFns73lypq9wlHsveMr1Tuk8aS9q5YdvvwG5b1TAui8JHIqvpz5SbtEpVm+C++AvnIW6r2pmL290Q7ovaSvX76kei6+bAhQvngGqL35Ao490qUCvgirEb1CD689F24Nvv5pN75u95G+n+KCvk7KdL3bTRw8m1w0vjoqsL36oAq7GAJ8vWjELL4ZRZG9Il0lvWnqwb0jrHe7KlrVvL2rO74rgGK+j4QrvnOXhLztxHa9LZQgvUSbirz811O+r7k1vvthsb0Ziz2+UCAlvvVYAbz+7fG9QtDYvLGiqr1ayhK+IPkovXVcD72oldK9sHT9uwxQ/rxJJ8S+dSHHPJCc/73Dtsi9MXXjvRyix70pWoG9mrb7vZedC76CiHO+ePpUPslmHL7hbya9mXYpvmQHKj3mgCu9Opi5vf44fL1V4UC+AuKnvFSmXb5pcyO+Tna7vZ+lM7600Xm9rjAOPb8llr1tD0W+g/BGvg4IwjteU6+9swbpvYFzDL7+tIi9iJuuveY/NT2Xs4O9ahjhvT5uML5Tt6C9+UBBvVIos72xWb+9OPrIvcKkcr2cTCg9dyu9vepvdzwKGdw9HpccPkFSbj3s6Fo+b8F7Parpmb3CYc+9tEt3vQlgC75dAMS9xk4bvaR54r3u9ei96UiCvUIH8rw8bQU+BCWbPTeqzz3pSiM+Pv5PPPKQEb3bMAG+DgcGvtF4kL2g2Lu85O6pPZaAuztJf4i9Zs8GPTSCKbxER1S8Q3hZO2UA3TtGp409YG3NPXSVLj7ZjaW8iMUDPiKBDr6hcsS9LaRePdXJCz0KjSk+0mkwvgDm4z1TaMQ9XycFPZcaGj4oeGq9bT4YvidY4TwRKtm9B1zKvT9Bgz1N3xA9fkjYvbulCb7tQ2C+9S2hPcgZQr2SI2o+","H+mjPaYeT7wb3PK90F7BPJHVHD4slaG921eJvNHbcDy/H+I7ciVQvHcnIL0H9qY92w/tPeF5Hb0yniM9PNqdPcgOzT2yYdg8ptMPumWsND2vxx0+TbGXvbuMXz1UVTa9VJRlvGLFyL1ZVZs982mBvCCsITyXYwS+nLJJPQCdvr1JqCe90LjPPUZZqD2+N0U9n7HAvSrkTrkOESO+pMH6PeCQRTwUh1Q9SL8xvl9ywT2K2dS92yjMOtgd0zz4CwG8m7ymvZ9E/byFlsQ9r//ZPbQYuj10EPC8TS/RPUOM1j1Ib749A7EmvQ8LtDuytF28Cy6hvD53yj2nVuO77OwyPeVvyDv9ek29IeQuvMrzsjxsASU+lK3gPYSZYD08ymI9RCQhPYWOXT18EHA9icKaPS+PDT6YL/I9EH/AvcK1oT0ZoMo95IN+PCmEWD57yfs9WvcXPueJDj4P5649pbv0vTegoj2HJPE9//2dPYDMET5FqgI+3aHdveuZwr1VaqG9CiLhPc3pXz6/GBs+yiRVvdN4OD5BDE0+W5qGPpP/nz4n/+M96VyYvIT+2D0p8lk9XisKPL+YkD4HmQo+Y5BFPtSqaT1YgxI+doJLvXUVvj1iZjY+WJOEPYlWpbwpIxs+2iMDPtgucj7BuVo+vo8GPVdGIT5CuwQ+AX8LPtKJvz0jhAg9G42fvaFkZDo8A4w9d9VWPhoKzzxyFs68csIZPqDvOz7BYTo9S+CwPdpygD4f04q8ndgiPpbGr7xcGlU8bUJjO4TKZz0oo6i8FfElPqUCk72QniA+FBgNvWTSvzyKVws+LsA7vQumvrz4b0g9GZNPPvi9vr2gpqY9Pa2yPQb41DuTO9M+USPGvASdKbp/KtY94lM4PkT0qz190mA+RFltPR+b8z1jFhc8CES/PZ00qbyuCYg9bdS6PVHMXD5JW1s81P6APTXo/j3qAzE+zi8sPVnAqD0MDzy8DDqKPqCukz0n5ZY9ew2zPZN8VT2lc9k9H8ECPUWnAD7GU3o9","3XOvvQKwmb0PcuC8irvoOz3CT74I7Oq90LYOPNsTkb0oMf466unMu8rSHL4Nq4o9Bw8MPcCWET2IejQ9wSrdPRD9KrwU5B4+ryiOPZLIqb2w4+S8VKJJvZlBwb27TRa+VPbLvHPJMDwkkbM9ZCfGPUdwgr2wstm9YacSvsvVIT1545Y9fMZXPRWLvLyx/LK91c2SvSwx3b0O3E69F5a1vUffFr6A82E9nqpoPV5PjDtc5+69+qk7vokFVzzTDGC92vWwupSt4r1KS8s85aJzPZ/+5DyzRpA9CLvJPQInmj6MaoG9JDlcvWFvET48ORM75DVFPLpkA72C3JC8pmvvvfryyj1KKCy+QZL/u9YqxzzVdcu9Xq9DvtL6f71z2cO9Px94PUD/mj01Sx49v6rLvZc8m71Wo5g9WhNKvHbpHz7PIAw+xMe4vWeC8rwwGcQ82qpyvdA+Cj4nfaY9cnU0PUF/CL6tzqM8XZRdPTWbxzyNFbo9rojCvE9PMT7em/M9RoowvlPgd71W/ay9YXJ+vFNqAr6BPtU9yWxYvO9dJT14SAQ9JtCkvYFZwr3ax5q8CvkGPZps1j1NV569IPuSuWdN4j1Nt6+9f1i7PIdjjb3JXk89xW/DPWd/Ar3PFcO9lD6XvYLvKb3cRcU+723RvHsKIL63Sk69lba2vTA9x70dBh0+E272PQbDVr5YU04+3E4TPFbbr70DlE09uGknvQr/Bz6Edog9dfgBPm/YNT6/GNU9M9G6PXsIzz17PcA8JXyrPTx9D73NbvU9SFmYPbt5jT6hWXM9kXjvPeWR2z3kDMQ9+SiQPKvRSj2RdGw95g8XPgXe/70j3Ss9bbmQPa4lDT5wySE+MuLkPXnY172tlfY88shEPgYTEz4vuNo9TXABPt568rwuSYA8f3UoPvlKsTzMmA8+qJdDPg8APD5sTu494apJPmbZmLwoJkg+ICgMPsrvLD3McAY9wA7ZPTGMgL05EcA9gdBbPbP3lT1o/+e9W0AjPZj88zwejrI9","zlziPYwObj2NkY47kOFFPrAAkT3k2a+9gXVRPgPOC70765M9Qa0tPsf5FT55YYC9Sxd6PVpElDz0cXc+2aiBvSl1Gb2fcDe9C+oAPu4ulL0l9+49I3v5PTK1VDyhPxA+BdBSPcsIZzwm9689KqWiPQ9ZfT0bQCq+vYgIPQHohT10SgI+a3aqPs4SbT0Wzdg881FMPuT3Az5lKCc7wUTRPFiX3T2uVeQ99ZyBPWeGrT3/+JM+cfTcPDLGpj2eDQc+PLc9vTqB7D1f7hY9Q1VxPovapz09kAk+QqF1vav3MT7Jq+E98DNbvXoW6r3ihJG8NvXwvC4iv7x26G09eI2EPnx5M7322a69ckShPSiD8TzwoLW7mWDpvcAPm72I1Ik8uuTjPdmAEzoEtn29101XPi1zQz237468ckgPvlmEuj2uecw8rTpePX1qWb3ZH8i9wiMvPV1vlTymuQS+vZSYPfQIhz2R8ey9sLpFPXbD37zZWlg89k4DvTm2kj2JC+W9j/yyPcxsib1TlNI7GrWjvZeRyDrSqM08SiwFvu6uy7wDIMu6m1mQPWyzcD1HUN687TSQvT1o270cpAY9752Bvc3mmjzTXmC+3WagveSviL2BR+Q9Juctu0Hg1TtROKA9kFgbPuZULr6tnRs9w/rIPSZhmT21AOc8Ea2OPYVF7rtKJ688u+HdvdcL/T1Lubq8+jSBveKOvT27uz49/5OQPW1zrj1AMJi9/wk+Pd/TDD0d/Ug9iK6APfFdqz0N0jY9rVaNO+wxFb0wz4i9OXcmPhp6oL0mnlw7eEvTPXVm0LwMXfu9Fm9vPpg8az2RPgu9DEugvSlXrr2Rno49WvINPaZieDxFs40+NX4lPtuPNr0qCEk9Qkm5PXykmz03lyE+/jORPd5LLT746zM9s1ZjvOlCPD59Nog9KqwUvHsHYr3tohO8KkevPFQWAL06IH69eAjjPdSOyb0EciA8gAv3PXAv9btQqHO9lxIAPYOxwDyflSY8FnctvKMA9T0day+9","UzauPfKbib2TtyK9zcIAveVGSj27UrE8tzlePU1FUj5vPAM9TYi0PbwqxT38cRw+8unEPWBLrz3hnD89wwsUPVttND5XWsQ9kfSNPSxWJz5fbC8+tDQUPmlXPj49UrI9jFHVPQZHYD0R3Ro+szCUvHRVDT59kBQ+j88OuvkePTwE1jk9WPKfPkHnPz4Pupk9NEzzPXPVNj4lkRQ+rcRBPqKNLD2YHZI9y0jsPUcOAT1P+ik9GuIpPnVMRD4PSb89n10aPsBEpzzRN0k946aJvAjzoj192/U9+2qbvRo4Qj6VYAc9iOzIPSqBGT4Us7W8uKAXPTZjWD10eCA+xUxWvSgLxz3m9Yg+qBwZPgcNkLwgFUE9clOPPYcOFT5kxva9CSBrPqxR0z0NZko6b0G0vVw5sT3Q28M95xuMO5VTxr2jH448DGxOPcNnyT17AcY9QkcQvi1luD0PkSY+3xfNPTUZZT0x9Wk+HgKiPWYM0D2OpUE+Xta8PUIEHj7ar/c9CF8ZvQS8Zr72pH49wtBbPb85mj1xTFs+hIf8vGSSHT68Ic08XVwXPpU+6T1eIOq8IEGAPjYi+jwM8gg+zGDVPalIxT1nwP28k+9wPEgKHr0E5I89jg4pvYc3fT3w6ns9KYVgvTpa7jtAFhE8v/68PFxcfj0u8o097tJ5PSe5LT7s2hW9UDPEvZwLPTviT5U9sOPMvdsVZL3cy7i9G1ezPbE3Or1TbrU9tojUvXXPyD1GuhA9IVbUu3jc071n8o49SZINvfKoLDxemk09G1mXPaoht7wBdsk8diLPOweH8bzR+uk8FEdJvRz4Kj4+Ygi7a59WPcbyoL35kRS91cUnvgR/0DwyGb48C2xkvXl3ybztcsS94VGKvbDniL5eOI+9BrMmPqL6ir0CbEs9pPFLPXLus7tmNYS+w+eePcLsKL2l8FO9A4UCPkacTr3/OMI8IgCvPCcSk71QOkM96kRUPAhWQ7v161s+rectPKvt4z2iPwi+Cr3pvXrF273MzNU9","2/CpvXNMyD0YzBg8zxtTOwdXOb7IunE+/GIDvhxjAL4R57E+w+n7PKkvJr4i2vY8LTSCvIZRjz2m2eS9Ilm9PeBnF76bRuG95Ao6PfYHAzjbJtE7bGYUPo+Vnr0FskE9mqBTu5Ya3r1ErSq++TacvTwM471jgNc9rIqDPtRS3b2IS7G8Vz3WvWGiN71wLQm+Ha7MPKEZIT2lkik+FC/fvO1+Zz3RHgO9NxE8Ow1YUr3tUwi9tPprvtr27728+Zy9ir0/PBLXED3BrM08hOWrvsIEwbxRloE9zX/ovMJg1r03ZMw9ZzLQPdAxjj1l6t+8n8KAPc2hsz08xBa+CmL6vWR98L26p6K997wfvcjMJ7tt3ay9y8bpvdr9Y73+evK9XdVhvlgAELzKO0S9ccxSvSfA7LwDn1a+Hecyvh3vXr0TQHu+rg8VvsVYTL1zBs69ZVpTvYQ+Gr5oJF+9g4sevuo1or2+ALa9feVavdjiP70pW9W9+CGRveFD/73ahFG+/yNtvrhajL4W4FC+CkLmvRxaqLzoCXy+B4+evvOP9b1hTUS+utiivaXpIL7nUYk8z6cfPV7RWr7624S+KuTwvdMOg738LH49ds+evaweLT37uiy9Q0gJO1+Hub3923++ZG8wvjdXXb0nChe+Q33OvJ8HPb7WTy6+60YjvrLzPL7PJ/69UHsnvPyi373LsAs9maBGvXGemTzqppE9fRTQvRHr4r1nY2C+v9epvC3LAj77P769sJ0WvtYMuT07Hja9qYCUvW2bvL24LSS9jc23vfjcDL77/vI6JZ4cvptoTr4RKRa+QDkmvqNNnb1z+iI9DIJNvmbD87z8Yla+XkKgvV9GpLzW/8w9qcyLvQnmNr4irCa+1Gw3vsb0Lj2gCF+9RpcLvi/j1b1WHcK9g53qvDhR2b0DdL4857n7vWQCML5cX5u9h4kYPoVgFju63Ms9RHuYvXtcbb7CJp28C6Yhvg2pSL1j7va9fGnOvRHTtjwLCAK+4XH+vY1Jq71anV++","/dLQPbd63bt/LhO8nUq9PcXRYzswnZa5dSqQPZ7QCj6LEFK9zADRvZ2Jmr3RMpA6dX/ru5BiEj0MdNk9jNC5PF6JrrxlTnG9/DlivONrOz3dgY66iZ3TPTL3Hj3oOGS9NXLBO3hEaDsLzCS+7wHlPDt+z7yGgi4+8z3YvdHpID0TaZ698yEyvBHXvD1pnx8+AQPoPRZWHz3fTRc9rqpGPHPyOj0NTcC9RdGiPPI2jLzHUQg9iZ06Poboj70uHu89ej+TPbE0b73axt48Tze/PdWn9L0Tj0E9fuonvqgD2r3VTDs+kQt4PKQAP71D8Pe8ipMJPUSbGj5+C8o8NHbgPSF/oLy/0B2+nIiAPWmM9rw38d09XBAXvpF3tr06Bw89/u0cvqRdxj0SqSQ+PPMLPgm4RLsejVe7PijpPYxV3TyPaeQ7GM9DPBNzyrpjjeO7k6CuPWU99j1QlaO9nH+ivf5Xyj1RyEe+HZY8Pm6bGb182Ea8P6qpPYaAJb63w9e6MqKcPJvPUT2DGiG73xwGPcG3Tz7DIlE9eBmLvqPnxz1MmWs587DiPE68971CExe9XiXMPPi2SD2oec69wicIPAC85L0ncpa6htwUPdtu/j3C0zs9UU2uPaYIwT2Fhv+7G2G3PDaDsr26CRC8yUgKPZ7eqj2Hi1u90ZA3PSRceTuRPHc9/jAbPHmaI77iXqU9w9nGPSEvNj6b0G49aqRLPI2lAD4NgQ296LMaPmibij3PUPk9uhSdPRVVLD3WjZo9drqCPk4JFT0ChwI+qj0jPsEXdj5KsLA9Jv0WPT1vrz0B4RM+5ixOuX1k4j1WLq69xLIlPv/c072C0FE9j1qrPTxmJj4ZMqI+i1BSPqImLTzxrmc+J8OYPakaVD703VI+5jfjPem0MD6C1yQ+354CPe9ZPrydk14+fp+KPQXgcT1CPw0+daoMvY0km72U0O09mjtSPbiqtT23lsw98nV9PsSrHj5itdM9GCrrPeZ67D0pdyC9SecfPlWRvD0a3+Q8","VnADPdKF4jxr6D+915kCPoWLQz4+wCW9G/TKPO+x+zwllfc91F+EPWpJSz2AeGI9ydVePoojTj3i8Ak+Bu6gOzV8Az5P1sk9N66uPRCp7T3IIgs8+VMlPmz9Cr32GR4+LipzPec2jD3tnMA9RLVOPfPRHj6JXuk806pWPB2HmD0Y/4682curPMwuSz6FaeO85QIlPTz5BT4UQiG9LktbPgc8Zj3V+qM9hflZPsvcnj2BgZM9jv/gPCCWXD2PgSI++fF2Pc6GMT79ceE9QAO6PGWWpz329jg+qSJsvbN1ED7EcyE+9k6yPbVL6z3qRKw9Pk0VPu6Qez0r5MQ98Zf6PUvKJjyAnWq84+gPPDvR6L0pGKG+7cCcva+3N770EFG9g+k0PY7bm7wC+ys8pB7cPeMr/z2E28k9Nq3dvBMH8TyR/gQ9bv3APfJ8YD3ZkLu7tYCMPXMddr1FkMe92z2VvJGxwz1mu5q84+sTPTVH3T2FFFi9Z/1OvcisxL0Z3BS95cPBPQvqyj1pjAC9atOePXx+k73NCvu9Yl5/veLvtLwqx5Q9DtHbO+Tyor0DeqO9DERrvafa873k8pE9nIBzvc8yRzuznus88FlWPHRD5zzownm9u6vWO4z6Jj2uoHA+pJvvvRVZpz1tYg4+BmWSPdw59j3r4b+8EAcoPnXDB77a6vG9FmaCvWtQgz34QQ+9RsA4O5TrbLvnsn88LHwLPpNIJL1+Ayk9npHQvQaaoDzwKlG9foPAvdO1SL0ShTu6+VC7PXR9gDyPGYu8UEvcPa3LtL2HUC09Ec1GPcJvej2pGS888RIdPp/pxD2wInC9LgeOvHUA2j26ZQS8MPnEPMGyCb15Vyw8aEX8PT79pT3KlYe9zx7HvSJ/WD7DdXc8K/a7PS7kD77zcb09+QTpPKw33L3uAla8TkymvZsk6j1HzY89b3h/PaKsp7ypEiG9yIOjPExNML3zF488OdvPPaUMnbwakQG+F4m2vHJbub1EZW69jDwWPb+O7DwOegU9","tM7KvGYrbr1LNZE9sTRAvuefq72WBNC9qE72vTdpPL6XVDS+qnbGvauwlb0VToO+s1aMvZ6Yy7yBBVS8Rbh3Pe3bD76HfZW8IzdxPDodzb0jJO69D+DFvdB+Rr5EC268WpQJvkcWIL6bLkW9kOU+vf2KG73O37y9t7NfPVdQxjx69yi+3teHvt1wGL0Fniq9Fij9vYcPm72vFIW+IaGMvn0CFTx1HU+94Wq0vQH2sb2Of4o91T9rvqCEpb0+Hl69ma/DvWMJ1z3C0AI9+1ayvPdvLL6DwTq+NYYfvji8ML4szBK+ZMlkPBCfhb08R1m8r+9UvnfWo70UlC6+HERSvVu+JLyELVe7fKqjva6Rhz35xxa+l3TcvXaCab1BtBi+aUlJvs5SDr7TrA8+B5xYPbnMBrvHK3g77GQ+PTQ0Yjwm5qy9JzkRvrI4ur27JnC+ZIWovIQTzb1nRW69SB8Uvgn6HL6hbcW9P8IZvkV6FD2lt269F0ozvbyOhby4iee9IP3UvZiCC77mjQi+gnknvc+Mdr2nyWW+PsM9PXJTWr69EAW+Oj8cvrIYPb1LxpC9twgAvtgs8L2qig08fFWRvX/IHr6zbqQ9SkQ8PP1t/r1OTt29qXgbvrfzHb6v4RC+vvnuvf3T773IaMa9lFcLPcwF672QVza+wB/MvW6ypjyfjzg9n90bPDwiyL1ecTC5AGttPe35aT2gHRM+cMkbPc3Qqb2C/2C+JZnQPYwh7by9gNq9Gr2JPUyzlT0K27M99NmmPAi3jL2FxAS+lLF+PUy3wj2h3oe9GvlOPoJZ2ruDT8C9rcBrvZiozL3cUBo+xQGyvbvrCj0O9qS9CX2JPYGdub13JK+9DhwbPgQyhj07dcQ8/ITnPSmoRz051oe7r1vRPOA31zx0Noq9q/7MO3sa/jzlAlc+HOaMvJyYZj0tNjE9ubN5u+b3UbuZCv28AMygvTkd+TyzLA++vR0Lvukg5z1wjOa9xl22vVgLUb0gHIG9oCy6PYSqkDpXMo09","42NbPUCCKr6Ghia9Ldr0O7RVLDy+s9M9X+gIPVQHerzLeOw8G8gcvqADxT1+D1q8o61bPdeml721jw8+H0nCPIZLlLxGvoc++9DIvcgvDrzAWWA+Z7/2vPEAHz4fOXE8+WOBvYDWor2utIA85AkPPs4xD753qQo9jK98PpiQ971Wy5k9CUd3PHS41z29Y5q8AdHWPU17fT1lmkO+iVdbPUazKD3nIIY80jdGvvYqFj2OHyw9eKHBPZIV47yyguq90SVCvuybNj38DUC9KPxvPfj6TLx5ATs8mkHgPXpyIz0CUAg+L7b6PWLxljyUpNs7ZQEfPo5FOTzdYoi8pg+MPQ=="],"bias":["h6+JPOdPtT13Z9S8vb3gPUpYGD7VPhs+ONjiPRcY5D03PbA+Q+/NPRx/rj2j7Js9nkWvPZbN3jxdO+w9vw6zPYG1Fz4tfjk9tmVjPjiZET6TAeU9soW8PTFHmz0y0yY+smaKPO7kwT14q5s9yYiXPQpvOD6G41s9VPDcPXOVSz1VaSE+uYOsvCrFJj7yEyA9bMTfPaztSz5LuI8+rdB9PaMfBj7ieBE+UXK0PSKFqzyD+7g99qECPsrDzD3t4789n5XyPSDKuD3WgCw9LKUiPq60Dz4T7Ls9xAxYPYh9Pj73mAw95giWPfkuET6RIFo9l/swPlzL7j1e8pA8xW1TPR12hj9L5YQ/lhN+P82/gD+V9pQ/B9qIPy7lhj+wDYk/SlCTP/d9iz9Dvoc/vJiAP7LmjT9EOIc/sgeFP/9ugD/FKYg/LXd/P8oiiD/lZo8/dqCDPyeqiD+024Y/q8yKPwcTiT8MRIo/5NSFP/gLhT+aaJE/gHZ8P7aajT/uH4w/TXuJP1sNhT+OIZU/222EP28HkD9N6ZE/cVCKP6JrmT8Xo40/u+yLP6sgjz8uooU/0aSFP92BiD8YUpE/tkqNP0/Thz+t7IY/Q5CCPxSWhT+oKYw/xbKOP27PgT/n+pQ/cTuEP9jYjT+KW4o/fe2BP7umij+eIIs/KVKLPzuNhj+83QK8btUgvXRBBb2FDzg8CNI3vid5C716EIc7YIdQvd2MY701AI48XnTJO6Jz/rqaCSy91GZivMXl+z1pgM28QnF+vSrZAD3lo8O8gbcuPUb2kTwCN+G85B6MvKOCg71Kfx09RVBRvajeST3TnYS9tZFXPXOKT7yTQZa9AYkBva//Kj4rxdA84mhuvcMv8rwaXhC+jY8yvpj9kL3r/ju8hwqhuwGaD71WSAS97rN6POfrx7y79rO9y5+1PS0NmL70MqG9JDuRvW4qTj0mJIW9Mf7NvLGEyLzk7+y6UG4lPjhsmLyCf1i9UpJ6PjFFHD1GS7c9rS1ZveeXubxB9L69","CG8BvSBpxzwL26M8iLpLPKX5pb1D9KY8Vlb4PDHBjjxS8Do956PeO4mgjbqdNTQ8BNWSvCdaBLyGI486BdwbPG6cq7zEkLe8EdsmuxhO3LvKLou9GjghPcO8v7wfQLm7ZBunvHishT1fHry8SeICPKZ8az0sKfi6pfu2PKWK8Dxx3zG7vAggPDalQbuKzwu8Z3Ulu4N6cbsRQt49gXyePDg+trshNhq8k/bAu5uy1LvCvcW7mpcTvbAnA7vUsaq8e6WkPPJylbvWhNK8Klp2PAfJFDyDLh0771WXPN17ETyogDu9CgnTug9zzjywlpi8VmJNvEHbVrtxZMq8PXc5vQ=="]}},"hash":"9b9e45f62bed58569113b378e195e1f3da21a927af47866e2b084e72f4442d68"} \ No newline at end of file diff --git a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_metadata.ktn.model b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_metadata.ktn.model index fc3a494020..e69de29bb2 100644 --- a/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_metadata.ktn.model +++ b/src/kernels/gfx942_ConvHipIgemmGroupFwdXdlops_metadata.ktn.model @@ -1,75 +0,0 @@ -{ - "predict_type": 1, - "num_tuning_params": { - "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3": 17, - "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle": 15 - }, - "decodings": { - "tunings": { - "0": "0", - "1": "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle", - "2": "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3", - "3": "256", - "4": "128", - "5": "64", - "6": "64", - "7": "16", - "8": "128", - "9": "32", - "10": "224", - "11": "256", - "12": "128", - "13": "32", - "14": "64", - "15": "16", - "16": "256", - "17": "224", - "18": "32", - "19": "64", - "20": "16", - "21": "128", - "22": "Default", - "23": "Filter1x1Pad0", - "24": "OddC", - "25": "Filter1x1Stride1Pad0", - "26": "32", - "27": "16", - "28": "32", - "29": "16", - "30": "1", - "31": "2", - "32": "7", - "33": "4", - "34": "8", - "35": "2", - "36": "1", - "37": "4", - "38": "8", - "39": "7", - "40": "8", - "41": "4", - "42": "1", - "43": "8", - "44": "4", - "45": "1", - "46": "8", - "47": "4", - "48": "1", - "49": "2", - "50": "1", - "51": "2", - "52": "1", - "53": "2", - "54": "-1", - "55": "BlkGemmPipelineScheduler:Intrawave", - "56": "BlkGemmPipelineScheduler:Interwave", - "57": "-1", - "58": "BlkGemmPipelineVersion:v1", - "59": "BlkGemmPipelineVersion:v2", - "60": "BlkGemmPipelineVersion:v4", - "61": "BlkGemmPipelineVersion:v3", - "62": "BlkGemmPipelineVersion:v5", - "63": "-1" - } - } -} \ No newline at end of file diff --git a/src/kernels/gfx942_metadata.tn.model b/src/kernels/gfx942_metadata.tn.model index f1b41b6c7d..e69de29bb2 100644 --- a/src/kernels/gfx942_metadata.tn.model +++ b/src/kernels/gfx942_metadata.tn.model @@ -1,486 +0,0 @@ -{ - "generated_on": "12 Sep 2024, 15:50:21", - "database": "tuna_net2", - "gpu": { - "arch": "gfx942", - "num_cu": "304" - }, - "golden_v": null, - "kernels": "hip", - "num_inputs": 18, - "num_outputs": 21, - "num_solvers": 21, - "encodings": { - "Direction": { - "B": 0, - "W": 1, - "F": 2 - }, - "Precision": { - "BF16": 0, - "FP32": 1, - "FP16": 2 - }, - "Layout": { - "NCHW": 0 - }, - "solver": { - "ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC": 0, - "ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC": 1, - "ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC": 2, - "ConvBinWinogradRxSf2x3": 3, - "ConvBinWinogradRxSf2x3g1": 4, - "ConvBinWinogradRxSf3x2": 5, - "ConvDirectNaiveConvBwd": 6, - "ConvDirectNaiveConvFwd": 7, - "ConvDirectNaiveConvWrw": 8, - "ConvHipImplicitGemmGroupBwdXdlops": 9, - "ConvHipImplicitGemmGroupFwdXdlops": 10, - "ConvHipImplicitGemmGroupWrwXdlops": 11, - "GemmBwd1x1_stride1": 12, - "GemmBwd1x1_stride2": 13, - "GemmBwdRest": 14, - "GemmFwd1x1_0_1": 15, - "GemmFwd1x1_0_2": 16, - "GemmFwdRest": 17, - "GemmWrw1x1_stride1": 18, - "GemmWrwUniversal": 19, - "fft": 20 - } - }, - "stats": { - "overall": { - "features": { - "mean": { - "Inp_0": 311.8293151855469, - "Inp_2": 219.17543029785156, - "Inp_3": 233.04478454589844, - "Out_0": 306.68072509765625, - "Out_2": 155.22740173339844, - "Out_3": 165.06072998046875, - "Fil_1": 2.209331750869751, - "Fil_2": 2.214216709136963, - "Pad_1": 0.5957369804382324, - "Pad_2": 0.5955945253372192, - "Str_1": 1.3636339902877808, - "Str_2": 1.3636349439620972, - "Dil_1": 1.0065430402755737, - "Dil_2": 1.0065430402755737, - "BatchSize": 15.462503433227539, - "Precision": 0.5352241396903992, - "Direction": 1.0002485513687134, - "GroupSize": 1.3724150657653809 - }, - "std": { - "Inp_0": 377.65289306640625, - "Inp_2": 249.92596435546875, - "Inp_3": 264.2414855957031, - "Out_0": 391.1197814941406, - "Out_2": 131.13681030273438, - "Out_3": 138.8323516845703, - "Fil_1": 1.870792031288147, - "Fil_2": 1.8942359685897827, - "Pad_1": 1.0165650844573975, - "Pad_2": 1.0164508819580078, - "Str_1": 0.4849567115306854, - "Str_2": 0.4849569797515869, - "Dil_1": 0.4083109498023987, - "Dil_2": 0.4083109498023987, - "BatchSize": 84.05658721923828, - "Precision": 0.5440565943717957, - "Direction": 0.8165649175643921, - "GroupSize": 9.064383506774902 - } - }, - "gt": { - "mean": { - "p0": 0.0960695669054985, - "p1": 0.09949314594268799, - "p2": 0.17995589971542358, - "p3": 0.00861922837793827, - "p4": 0.05461062490940094, - "p5": 0.06398279219865799, - "p6": 0.0021943366155028343, - "p7": 0.0033684158697724342, - "p8": 0.0026517012156546116, - "p9": 0.029072733595967293, - "p10": 0.044882066547870636, - "p11": 0.044395118951797485, - "p12": 0.08627412468194962, - "p13": 0.0201451163738966, - "p14": 0.045423805713653564, - "p15": 0.08265639841556549, - "p16": 0.018358981236815453, - "p17": 0.031365618109703064, - "p18": 0.04177531599998474, - "p19": 0.04464886710047722, - "p20": 7.558641664218158e-05, - "t0": -0.25846436619758606, - "t1": -0.5613387227058411, - "t2": -0.5604408979415894, - "t3": -0.4562467038631439, - "t4": 0.23823484778404236, - "t5": 0.5563516616821289, - "t6": 105.59883117675781, - "t7": 4.235169887542725, - "t8": 23.11374282836914, - "t9": -0.02311881259083748, - "t10": -0.4534555673599243, - "t11": -0.6263357400894165, - "t12": -0.8252652287483215, - "t13": -0.943831741809845, - "t14": -0.7131063342094421, - "t15": -0.8236099481582642, - "t16": -0.9444738626480103, - "t17": -0.6957222819328308, - "t18": 1.2380682229995728, - "t19": 4.227562427520752, - "t20": -0.9996898770332336 - }, - "std": { - "p0": 0.1670387089252472, - "p1": 0.1623845398426056, - "p2": 0.2798249423503876, - "p3": 0.0328247994184494, - "p4": 0.08603879064321518, - "p5": 0.09141649305820465, - "p6": 0.020351815968751907, - "p7": 0.021183520555496216, - "p8": 0.013209610246121883, - "p9": 0.05328960716724396, - "p10": 0.07091464102268219, - "p11": 0.10614921897649765, - "p12": 0.2076387107372284, - "p13": 0.09995308518409729, - "p14": 0.14905880391597748, - "p15": 0.19885306060314178, - "p16": 0.08844757825136185, - "p17": 0.0916207805275917, - "p18": 0.13645581901073456, - "p19": 0.1354658454656601, - "p20": 0.004838536959141493, - "t0": 11.287893295288086, - "t1": 1.3553531169891357, - "t2": 1.4136779308319092, - "t3": 3.0234830379486084, - "t4": 5.053164958953857, - "t5": 6.3723649978637695, - "t6": 723.852783203125, - "t7": 28.936826705932617, - "t8": 233.1973876953125, - "t9": 5.534809589385986, - "t10": 2.193744659423828, - "t11": 12.12098503112793, - "t12": 0.5163028836250305, - "t13": 0.43445533514022827, - "t14": 2.4269120693206787, - "t15": 0.5618398785591125, - "t16": 0.8459757566452026, - "t17": 4.2913289070129395, - "t18": 21.263639450073242, - "t19": 98.69822692871094, - "t20": 0.019548865035176277 - } - } - }, - "train": { - "features": { - "mean": { - "Inp_0": 311.9007263183594, - "Inp_2": 218.98898315429688, - "Inp_3": 232.8292999267578, - "Out_0": 306.86285400390625, - "Out_2": 155.14125061035156, - "Out_3": 164.9310760498047, - "Fil_1": 2.2081074714660645, - "Fil_2": 2.2131288051605225, - "Pad_1": 0.5951849222183228, - "Pad_2": 0.5950822830200195, - "Str_1": 1.363242268562317, - "Str_2": 1.3632433414459229, - "Dil_1": 1.0065900087356567, - "Dil_2": 1.0065900087356567, - "BatchSize": 15.449292182922363, - "Precision": 0.5352196097373962, - "Direction": 1.00015389919281, - "GroupSize": 1.3728113174438477 - }, - "std": { - "Inp_0": 377.3197937011719, - "Inp_2": 250.1122283935547, - "Inp_3": 264.5950622558594, - "Out_0": 391.1640930175781, - "Out_2": 131.22291564941406, - "Out_3": 138.99365234375, - "Fil_1": 1.8697174787521362, - "Fil_2": 1.893709421157837, - "Pad_1": 1.016906499862671, - "Pad_2": 1.0168226957321167, - "Str_1": 0.48490116000175476, - "Str_2": 0.48490145802497864, - "Dil_1": 0.4103623628616333, - "Dil_2": 0.4103623628616333, - "BatchSize": 84.02078247070312, - "Precision": 0.543916642665863, - "Direction": 0.8165029883384705, - "GroupSize": 8.983871459960938 - } - }, - "gt": { - "mean": { - "p0": 0.0960896909236908, - "p1": 0.09943703562021255, - "p2": 0.17996086180210114, - "p3": 0.008643264882266521, - "p4": 0.05461084470152855, - "p5": 0.06395480036735535, - "p6": 0.0022060745395720005, - "p7": 0.003366228425875306, - "p8": 0.0026561289560049772, - "p9": 0.02905450016260147, - "p10": 0.044861309230327606, - "p11": 0.04440837725996971, - "p12": 0.08628346771001816, - "p13": 0.020143287256360054, - "p14": 0.04540954902768135, - "p15": 0.08266698569059372, - "p16": 0.018406329676508904, - "p17": 0.0313330814242363, - "p18": 0.04185955226421356, - "p19": 0.04457731172442436, - "p20": 7.539901707787067e-05, - "t0": -0.26193925738334656, - "t1": -0.5615989565849304, - "t2": -0.5598130226135254, - "t3": -0.45621049404144287, - "t4": 0.23961539566516876, - "t5": 0.5567820072174072, - "t6": 106.39617156982422, - "t7": 4.2253851890563965, - "t8": 23.23782730102539, - "t9": -0.021923955529928207, - "t10": -0.4539487063884735, - "t11": -0.6242122054100037, - "t12": -0.8252379298210144, - "t13": -0.9438983201980591, - "t14": -0.7127044796943665, - "t15": -0.8236484527587891, - "t16": -0.9442780017852783, - "t17": -0.6947833895683289, - "t18": 1.232773780822754, - "t19": 4.232138633728027, - "t20": -0.9996882081031799 - }, - "std": { - "p0": 0.1670711189508438, - "p1": 0.16230806708335876, - "p2": 0.27983060479164124, - "p3": 0.03288647532463074, - "p4": 0.0860302746295929, - "p5": 0.09136135876178741, - "p6": 0.020536543801426888, - "p7": 0.021159816533327103, - "p8": 0.013212117366492748, - "p9": 0.05322789028286934, - "p10": 0.07088592648506165, - "p11": 0.1061607152223587, - "p12": 0.20765794813632965, - "p13": 0.09999261051416397, - "p14": 0.14899584650993347, - "p15": 0.19886651635169983, - "p16": 0.08854780346155167, - "p17": 0.0915924608707428, - "p18": 0.1365906000137329, - "p19": 0.13536596298217773, - "p20": 0.004827538505196571, - "t0": 10.987695693969727, - "t1": 1.3884049654006958, - "t2": 1.448920488357544, - "t3": 3.023371458053589, - "t4": 5.134879112243652, - "t5": 6.384313106536865, - "t6": 733.6824340820312, - "t7": 29.073034286499023, - "t8": 232.49407958984375, - "t9": 5.657901763916016, - "t10": 2.227865219116211, - "t11": 12.524832725524902, - "t12": 0.5252835750579834, - "t13": 0.4437565505504608, - "t14": 2.397005081176758, - "t15": 0.5709920525550842, - "t16": 0.8692423701286316, - "t17": 4.418712615966797, - "t18": 20.587675094604492, - "t19": 97.82730865478516, - "t20": 0.019699450582265854 - } - } - }, - "test": { - "features": { - "mean": { - "Inp_0": 312.3663635253906, - "Inp_2": 220.30160522460938, - "Inp_3": 233.94407653808594, - "Out_0": 305.9991149902344, - "Out_2": 155.6833038330078, - "Out_3": 165.35264587402344, - "Fil_1": 2.2203524112701416, - "Fil_2": 2.2240068912506104, - "Pad_1": 0.6007053852081299, - "Pad_2": 0.600204348564148, - "Str_1": 1.3671588897705078, - "Str_2": 1.3671588897705078, - "Dil_1": 1.0061206817626953, - "Dil_2": 1.0061206817626953, - "BatchSize": 15.581399917602539, - "Precision": 0.535264790058136, - "Direction": 1.0011003017425537, - "GroupSize": 1.3688486814498901 - }, - "std": { - "Inp_0": 380.6393737792969, - "Inp_2": 248.24119567871094, - "Inp_3": 261.0367736816406, - "Out_0": 390.7218322753906, - "Out_2": 130.35887145996094, - "Out_3": 137.3720703125, - "Fil_1": 1.8804082870483398, - "Fil_2": 1.8989499807357788, - "Pad_1": 1.0134791135787964, - "Pad_2": 1.0130925178527832, - "Str_1": 0.48544469475746155, - "Str_2": 0.48544469475746155, - "Dil_1": 0.3893638551235199, - "Dil_2": 0.3893638551235199, - "BatchSize": 84.37842559814453, - "Precision": 0.5453169941902161, - "Direction": 0.8171252608299255, - "GroupSize": 9.759186744689941 - } - }, - "gt": { - "mean": { - "p0": 0.09589540958404541, - "p1": 0.1000104695558548, - "p2": 0.17976097762584686, - "p3": 0.008403163403272629, - "p4": 0.05459993705153465, - "p5": 0.0642465204000473, - "p6": 0.0021069420035928488, - "p7": 0.0033879836555570364, - "p8": 0.00261111743748188, - "p9": 0.029240209609270096, - "p10": 0.045067887753248215, - "p11": 0.04426904767751694, - "p12": 0.08619356900453568, - "p13": 0.02016228623688221, - "p14": 0.04555174335837364, - "p15": 0.08256105333566666, - "p16": 0.01793222688138485, - "p17": 0.03166221082210541, - "p18": 0.04100421816110611, - "p19": 0.04525507241487503, - "p20": 7.727346383035183e-05, - "t0": -0.22696298360824585, - "t1": -0.5590552687644958, - "t2": -0.5663082003593445, - "t3": -0.4565463662147522, - "t4": 0.225840762257576, - "t5": 0.552692711353302, - "t6": 102.40654754638672, - "t7": 4.321577072143555, - "t8": 22.71693992614746, - "t9": -0.03393154591321945, - "t10": -0.44889819622039795, - "t11": -0.6453760266304016, - "t12": -0.8257853984832764, - "t13": -0.9432471990585327, - "t14": -0.716819703578949, - "t15": -0.8230522274971008, - "t16": -0.946202278137207, - "t17": -0.7043951153755188, - "t18": 1.2856698036193848, - "t19": 4.184349060058594, - "t20": -0.9997062683105469 - }, - "std": { - "p0": 0.166747584939003, - "p1": 0.16307106614112854, - "p2": 0.27977532148361206, - "p3": 0.03226378560066223, - "p4": 0.0861157700419426, - "p5": 0.09191123396158218, - "p6": 0.018606791272759438, - "p7": 0.021395757794380188, - "p8": 0.013187017291784286, - "p9": 0.05384185165166855, - "p10": 0.07117258757352829, - "p11": 0.10604614019393921, - "p12": 0.2074665129184723, - "p13": 0.09959709644317627, - "p14": 0.14962492883205414, - "p15": 0.1987328827381134, - "p16": 0.08753963559865952, - "p17": 0.09187520295381546, - "p18": 0.13523495197296143, - "p19": 0.13636067509651184, - "p20": 0.0049364445731043816, - "t0": 13.696746826171875, - "t1": 1.010352611541748, - "t2": 1.0442678928375244, - "t3": 3.0245022773742676, - "t4": 4.247585296630859, - "t5": 6.263833045959473, - "t6": 628.5001831054688, - "t7": 27.680801391601562, - "t8": 239.43490600585938, - "t9": 4.270197868347168, - "t10": 1.8586872816085815, - "t11": 7.572343349456787, - "t12": 0.4270634949207306, - "t13": 0.33946493268013, - "t14": 2.6811134815216064, - "t15": 0.471545547246933, - "t16": 0.5970855951309204, - "t17": 2.9034297466278076, - "t18": 26.584880828857422, - "t19": 106.21611022949219, - "t20": 0.01813751459121704 - } - } - } - }, - "conv_params_used_as_features": [ - "Inp_0", - "Inp_2", - "Inp_3", - "Out_0", - "Out_2", - "Out_3", - "Fil_1", - "Fil_2", - "Pad_1", - "Pad_2", - "Str_1", - "Str_2", - "Dil_1", - "Dil_2", - "BatchSize", - "Precision", - "Direction", - "GroupSize" - ], - "redundant_columns": { - "SpatialDim": 2.0, - "Inp_1": 1.0, - "Out_1": 1.0, - "Fil_0": 1.0, - "Pad_0": 0.0, - "Str_0": 1.0, - "Dil_0": 1.0, - "BiasFlag": 0.0, - "Layout": 0.0 - } -} \ No newline at end of file diff --git a/src/kernels/radix.hpp b/src/kernels/radix.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp b/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp deleted file mode 100644 index 02e675203d..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp +++ /dev/null @@ -1,210 +0,0 @@ -#ifndef CK_CONSTANT_MERGED_TENSOR_DESCRIPTOR_DEPRECATED_HPP -#define CK_CONSTANT_MERGED_TENSOR_DESCRIPTOR_DEPRECATED_HPP - -#include "static_kernel_common_header.hpp" -#include "static_kernel_ConstantTensorDescriptor_deprecated.hpp" - -namespace ck { - -// OriginalTensorDesc : ConstantTensorDescriptor_deprecated<...> -// it's the tensor whose dimensions are to be merged -// OriginalDimMergeSeqs : Sequence<...>... -// each is a sequence of original dimensions (of OriginalTensorDesc) to be merged -template -struct ConstantMergedTensorDescriptor_deprecated -{ - using Type = ConstantMergedTensorDescriptor_deprecated; - - static constexpr auto mOriginalDimMergeSeqs = std::tuple{}; - - static constexpr index_t nDim = sizeof...(OriginalDimMergeSeqs); - static constexpr index_t nOriginalDim = OriginalTensorDesc::GetNumOfDimension(); - - __host__ __device__ constexpr ConstantMergedTensorDescriptor_deprecated() - { - static_assert(nDim <= nOriginalDim, "wrong!"); - - // TODO: check each of OriginalDimMergeSeqs contains at least 1, and at most - // OriginalTensorDesc::nDim number of dimensions - - // TODO: check OriginalDimMergeSeqs contains all original dimensions - - // TODO: check there is no duplication in OriginalDimMergeSeqs - } - - __host__ __device__ static constexpr auto GetOriginalTensorDescriptor() - { - return OriginalTensorDesc{}; - } - - __host__ __device__ static constexpr auto GetNumOfDimension() { return Number{}; } - - template - __host__ __device__ static constexpr auto GetContainedOriginalDimensions(Number) - { - return std::get(mOriginalDimMergeSeqs); - } - - template - __host__ __device__ static constexpr bool ContainMultipleOriginalDimensions(Number) - { - return (std::get(mOriginalDimMergeSeqs).GetSize() > 1); - } - - template - __host__ __device__ static constexpr auto GetLength(Number) - { - constexpr auto original_dims_partial = std::get(mOriginalDimMergeSeqs); - - return OriginalTensorDesc::Extract(original_dims_partial).GetElementSize(); - } - - template - __host__ __device__ static constexpr auto GetStride(Number) - { - static_assert(!ContainMultipleOriginalDimensions(Number{}), - "wrong! stride of a merged dimension is undefined"); - - constexpr auto idim_original = std::get(mOriginalDimMergeSeqs).Back(); - - return OriginalTensorDesc::GetStride(Number{}); - } - - // this is a hack to return the stride of the last original dimension of a merged dimension - // TODO: refactor this once the concept of "dimension" is used - template - __host__ __device__ static constexpr auto GetLastOriginalDimensionStride(Number) - { - constexpr auto idim_last_original = std::get(mOriginalDimMergeSeqs).Back(); - - return OriginalTensorDesc::GetStride(Number{}); - } - - __host__ __device__ static constexpr auto GetLengths() - { - return Sequence{}; - } - - __host__ __device__ static constexpr auto GetElementSize() - { - return OriginalTensorDesc::GetElementSize(); - } - - template - struct lambda_1_GetOriginalMultiIndexFromMultiIndex - { - const Array& original_multi_id_partial; - Array& original_multi_id; - - __host__ __device__ constexpr lambda_1_GetOriginalMultiIndexFromMultiIndex( - const Array& original_multi_id_partial_, - Array& original_multi_id_) - : original_multi_id_partial(original_multi_id_partial_), - original_multi_id(original_multi_id_) - { - } - - template - __host__ __device__ constexpr void operator()(Number) const - { - constexpr index_t idim_original = OriginalDimsPartial::Get(Number{}); - - index_t itmp = original_multi_id_partial[I]; - - original_multi_id(idim_original) = itmp; - } - }; - - struct lambda_0_GetOriginalMultiIndexFromMultiIndex - { - const Array& multi_id; - Array& original_multi_id; - - __host__ __device__ constexpr lambda_0_GetOriginalMultiIndexFromMultiIndex( - const Array& multi_id_, Array& original_multi_id_) - : multi_id(multi_id_), original_multi_id(original_multi_id_) - { - } - - template - __host__ __device__ constexpr void operator()(Number) const - { - constexpr auto original_dims_partial = std::get(Type::mOriginalDimMergeSeqs); - - // get partial original-multi-id corresponding to this merged dimension - const auto original_multi_id_partial = - OriginalTensorDesc::Extract(original_dims_partial) - .GetMultiIndexFrom1dIndex(multi_id[IDim]); - - static_for<0, original_dims_partial.GetSize(), 1>{}( - lambda_1_GetOriginalMultiIndexFromMultiIndex( - original_multi_id_partial, original_multi_id)); - } - }; - - // return type is Array<...> - __host__ __device__ static constexpr auto - GetOriginalMultiIndexFromMultiIndex(Array multi_id) - { - Array original_multi_id; - - static_for<0, nDim, 1>{}( - lambda_0_GetOriginalMultiIndexFromMultiIndex(multi_id, original_multi_id)); - - return original_multi_id; - } - - template - __host__ __device__ static constexpr index_t GetOffsetFromMultiIndex(Sequence) - { - constexpr auto multi_id = sequence2array(Sequence{}); - - constexpr auto original_multi_id = GetOriginalMultiIndexFromMultiIndex(multi_id); - - return OriginalTensorDesc::GetOffsetFromMultiIndex(original_multi_id); - } - - __host__ __device__ static constexpr index_t - GetOffsetFromMultiIndex(Array multi_id) - { - auto original_multi_id = GetOriginalMultiIndexFromMultiIndex(multi_id); - - return OriginalTensorDesc::GetOffsetFromMultiIndex(original_multi_id); - } - - template - __host__ __device__ static constexpr index_t GetOffsetFromMultiIndex(Is... is) - { - return GetOffsetFromMultiIndex(Array{is...}); - } - - __host__ __device__ static constexpr Array GetMultiIndexFrom1dIndex(index_t id) - { - constexpr auto packed_desc = make_ConstantTensorDescriptor_packed(GetLengths()); - - return packed_desc.GetMultiIndexFrom1dIndex(id); - } - - __host__ __device__ static constexpr auto Pack() - { - constexpr auto lengths = GetLengths(); - constexpr auto strides = calculate_tensor_strides_packed(lengths); - return ConstantTensorDescriptor_deprecated{}; - } -}; - -template -__host__ __device__ constexpr auto make_ConstantMergedTensorDescriptor(OriginalTensorDesc, - OriginalDimMergeSeqs...) -{ - return ConstantMergedTensorDescriptor_deprecated{}; -} - -template -__host__ __device__ void print_ConstantMergedTensorDescriptor(const char* s, TDesc) -{ - print_ConstantTensorDescriptor(s, TDesc::GetOriginalTensorDescriptor()); -} - -} // namespace ck -#endif diff --git a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_deprecated.hpp b/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_deprecated.hpp deleted file mode 100644 index 494ef1ddd8..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_deprecated.hpp +++ /dev/null @@ -1,348 +0,0 @@ -#ifndef CK_TENSOR_COORDINATE_DEPRECATED_HPP -#define CK_TENSOR_COORDINATE_DEPRECATED_HPP - -#include "static_kernel_common_header.hpp" -#include "static_kernel_ConstantTensorDescriptor_deprecated.hpp" -#include "static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp" - -namespace ck { - -// TensorDesc is ConstantTensorDescriptor_deprecated -template -struct NormalTensorCoordinate_deprecated -{ - using type = NormalTensorCoordinate_deprecated; - using tensor_desc_type = TensorDesc; - - static constexpr index_t nDim = tensor_desc_type::GetNumOfDimension(); - - __host__ - __device__ constexpr NormalTensorCoordinate_deprecated(Array tensor_index) - : mOffset{tensor_desc_type::GetOffsetFromMultiIndex(tensor_index)} - { - } - - template - __host__ __device__ constexpr NormalTensorCoordinate_deprecated(Xs... xs) - : NormalTensorCoordinate_deprecated(Array{xs...}) - { - } - - template - __host__ __device__ constexpr NormalTensorCoordinate_deprecated(Sequence) - : NormalTensorCoordinate_deprecated(Array{Xs...}) - { - } - - __host__ __device__ constexpr index_t GetOffset() const { return mOffset; } - - // T is Array or Sequence - template - __host__ __device__ type operator+=(T step_sizes) - { - static_assert(is_same{} && T::GetSize() == nDim, "wrong!"); - - mOffset += tensor_desc_type::GetOffsetFromMultiIndex(step_sizes); - - return *this; - } - - template - __host__ __device__ type operator-=(T step_sizes) - { - static_assert(is_same{} && T::GetSize() == nDim, "wrong!"); - - mOffset -= tensor_desc_type::GetOffsetFromMultiIndex(step_sizes); - - return *this; - } - - template - __host__ __device__ constexpr type operator+(T step_sizes) const - { - type coord = *this; - coord += step_sizes; - return coord; - } - - template - __host__ __device__ constexpr type operator-(T step_sizes) const - { - type coord = *this; - coord -= step_sizes; - return coord; - } - - // reposition point of origin, and return compensated offset. - // This is a hack to reduce index calculation during looping over - // a tensor whose origin is this TensorCoordinate. It does so, by spitting - // out the run-time offset to the pointer (to the tensor data) held by this - // TensorCoordiante, so the caller can add the offset into the run-time pointer of - // the data, so only 1 run-time variable (update pointer) is needed, instead - // of 2 run-time variables (old pointer and this offset) - // TODO: after introducing the concept of "run-time tensor view", which contains the - // run-time pointer to the data, always keep track of the pointer, instead of both - // offset and the pointer. This also bring additional benefit that we don't need to - // worry the offset might underflow (because offset is unsigned integer) when updating it. - __host__ __device__ constexpr index_t RepositionOrigin() - { - index_t offset_diff = mOffset; - mOffset = 0; - return offset_diff; - } - -private: - index_t mOffset; -}; - -// TensorDesc is ConstantMergedTensorDescriptor_deprecated -template -struct MergedTensorCoordinate_deprecated -{ - using type = MergedTensorCoordinate_deprecated; - using tensor_desc_type = TensorDesc; - - static constexpr index_t nDim = tensor_desc_type::GetNumOfDimension(); - static constexpr index_t nOriginalDim = - tensor_desc_type::GetOriginalTensorDescriptor().GetNumOfDimension(); - - __host__ - __device__ constexpr MergedTensorCoordinate_deprecated(Array tensor_index) - : mOriginalIndex{tensor_desc_type::GetOriginalMultiIndexFromMultiIndex(tensor_index)} - { - // partial offset on each dimension - static_for<0, nDim, 1>{}([&](auto idim) { - constexpr auto partial_original_dims = - tensor_desc_type::GetContainedOriginalDimensions(idim); - - constexpr auto partial_original_desc = - tensor_desc_type::GetOriginalTensorDescriptor().Extract(partial_original_dims); - - mPartialOffsets(idim) = partial_original_desc.GetOffsetFromMultiIndex( - extract_array(mOriginalIndex, partial_original_dims)); - }); - - // complete offset - mOffset = - accumulate_on_array(mPartialOffsets, math::plus{}, static_cast(0)); - } - - template - __host__ __device__ constexpr MergedTensorCoordinate_deprecated(Xs... xs) - : MergedTensorCoordinate_deprecated(Array{xs...}) - { - } - - __host__ __device__ constexpr index_t GetOffset() const { return mOffset; } - - template - __host__ __device__ void - MoveOnDimension(IDim idim_, T step_size, integral_constant) - { - constexpr auto idim = idim_; - - // if step_size is known at compile time - static_if::value>{}( - [&](auto) { static_if{}([&](auto) { return; }); }); - - // update original index - static_if{}([&](auto) { - constexpr auto partial_original_dims = - tensor_desc_type::GetContainedOriginalDimensions(idim); - - constexpr index_t ndim_partial_original = partial_original_dims.GetSize(); - - constexpr auto partial_original_desc = - tensor_desc_type::GetOriginalTensorDescriptor().Extract(partial_original_dims); - - const auto partial_original_step_sizes = - partial_original_desc.GetMultiIndexFrom1dIndex(step_size); - - // update partial original multi-id - auto partial_original_id = extract_array(mOriginalIndex, partial_original_dims); - - static_if{}([&](auto) { - partial_original_id += partial_original_step_sizes; - - bool carry = false; - - // do carry check in reversed order, starting from lowest dimension - // don't check the highest dimension - static_for<0, ndim_partial_original - 1, 1>{}([&](auto IReverse) { - constexpr index_t i = ndim_partial_original - 1 - IReverse; - - if(carry) - { - ++partial_original_id(i); - } - - carry = false; - - if(partial_original_id[i] >= partial_original_desc.GetLength(i)) - { - partial_original_id(i) -= partial_original_desc.GetLength(i); - carry = true; - } - }); - - // highest dimension - if(carry) - { - ++partial_original_id(0); - } - }).Else([&](auto) { - // shift up multi-id to avoid unsigned integer underflow during intermediate - // calculations. After the shift, should have new_multi_id[...] >= 1 - partial_original_id += - partial_original_desc.GetLengths() - partial_original_step_sizes; - - bool borrow = false; - - // do borrow check in reversed order, starting from lowest dimension - // don't check the highest dimension - static_for<0, ndim_partial_original - 1, 1>{}([&](auto IReverse) { - constexpr index_t i = ndim_partial_original - 1 - IReverse; - - if(borrow) - { - --partial_original_id(i); - } - - borrow = false; - - if(partial_original_id[i] < partial_original_desc.GetLength(i)) - { - partial_original_id(i) += partial_original_desc.GetLength(i); - borrow = true; - } - }); - - // highest dimension - if(borrow) - { - --partial_original_id(0); - } - - // shift back down multi-id - // here, should have new_multi_id[...] >= GetLengths() - partial_original_id = partial_original_id - partial_original_desc.GetLengths(); - }); - - // update "mOriginalIndex" - static_for<0, ndim_partial_original, 1>{}([&](auto I) { - constexpr auto idim_original = partial_original_dims[I]; - - mOriginalIndex(idim_original) = partial_original_id[I]; - }); - - // calculate new partial offset on this merged dimension - const index_t old_partial_offset = mPartialOffsets[idim]; - - mPartialOffsets(idim) = - partial_original_desc.GetOffsetFromMultiIndex(partial_original_id); - - // update "mThreadSrcOffset", do "+" before "-" to avoid underflow - mOffset = (mOffset + mPartialOffsets[idim]) - old_partial_offset; - }).Else([&](auto fwd) { - static_if{}([&](auto) { - mOffset += step_size * fwd(tensor_desc_type{}).GetStride(idim); - }).Else([&](auto) { mOffset -= step_size * fwd(tensor_desc_type{}).GetStride(idim); }); - }); - } - - // T is Array or Sequence - template - __host__ __device__ type operator+=(T step_sizes) - { - static_assert(is_same{} && T::GetSize() == nDim, "wrong!"); - - static_for<0, nDim, 1>{}([&](auto idim) { - // compiler should remove dead code path, because step_sizes is known at - // compile time - if(step_sizes[idim] != 0) - { - this->MoveOnDimension(idim, step_sizes[idim], integral_constant{}); - } - }); - - return *this; - } - - template - __host__ __device__ type operator-=(T step_sizes) - { - static_assert(is_same{} && T::GetSize() == nDim, "wrong!"); - - static_for<0, nDim, 1>{}([&](auto idim) { - // compiler should remove dead code path, because step_sizes is known at - // compile time - if(step_sizes[idim] != 0) - { - this->MoveOnDimension(idim, step_sizes[idim], integral_constant{}); - } - }); - - return *this; - } - - template - __host__ __device__ constexpr type operator+(T step_sizes) const - { - type coord = *this; - coord += step_sizes; - return coord; - } - - template - __host__ __device__ constexpr type operator-(T step_sizes) const - { - type coord = *this; - coord -= step_sizes; - return coord; - } - - __host__ __device__ static constexpr index_t RepositionOrigin() { return 0; } - -private: - // Allocate register memory for all merged dimensions and normal dimensions. - // However, only those merged dimensions, whose index will be involved in arithmetic - // after the construction of this TensorCoordinate (e.g. when user move a slicing - // window on the merged dimension), will use these register memory. - // Let's hope compiler will optimize away those register memory allocated for normal - // dimensions, and those merged dimensions, that would never be involved in index - // arithmetic after construction of TensorCoordinate. - // TODO: refactor TensorCoordinate, after introducing the concept of "dimensions" - // and simplify implementation of ConstantMergedTensorDescriptor_deprecated, so we don't need to - // count on compiler to optimize away those register memory for us - Array mOriginalIndex; - Array mPartialOffsets; - - // complete offset - index_t mOffset; -}; - -template -struct TensorCoordinate_deprecated -{ -private: - template - __host__ __device__ static constexpr auto - MakeDummyTensorCoordinate(ConstantTensorDescriptor_deprecated) - { - return NormalTensorCoordinate_deprecated>(); - } - - template - __host__ __device__ static constexpr auto - MakeDummyTensorCoordinate(ConstantMergedTensorDescriptor_deprecated) - { - return MergedTensorCoordinate_deprecated< - ConstantMergedTensorDescriptor_deprecated>(); - } - -public: - using type = decltype(MakeDummyTensorCoordinate(TensorDesc{})); -}; - -} // namespace ck -#endif diff --git a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_helper.hpp b/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_helper.hpp deleted file mode 100644 index 2cacb329cb..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_description/static_kernel_tensor_coordinate_helper.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef CK_TENSOR_COORDINATE_HELPER_HPP -#define CK_TENSOR_COORDINATE_HELPER_HPP - -#include "tensor_coordiante_hpp" - -namespace ck { - -template -__host__ __device__ constexpr auto -make_tensor_coordinate(TensorDesc, MultiIndex idx) -{ - return typename TensorCoordinate::type(idx); -} - -} // namespace ck -#endif diff --git a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_blockwise_generic_tensor_slice_copy_deprecated.hpp b/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_blockwise_generic_tensor_slice_copy_deprecated.hpp deleted file mode 100644 index 806a38a0c7..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_blockwise_generic_tensor_slice_copy_deprecated.hpp +++ /dev/null @@ -1,613 +0,0 @@ -#ifndef CK_BLOCKWISE_GENERIC_TENSOR_SLICE_COPY_DEPRECATED_HPP -#define CK_BLOCKWISE_GENERIC_TENSOR_SLICE_COPY_DEPRECATED_HPP - -#include "static_kernel_common_header.hpp" -#include "static_kernel_ConstantTensorDescriptor_deprecated.hpp" -#include "static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp" -#include "static_kernel_tensor_coordinate_deprecated.hpp" -#include "static_kernel_threadwise_generic_tensor_slice_copy_deprecated.hpp" - -namespace ck { - -// Slice a (normal or merged) tensor, and copy it into another (normal or merged) tensor -// memory layout (ordering of dimensions) can be different between src and dst. -// This functions assume each thread is reading and writing a normal (not merged) tensor, -// to simplify index calculations. To satisfy this assumption, the user need to make sure -// that, on a merged dimension that constains multiple original dimensions, the length of -// the last original dimension need to be evenly dividable by its sub-lengths. Also, the -// repeat-length on the merged dimension need to be 1. These sanity checks are performed -// in constructor of BlockwiseGenericTensorSliceCopy_v1_deprecated -template -struct BlockwiseGenericTensorSliceCopy_v1_deprecated -{ - static constexpr index_t nDim = SrcDesc::GetNumOfDimension(); - - static constexpr index_t nOriginalDimSrc = - SrcDesc::GetOriginalTensorDescriptor().GetNumOfDimension(); - static constexpr index_t nOriginalDimDst = - DstDesc::GetOriginalTensorDescriptor().GetNumOfDimension(); - - // per-thread offset - index_t mThreadSrcOffset; - index_t mThreadDstOffset; - - // "mThreadSrcOriginalMultiId", "mThreadSrcPartialOffsets, "mThreadDstOriginalMultiId", - // "mThreadDstPartialOffsets" are always calculated inside constructor, and would be - // updated if slicing-window is moved. However, they will not be used if you always move - // the slicing-window along a non-merged dimension. In that case, compiler should be - // able to remove these calculation. - // TODO: make sure compiler would actually remove them in that case - - // partial offset in each (merged) dimension - Array mThreadSrcPartialOffsets; - Array mThreadDstPartialOffsets; - - // multi-id of original tensor - Array mThreadSrcOriginalMultiId; - Array mThreadDstOriginalMultiId; - - __device__ - BlockwiseGenericTensorSliceCopy_v1_deprecated(Array src_block_data_id_begin, - Array dst_block_data_id_begin) - { - // check NDim consistency - static_assert( - nDim == SrcDesc::GetNumOfDimension() && nDim == DstDesc::GetNumOfDimension() && - nDim == SliceLengths::GetSize() && nDim == SubLengths::GetSize() && - nDim == ThreadClusterLengths::GetSize() && - nDim == ThreadClusterArrangeOrder::GetSize() && - nDim == SrcDimAccessOrder::GetSize() && nDim == DstDimAccessOrder::GetSize(), - "wrong"); - - // check thread arrange order and read/write access order are valid - static_assert(is_valid_sequence_map::value && - is_valid_sequence_map::value && - is_valid_sequence_map::value, - "wrong!"); - - // thread cluster - constexpr auto thread_cluster_desc = make_ConstantTensorDescriptor_packed( - ThreadClusterLengths::ReorderGivenNew2Old(ThreadClusterArrangeOrder{})); - - // BlockSize - static_assert(BlockSize == thread_cluster_desc.GetElementSize(), "wrong! BlockSize"); - - // divide work - constexpr auto data_per_cluster_per_dims = SubLengths{} * ThreadClusterLengths{}; - - static_for<0, nDim, 1>{}([&](auto IDim) { - static_assert(SliceLengths::Get(IDim) % data_per_cluster_per_dims.Get(IDim) == 0, - "wrong! cannot evenly divide sliced tensor into cluster"); - }); - - constexpr auto repeat_lengths = SliceLengths{} / data_per_cluster_per_dims; - - // additional check for merged dimension - static_for<0, nDim, 1>{}([&](auto IDim_) { - // src - static_if{}([&](auto) { - constexpr auto IDim = decltype(IDim_){}; - - // on a merged dimension that constains multiple original dimensions, - // the length of the last original dimension need to evenly dividable by its - // sub-length, - // so each thread is effectively reading a normal (not merged) tensor - constexpr auto idim_last_original_src = - SrcDesc::GetContainedOriginalDimensions(IDim).Back(); - static_assert( - SrcDesc::GetOriginalTensorDescriptor().GetLength(idim_last_original_src) % - SubLengths::Get(IDim) == - 0, - "wrong!"); - - // merged dimension should have repeat_lengths = 1 - static_assert(repeat_lengths[IDim] == 1, - "wrong! repeat_lengths shoud be 1 on merged dimension"); - }); - - // dst - static_if{}([&](auto) { - constexpr auto IDim = decltype(IDim_){}; - - // on a merged dimension that constains multiple original dimensions, - // the length of the last original dimension need to evenly dividable by its - // sub-length, - // so each thread is effectively reading a normal (not merged) tensor - constexpr auto idim_last_original_dst = - DstDesc::GetContainedOriginalDimensions(IDim).Back(); - static_assert( - DstDesc::GetOriginalTensorDescriptor().GetLength(idim_last_original_dst) % - SubLengths::Get(IDim) == - 0, - "wrong!"); - - // merged dimension should have repeat_lengths = 1 - static_assert(repeat_lengths[IDim] == 1, - "wrong! repeat_lengths shoud be 1 on merged dimension"); - }); - }); - - // calculate mThreadSrcOffset, mThreadDstOffset - const auto thread_cluster_id = - thread_cluster_desc.GetMultiIndexFrom1dIndex(get_thread_local_1d_id()); - - const auto data_cluster_id = - reorder_array_given_old2new(thread_cluster_id, ThreadClusterArrangeOrder{}); - - const auto thread_data_id_begin = data_cluster_id * SubLengths{}; - - // original multi-id - mThreadSrcOriginalMultiId = SrcDesc::GetOriginalMultiIndexFromMultiIndex( - src_block_data_id_begin + thread_data_id_begin); - - mThreadDstOriginalMultiId = DstDesc::GetOriginalMultiIndexFromMultiIndex( - dst_block_data_id_begin + thread_data_id_begin); - - // partial offset on each dimension - static_for<0, nDim, 1>{}([&](auto IDim) { - constexpr auto src_partial_original_dims = - SrcDesc::GetContainedOriginalDimensions(IDim); - - constexpr auto src_partial_original_desc = - SrcDesc::GetOriginalTensorDescriptor().Extract(src_partial_original_dims); - - mThreadSrcPartialOffsets(IDim) = src_partial_original_desc.GetOffsetFromMultiIndex( - extract_array(mThreadSrcOriginalMultiId, src_partial_original_dims)); - }); - - static_for<0, nDim, 1>{}([&](auto IDim) { - constexpr auto dst_partial_original_dims = - DstDesc::GetContainedOriginalDimensions(IDim); - - constexpr auto dst_partial_original_desc = - DstDesc::GetOriginalTensorDescriptor().Extract(dst_partial_original_dims); - - mThreadDstPartialOffsets(IDim) = dst_partial_original_desc.GetOffsetFromMultiIndex( - extract_array(mThreadDstOriginalMultiId, dst_partial_original_dims)); - }); - - // complete offset - mThreadSrcOffset = accumulate_on_array( - mThreadSrcPartialOffsets, math::plus{}, static_cast(0)); - - mThreadDstOffset = accumulate_on_array( - mThreadDstPartialOffsets, math::plus{}, static_cast(0)); - } - - __device__ static constexpr auto GetRegisterBufferDescriptor() - { - constexpr auto repeat_lengths = SliceLengths{} / (SubLengths{} * ThreadClusterLengths{}); - - return make_ConstantTensorDescriptor_packed(SubLengths{} * repeat_lengths); - } - - __device__ static constexpr index_t GetThreadBufferSize() - { - return GetRegisterBufferDescriptor().GetElementSpace(); - } - - template - __device__ void RunLoadThreadBuffer(const TData* __restrict__ p_src, - TData* __restrict__ p_buffer) const - { - constexpr auto thread_sub_tensor_lengths = SubLengths{}; - - constexpr auto data_per_cluster_per_dims = - thread_sub_tensor_lengths * ThreadClusterLengths{}; - - constexpr auto repeat_lengths = SliceLengths{} / (SubLengths{} * ThreadClusterLengths{}); - - constexpr auto thread_buffer_desc = GetRegisterBufferDescriptor(); - -#if CK_EXPERIMENTAL_USE_MORE_COMPILE_STATIC_BLOCKWISE_GENERIC_SLICE_COPY_V1 - static_ford{}([&](auto repeat_id) { - constexpr auto src_thread_data_id_begin = repeat_id * data_per_cluster_per_dims; - - constexpr auto buffer_data_id_begin = repeat_id * thread_sub_tensor_lengths; - - constexpr index_t src_offset = - SrcDesc::GetOffsetFromMultiIndex(src_thread_data_id_begin); - - constexpr index_t buffer_offset = - thread_buffer_desc.GetOffsetFromMultiIndex(buffer_data_id_begin); -#else - ford{}([&](auto repeat_id) { - const auto src_thread_data_id_begin = repeat_id * data_per_cluster_per_dims; - - const auto buffer_data_id_begin = repeat_id * thread_sub_tensor_lengths; - - const index_t src_offset = SrcDesc::GetOffsetFromMultiIndex(src_thread_data_id_begin); - - const index_t buffer_offset = - thread_buffer_desc.GetOffsetFromMultiIndex(buffer_data_id_begin); -#endif - - // By position the origin of the per-thread window at the point, where multi-index - // of the SrcDesc (might be a merged tensor) is all-zero. This threadwise slice copy - // is assuming each thread is copy a noraml (not merged) tensor. - // To satisfy this assumption, the user need to make sure that, on a merged dimension - // that constains multiple original dimensions, the length of the last original - // dimension need to be evenly dividable by its sub-lengths. Also, the repeat-length on - // the merged dimension need to be 1. These sanity checks are performed in constructor - // of BlockwiseGenericTensorSliceCopy_v1_deprecated - ThreadwiseGenericTensorSliceCopy_v1r2_deprecated(make_zero_array(), - make_zero_array()) - .Run(p_src + src_offset + mThreadSrcOffset, p_buffer + buffer_offset); - }); - } - - template - __device__ void RunStoreThreadBuffer(const TData* __restrict__ p_buffer, - TData* __restrict__ p_dst) const - { - constexpr auto thread_sub_tensor_lengths = SubLengths{}; - - constexpr auto data_per_cluster_per_dims = - thread_sub_tensor_lengths * ThreadClusterLengths{}; - - constexpr auto repeat_lengths = SliceLengths{} / (SubLengths{} * ThreadClusterLengths{}); - - constexpr auto thread_buffer_desc = GetRegisterBufferDescriptor(); - -#if CK_EXPERIMENTAL_USE_MORE_COMPILE_STATIC_BLOCKWISE_GENERIC_SLICE_COPY_V1 - static_ford{}([&](auto repeat_id) { - constexpr auto buffer_data_id_begin = repeat_id * thread_sub_tensor_lengths; - - constexpr auto dst_data_id_begin = repeat_id * data_per_cluster_per_dims; - - constexpr index_t buffer_offset = - thread_buffer_desc.GetOffsetFromMultiIndex(buffer_data_id_begin); - - constexpr index_t dst_offset = DstDesc::GetOffsetFromMultiIndex(dst_data_id_begin); -#else - ford{}([&](auto repeat_id) { - const auto buffer_data_id_begin = repeat_id * thread_sub_tensor_lengths; - - const auto dst_data_id_begin = repeat_id * data_per_cluster_per_dims; - - const index_t buffer_offset = - thread_buffer_desc.GetOffsetFromMultiIndex(buffer_data_id_begin); - - const index_t dst_offset = DstDesc::GetOffsetFromMultiIndex(dst_data_id_begin); -#endif - - // By position the origin of the per-thread window at the point, where multi-index - // of the SrcDesc (might be a merged tensor) is all-zero. This threadwise slice copy - // is assuming each thread is copy a noraml (not merged) tensor. - // To satisfy this assumption, the user need to make sure that, on a merged dimension - // that constains multiple original dimensions, the length of the last original - // dimension need to be evenly dividable by its sub-lengths. Also, the repeat-length on - // the merged dimension need to be 1. These sanity checks are performed in constructor - // of BlockwiseGenericTensorSliceCopy_v1_deprecated - ThreadwiseGenericTensorSliceCopy_v1r2_deprecated( - make_zero_array(), make_zero_array()) - .Run(p_buffer + buffer_offset, p_dst + dst_offset + mThreadDstOffset); - }); - } - - template - __device__ void Run(const TData* __restrict__ p_src, TData* __restrict__ p_dst) const - { - TData p_buffer[GetThreadBufferSize()]; - - RunLoadThreadBuffer(p_src, p_buffer); - RunStoreThreadBuffer(p_buffer, p_dst); - } - - // When moving the slicing windows along a merged dimension, if the strides of the - // contained (by the merged dimension) original dimensions are not in descending order, - // then there is no guarantee that the new offset will be larger than the old offset - // for movement in positive direction (vice versue for movement in negative direction). - // As a result, there is the possiblity that the offset calculation may result in - // unsigned integer underflow (due to "-" operation). However, this hazard should not - // happen, as long as the users make sure the slicing window would not be moved out of - // the boundary of the tensor being sliced. This functions doesn't do runtime sanity - // check on out-of-bound slicing window, for performance reason - template - __device__ void MoveSlicingWindowOnSourceTensor( - Number, Number, integral_constant direction) - { - constexpr auto IDim = Number{}; - - static_if{}([&](auto) { - // logic for a merged dimension, also works for non-merged dimension, but its logic may - // be unncessarily complicated for compiler to remove calculations that are useless for - // a non-merged dimension - - // extract partial original dimensions - constexpr auto src_partial_original_dims = - SrcDesc::GetContainedOriginalDimensions(IDim); - - constexpr auto src_partial_original_desc = - SrcDesc::GetOriginalTensorDescriptor().Extract(src_partial_original_dims); - - // calculate new partial original multi-id - auto old_src_partial_original_id = - extract_array(mThreadSrcOriginalMultiId, src_partial_original_dims); - - auto new_src_partial_original_id = - src_partial_original_desc.UpdateMultiIndexGivenStepSizeOf1dIndex( - old_src_partial_original_id, StepSize, direction); - - // update "mThreadSrcOriginalMultiId" - static_for<0, decltype(src_partial_original_dims)::GetSize(), 1>{}([&](auto I) { - constexpr auto IDimOriginal = src_partial_original_dims[I]; - - mThreadSrcOriginalMultiId(IDimOriginal) = new_src_partial_original_id[I]; - }); - - // calculate new partial offset on this merged dimension - const index_t old_src_partial_offset = mThreadSrcPartialOffsets[IDim]; - - const index_t new_src_partial_offset = - src_partial_original_desc.GetOffsetFromMultiIndex(new_src_partial_original_id); - - // update "mThreadSrcPartialOffsets" - mThreadSrcPartialOffsets(IDim) = new_src_partial_offset; - - // update "mThreadSrcOffset", do "+" before "-" to avoid underflow - mThreadSrcOffset = (mThreadSrcOffset + new_src_partial_offset) - old_src_partial_offset; - }).Else([&](auto) { - // Logic for non-merged dimension. If you are never going to move the slicing window on - // a merged dimension, then "mThreadSrcOriginalMultiId" and "mThreadSrcPartialOffsets", - // which are being calculated here, will never be used later. In this case, compiler - // should be able to remove these calculations. - // TODO: make sure compiler would actually remove them in this case. - - // It is the user's responsiblity to make sure the slicing window will not be moved out - // of the boundary of the tensor being sliced. Otherwise, there might be hazard like - // unsigned integer underflow. That is NO runtime sanity check to prevent the hazard - - constexpr auto IDimOriginal = SrcDesc::GetContainedOriginalDimensions(IDim).Front(); - - static_if{}([&](auto fwd) { - mThreadSrcOffset += StepSize * fwd(SrcDesc{}).GetStride(IDim); - - mThreadSrcOriginalMultiId(IDimOriginal) += StepSize; - - mThreadSrcPartialOffsets(IDim) += StepSize * fwd(SrcDesc{}).GetStride(IDim); - }).Else([&](auto fwd) { - mThreadSrcOffset -= StepSize * fwd(SrcDesc{}).GetStride(IDim); - - mThreadSrcOriginalMultiId(IDimOriginal) -= StepSize; - - mThreadSrcPartialOffsets(IDim) -= StepSize * fwd(SrcDesc{}).GetStride(IDim); - }); - }); - } - - template - __device__ void - MoveSrcSliceWindow(T step_sizes, integral_constant positive_direction) - { - static_for<0, nDim, 1>{}([&](auto idim) { - if(step_sizes[idim] != 0) - { - MoveSlicingWindowOnSourceTensor(idim, step_sizes[idim], positive_direction); - } - }); - } -}; - -// This version use TensorCoordiante -// Slice a (normal or merged) tensor, and copy it into another (normal or merged) tensor -// memory layout (ordering of dimensions) can be different between src and dst. -template -struct BlockwiseGenericTensorSliceCopy_v2_deprecated -{ - static constexpr index_t nDim = SrcDesc::GetNumOfDimension(); - - using Index = MultiIndex; - - __device__ constexpr BlockwiseGenericTensorSliceCopy_v2_deprecated( - const Index& src_block_slice_origin, const Index& dst_block_slice_origin) - { - static_assert( - nDim == SrcDesc::GetNumOfDimension() && nDim == DstDesc::GetNumOfDimension() && - nDim == SliceLengths::GetSize() && nDim == SubLengths::GetSize() && - nDim == ThreadClusterLengths::GetSize() && - nDim == ThreadClusterArrangeOrder::GetSize() && - nDim == SrcDimAccessOrder::GetSize() && nDim == DstDimAccessOrder::GetSize(), - "wrong! nDim not consistent"); - - static_assert(is_same{}, - "wrong! threads should be mapped to cover entire slicing window"); - - constexpr auto thread_cluster_desc = make_ConstantTensorDescriptor_packed( - ThreadClusterLengths::ReorderGivenNew2Old(ThreadClusterArrangeOrder{})); - - static_assert(BlockSize == thread_cluster_desc.GetElementSize(), - "wrong! BlockSize not consistent with ThreadClusterLengths"); - - const auto thread_cluster_id = - thread_cluster_desc.GetMultiIndexFrom1dIndex(get_thread_local_1d_id()); - - const auto data_cluster_id = - reorder_array_given_old2new(thread_cluster_id, ThreadClusterArrangeOrder{}); - - const auto thread_data_id_begin = data_cluster_id * SubLengths{}; - - mThreadwiseLoad.SetSrcSliceOrigin(src_block_slice_origin + thread_data_id_begin); - mThreadwiseLoad.SetDstSliceOrigin(make_zero_array()); - - mThreadwiseStore.SetSrcSliceOrigin(make_zero_array()); - mThreadwiseStore.SetDstSliceOrigin(dst_block_slice_origin + thread_data_id_begin); - } - - __device__ static constexpr index_t GetThreadBufferSize() - { - return ThreadBufferDesc::GetElementSpace(); - } - - template - __device__ void - RunLoadThreadBuffer(const BlockSrcData* p_block_src, - ThreadBufferData* p_thread_buffer, - integral_constant, - integral_constant) const - { - constexpr auto block_src_address_space = - integral_constant{}; - constexpr auto thread_buffer_address_space = - integral_constant{}; - - mThreadwiseLoad.Run( - p_block_src, p_thread_buffer, block_src_address_space, thread_buffer_address_space); - } - - template - __device__ void RunLoadThreadBuffer(const BlockSrcData* p_block_src, - ThreadBufferData* p_thread_buffer) const - { - constexpr auto generic_address_space = - integral_constant{}; - - RunLoadThreadBuffer( - p_block_src, p_thread_buffer, generic_address_space, generic_address_space); - } - - template - __device__ void - RunStoreThreadBuffer(const ThreadBufferData* p_thread_buffer, - BlockDstData* p_block_dst, - integral_constant, - integral_constant) const - { - constexpr auto thread_buffer_address_space = - integral_constant{}; - constexpr auto block_dst_address_space = - integral_constant{}; - - mThreadwiseStore.Run( - p_thread_buffer, p_block_dst, thread_buffer_address_space, block_dst_address_space); - } - - template - __device__ void RunStoreThreadBuffer(const ThreadBufferData* p_thread_buffer, - BlockDstData* p_block_dst) const - { - constexpr auto generic_address_space = - integral_constant{}; - - RunStoreThreadBuffer( - p_thread_buffer, p_block_dst, generic_address_space, generic_address_space); - } - - template - __device__ void - Run(const BlockSrcData* p_block_src, - BlockDstData* p_block_dst, - integral_constant block_src_address_space, - integral_constant block_dst_address_space) const - { - BlockSrcData p_thread_buffer[GetThreadBufferSize()]; - - constexpr auto generic_address_space = - integral_constant{}; - - RunLoadThreadBuffer( - p_block_src, p_thread_buffer, block_src_address_space, generic_address_space); - - // if there is type conversion, it's done during store - RunStoreThreadBuffer( - p_thread_buffer, p_block_dst, generic_address_space, block_dst_address_space); - } - - template - __device__ void Run(const BlockSrcData* p_block_src, BlockDstData* p_block_dst) const - { - constexpr auto generic_address_space = - integral_constant{}; - - Run(p_block_src, p_block_dst, generic_address_space, generic_address_space); - } - - template - __device__ void - MoveSrcSliceWindow(T step_sizes, integral_constant positive_direction) - { - mThreadwiseLoad.MoveSrcSliceWindow(step_sizes, positive_direction); - } - - template - __device__ void - MoveDstSliceWindow(T step_sizes, integral_constant positive_direction) - { - mThreadwiseStore.MoveDstSliceWindow(step_sizes, positive_direction); - } - -private: - using ThreadBufferDesc = decltype(make_ConstantTensorDescriptor_packed(SubLengths{})); - - using ThreadwiseLoad = ThreadwiseGenericTensorSliceCopy_v2r1_deprecated; - - using ThreadwiseStore = ThreadwiseGenericTensorSliceCopy_v2r1_deprecated; - - ThreadwiseLoad mThreadwiseLoad; - ThreadwiseStore mThreadwiseStore; -}; - -} // namespace ck - -#endif diff --git a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_gridwise_gemm_xdlops.hpp b/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_gridwise_gemm_xdlops.hpp deleted file mode 100644 index 44652321f7..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_gridwise_gemm_xdlops.hpp +++ /dev/null @@ -1,650 +0,0 @@ -#ifndef CK_GRIDWISE_GEMM_XDLOPS_HPP -#define CK_GRIDWISE_GEMM_XDLOPS_HPP - -#include "static_kernel_common_header.hpp" -#include "static_kernel_tensor_descriptor.hpp" -#include "static_kernel_tensor_descriptor_helper.hpp" -#include "static_kernel_ConstantMatrixDescriptor.hpp" -#include "static_kernel_blockwise_generic_tensor_slice_copy.hpp" -#include "static_kernel_threadwise_generic_tensor_slice_copy.hpp" -#include "static_kernel_blockwise_gemm_xdlops.hpp" - -namespace ck { - -template -struct GridwiseGemmTransposedANormalBNormalCXdlops_v1 -{ - __device__ void Run(const Float* const __restrict__ p_a_global, - const Float* const __restrict__ p_b_global, - Float* const __restrict__ p_c_global) const - { - - constexpr auto True = integral_constant{}; - - constexpr auto a_k_m_global_desc = AGlobalDesc{}; - constexpr auto b_k_n_global_desc = BGlobalDesc{}; - constexpr auto c_m_n_global_desc = CGlobalDesc{}; - - constexpr auto K = b_k_n_global_desc.GetLengths()[0]; - constexpr auto N = b_k_n_global_desc.GetLengths()[1]; - constexpr auto M = a_k_m_global_desc.GetLengths()[1]; - - // divide block work by [M, N] - static_assert(M % MPerBlock == 0 && N % NPerBlock == 0 && K % KPerBlock == 0, - "wrong! cannot divide work evenly among block"); - - constexpr index_t MBlockWork = M / MPerBlock; - constexpr index_t NBlockWork = N / NPerBlock; - - static_assert(MPerBlock % MPerWave == 0 && NPerBlock % NPerWave == 0, - "wrong! M/NPerBlock % M/NPerWave != 0"); - - constexpr index_t MWaves = MPerBlock / MPerWave; - constexpr index_t NWaves = NPerBlock / NPerWave; - - constexpr auto block_work_desc = - make_cluster_descriptor(Sequence{}); - - const auto block_work_id = block_work_desc.CalculateClusterIndex(get_block_1d_id()); - - const index_t m_block_data_on_global = block_work_id[0] * MPerBlock; - const index_t n_block_data_on_global = block_work_id[1] * NPerBlock; - - // LDS mem - constexpr index_t max_align = math::lcm(BBlockCopyDstDataPerWrite_N, - ABlockCopyDstDataPerWrite_M, - GemmDataPerReadM, - GemmDataPerReadN); - - // LDS - // be careful of LDS alignment - constexpr auto a_k_m_block_desc = make_native_tensor_descriptor_aligned( - Sequence{}, Number{}); - - auto a_blockwise_copy = - BlockwiseGenericTensorSliceCopy_v4({0, m_block_data_on_global}, - {0, 0}); - - constexpr auto b_k_n_block_desc = make_native_tensor_descriptor_aligned( - Sequence{}, Number{}); - - auto b_blockwise_copy = - BlockwiseGenericTensorSliceCopy_v4({0, n_block_data_on_global}, - {0, 0}); - - // GEMM definition - // c_mtx += transpose(a_mtx) * b_mtx - // a_mtx[KPerBlock, MPerBlock] is in LDS - // b_mtx[EPerBlocl, NPerBlock] is in LDS - // c_mtx[MPerBlock, NPerBlock] is distributed among threads, and saved in - // register - constexpr auto a_k_m_block_mtx_desc = make_ConstantMatrixDescriptor(a_k_m_block_desc); - constexpr auto b_k_n_block_mtx_desc = make_ConstantMatrixDescriptor(b_k_n_block_desc); - - const auto blockwise_gemm = BlockwiseGemmBlockABlockBThreadCTransANormalBNormalC_xdlops< - BlockSize, - decltype(a_k_m_block_mtx_desc), - decltype(b_k_n_block_mtx_desc), - Float, - MPerWave, - NPerWave, - MWaves, - NWaves, - GemmDataPerReadM, - GemmDataPerReadN>{}; - - constexpr index_t a_block_space = - math::integer_least_multiple(a_k_m_block_desc.GetElementSpace(), max_align); - - constexpr index_t b_block_space = - math::integer_least_multiple(b_k_n_block_desc.GetElementSpace(), max_align); - - __shared__ Float p_a_block_double[2 * a_block_space]; - __shared__ Float p_b_block_double[2 * b_block_space]; - - // get zero-initialized output register of vector type - auto c_thread_vec = blockwise_gemm.CreateOutputVecZero(); - - // LDS double buffer: preload data into LDS - { - a_blockwise_copy.Run(p_a_global, p_a_block_double); - b_blockwise_copy.Run(p_b_global, p_b_block_double); - } - - using b_blockwise_copy_src_step = Sequence; - using a_blockwise_copy_src_step = Sequence; - - // LDS double buffer: main body - for(index_t k_block_data_begin = 0; k_block_data_begin + 2 * KPerBlock < K; - k_block_data_begin += 2 * KPerBlock) - { -#pragma unroll - for(index_t iloop = 0; iloop < 2; ++iloop) - { - const bool even_loop = (iloop % 2 == 0); - - Float* p_a_block_now = - even_loop ? p_a_block_double : p_a_block_double + a_block_space; - Float* p_b_block_now = - even_loop ? p_b_block_double : p_b_block_double + b_block_space; - - Float* p_a_block_next = - even_loop ? p_a_block_double + a_block_space : p_a_block_double; - Float* p_b_block_next = - even_loop ? p_b_block_double + b_block_space : p_b_block_double; - - Float p_a_thread_buffer[a_blockwise_copy.GetThreadBufferSize()]; - Float p_b_thread_buffer[b_blockwise_copy.GetThreadBufferSize()]; - - a_blockwise_copy.MoveSrcSliceWindow(a_blockwise_copy_src_step{}, True); - b_blockwise_copy.MoveSrcSliceWindow(b_blockwise_copy_src_step{}, True); - - __syncthreads(); - - // LDS doubel buffer: load next data from device mem - a_blockwise_copy.RunLoadThreadBuffer(p_a_global, p_a_thread_buffer); - b_blockwise_copy.RunLoadThreadBuffer(p_b_global, p_b_thread_buffer); - - // LDS double buffer: GEMM on current data - c_thread_vec = blockwise_gemm.Run(p_a_block_now, p_b_block_now, c_thread_vec); - - // LDS double buffer: store next data to LDS - a_blockwise_copy.RunStoreThreadBuffer(p_a_thread_buffer, p_a_block_next); - b_blockwise_copy.RunStoreThreadBuffer(p_b_thread_buffer, p_b_block_next); - } - } - - // LDS double buffer: tail - { - constexpr bool has_two_iteration_left = (K % (2 * KPerBlock) == 0); - - if(has_two_iteration_left) // if has 2 iteration left - { - Float p_a_thread_buffer[a_blockwise_copy.GetThreadBufferSize()]; - Float p_b_thread_buffer[b_blockwise_copy.GetThreadBufferSize()]; - - a_blockwise_copy.MoveSrcSliceWindow(a_blockwise_copy_src_step{}, True); - b_blockwise_copy.MoveSrcSliceWindow(b_blockwise_copy_src_step{}, True); - - __syncthreads(); - - // LDS double buffer: load last data from device mem - a_blockwise_copy.RunLoadThreadBuffer(p_a_global, p_a_thread_buffer); - b_blockwise_copy.RunLoadThreadBuffer(p_b_global, p_b_thread_buffer); - - // LDS double buffer: GEMM on 2nd-last data - c_thread_vec = blockwise_gemm.Run(p_a_block_double, p_b_block_double, c_thread_vec); - - // LDS double buffer: store last data to LDS - a_blockwise_copy.RunStoreThreadBuffer(p_a_thread_buffer, - p_a_block_double + a_block_space); - b_blockwise_copy.RunStoreThreadBuffer(p_b_thread_buffer, - p_b_block_double + b_block_space); - - __syncthreads(); - - // LDS double buffer: GEMM on current data - c_thread_vec = blockwise_gemm.Run(p_a_block_double + a_block_space, - p_b_block_double + b_block_space, - c_thread_vec); - } - else // if has 1 iteration left - { - __syncthreads(); - - // LDS double buffer: GEMM on last data - c_thread_vec = blockwise_gemm.Run(p_a_block_double, p_b_block_double, c_thread_vec); - } - } - - // copy output: register to global memory - { - ///\todo inconsistent layout of xdlops and tensor - // xdlops layout - // M1 = num_groups; - // M0 = group_size; - // N1 = num_blks_per_wave; - // N0 = num_threads_per_blks; - constexpr auto CLayout = blockwise_gemm.GetOutputLayout(); - constexpr index_t M0 = CLayout.M1(); - constexpr index_t M1 = CLayout.N1(); - constexpr index_t M2 = CLayout.M0(); - - constexpr auto c_m0_m1_m2_n_global_desc = transform_tensor_descriptor( - c_m_n_global_desc, - make_tuple(UnMerge>{}, PassThrough{}), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0, 1, 2>{}, Sequence<3>{})); - - // src descriptor - constexpr auto c_m0_m1_m2_n_thread_desc = - make_native_tensor_descriptor_packed(Sequence{}); - - using CThreadCopySliceLengths = Sequence; - - constexpr index_t BlkSize = blockwise_gemm.GetBlkSize(); - constexpr index_t NumBlks = blockwise_gemm.GetNumBlks(); - - for(index_t i = 0; i < NumBlks; ++i) - { - // calculate origin of thread output tensor on global memory - // blockwise GEMM c matrix starting index - const auto c_thread_mtx_on_block = blockwise_gemm.GetBeginOfThreadMatrixC(i); - - const index_t m_thread_data_on_global = - m_block_data_on_global + c_thread_mtx_on_block.row; - - const index_t n_thread_data_on_global = - n_block_data_on_global + c_thread_mtx_on_block.col; - - ThreadwiseGenericTensorSliceCopy_v4r2::type, - 3, - 1, - 1, - AddressSpace::Vgpr, - AddressSpace::Global, - CGlobalMemoryDataOperation>( - {0, 0, 0, 0}, - {m_thread_data_on_global / (M2 * M1), - m_thread_data_on_global % (M2 * M1) / M2, - m_thread_data_on_global % M2, - n_thread_data_on_global}) - .Run(c_thread_vec.n + i * BlkSize, p_c_global); - } - } - } -}; - -template -struct GridwiseBatchedGemmTransposedANormalBNormalCXdlops_v1 -{ - __device__ void Run(const Float* const __restrict__ p_a_global, - const Float* const __restrict__ p_b_global, - Float* const __restrict__ p_c_global) const - { - - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - constexpr auto True = integral_constant{}; - - constexpr auto a_g_k_m_global_desc = AGlobalDesc{}; - constexpr auto b_g_k_n_global_desc = BGlobalDesc{}; - constexpr auto c_g_m_n_global_desc = CGlobalDesc{}; - - constexpr auto G = b_g_k_n_global_desc.GetLengths()[0]; - - constexpr auto K = b_g_k_n_global_desc.GetLengths()[1]; - constexpr auto N = b_g_k_n_global_desc.GetLengths()[2]; - constexpr auto M = a_g_k_m_global_desc.GetLengths()[2]; - - // divide block work by [M, N] - static_assert(M % MPerBlock == 0 && N % NPerBlock == 0 && K % KPerBlock == 0, - "wrong! cannot divide work evenly among block"); - - constexpr index_t MBlockWork = M / MPerBlock; - constexpr index_t NBlockWork = N / NPerBlock; - - static_assert(MPerBlock % MPerWave == 0 && NPerBlock % NPerWave == 0, - "wrong! M/NPerBlock % M/NPerWave != 0"); - - constexpr index_t MWaves = MPerBlock / MPerWave; - constexpr index_t NWaves = NPerBlock / NPerWave; - - constexpr auto block_work_desc = - make_cluster_descriptor(Sequence{}); - - const auto block_work_id = block_work_desc.CalculateClusterIndex(get_block_1d_id()); - - const index_t group_id = block_work_id[0]; - const index_t m_block_data_on_global = block_work_id[1] * MPerBlock; - const index_t n_block_data_on_global = block_work_id[2] * NPerBlock; - - // LDS mem - constexpr index_t max_align = math::lcm(BBlockCopyDstDataPerWrite_N, - ABlockCopyDstDataPerWrite_M, - GemmDataPerReadM, - GemmDataPerReadN); - - // LDS - // be careful of LDS alignment - constexpr auto a_g_k_m_block_desc = make_native_tensor_descriptor_aligned( - Sequence<1, KPerBlock, MPerBlock>{}, Number{}); - - auto a_blockwise_copy = - BlockwiseGenericTensorSliceCopy_v4( - {group_id, 0, m_block_data_on_global}, {0, 0, 0}); - - constexpr auto b_g_k_n_block_desc = make_native_tensor_descriptor_aligned( - Sequence<1, KPerBlock, NPerBlock>{}, Number{}); - - auto b_blockwise_copy = - BlockwiseGenericTensorSliceCopy_v4( - {group_id, 0, n_block_data_on_global}, {0, 0, 0}); - - // GEMM definition - // c_mtx += transpose(a_mtx) * b_mtx - // a_mtx[KPerBlock, MPerBlock] is in LDS - // b_mtx[EPerBlocl, NPerBlock] is in LDS - // c_mtx[MPerBlock, NPerBlock] is distributed among threads, and saved in - // register - constexpr auto a_k_m_block_mtx_desc = make_ConstantMatrixDescriptor_packed( - a_g_k_m_block_desc.GetLength(I1), a_g_k_m_block_desc.GetLength(I2)); - constexpr auto b_k_n_block_mtx_desc = make_ConstantMatrixDescriptor_packed( - b_g_k_n_block_desc.GetLength(I1), b_g_k_n_block_desc.GetLength(I2)); - - const auto blockwise_gemm = BlockwiseGemmBlockABlockBThreadCTransANormalBNormalC_xdlops< - BlockSize, - decltype(a_k_m_block_mtx_desc), - decltype(b_k_n_block_mtx_desc), - Float, - MPerWave, - NPerWave, - MWaves, - NWaves, - GemmDataPerReadM, - GemmDataPerReadN>{}; - - constexpr index_t a_block_space = - math::integer_least_multiple(a_g_k_m_block_desc.GetElementSpace(), max_align); - - constexpr index_t b_block_space = - math::integer_least_multiple(b_g_k_n_block_desc.GetElementSpace(), max_align); - - __shared__ Float p_a_block_double[2 * a_block_space]; - __shared__ Float p_b_block_double[2 * b_block_space]; - - // get zero-initialized output register of vector type - auto c_thread_vec = blockwise_gemm.CreateOutputVecZero(); - - // LDS double buffer: preload data into LDS - { - a_blockwise_copy.Run(p_a_global, p_a_block_double); - b_blockwise_copy.Run(p_b_global, p_b_block_double); - } - - using b_blockwise_copy_src_step = Sequence<0, KPerBlock, 0>; - using a_blockwise_copy_src_step = Sequence<0, KPerBlock, 0>; - - // LDS double buffer: main body - for(index_t k_block_data_begin = 0; k_block_data_begin + 2 * KPerBlock < K; - k_block_data_begin += 2 * KPerBlock) - { -#pragma unroll - for(index_t iloop = 0; iloop < 2; ++iloop) - { - const bool even_loop = (iloop % 2 == 0); - - Float* p_a_block_now = - even_loop ? p_a_block_double : p_a_block_double + a_block_space; - Float* p_b_block_now = - even_loop ? p_b_block_double : p_b_block_double + b_block_space; - - Float* p_a_block_next = - even_loop ? p_a_block_double + a_block_space : p_a_block_double; - Float* p_b_block_next = - even_loop ? p_b_block_double + b_block_space : p_b_block_double; - - Float p_a_thread_buffer[a_blockwise_copy.GetThreadBufferSize()]; - Float p_b_thread_buffer[b_blockwise_copy.GetThreadBufferSize()]; - - a_blockwise_copy.MoveSrcSliceWindow(a_blockwise_copy_src_step{}, True); - b_blockwise_copy.MoveSrcSliceWindow(b_blockwise_copy_src_step{}, True); - - __syncthreads(); - - // LDS doubel buffer: load next data from device mem - a_blockwise_copy.RunLoadThreadBuffer(p_a_global, p_a_thread_buffer); - b_blockwise_copy.RunLoadThreadBuffer(p_b_global, p_b_thread_buffer); - - // LDS double buffer: GEMM on current data - c_thread_vec = blockwise_gemm.Run(p_a_block_now, p_b_block_now, c_thread_vec); - - // LDS double buffer: store next data to LDS - a_blockwise_copy.RunStoreThreadBuffer(p_a_thread_buffer, p_a_block_next); - b_blockwise_copy.RunStoreThreadBuffer(p_b_thread_buffer, p_b_block_next); - } - } - - // LDS double buffer: tail - { - constexpr bool has_two_iteration_left = (K % (2 * KPerBlock) == 0); - - if(has_two_iteration_left) // if has 2 iteration left - { - Float p_a_thread_buffer[a_blockwise_copy.GetThreadBufferSize()]; - Float p_b_thread_buffer[b_blockwise_copy.GetThreadBufferSize()]; - - a_blockwise_copy.MoveSrcSliceWindow(a_blockwise_copy_src_step{}, True); - b_blockwise_copy.MoveSrcSliceWindow(b_blockwise_copy_src_step{}, True); - - __syncthreads(); - - // LDS double buffer: load last data from device mem - a_blockwise_copy.RunLoadThreadBuffer(p_a_global, p_a_thread_buffer); - b_blockwise_copy.RunLoadThreadBuffer(p_b_global, p_b_thread_buffer); - - // LDS double buffer: GEMM on 2nd-last data - c_thread_vec = blockwise_gemm.Run(p_a_block_double, p_b_block_double, c_thread_vec); - - // LDS double buffer: store last data to LDS - a_blockwise_copy.RunStoreThreadBuffer(p_a_thread_buffer, - p_a_block_double + a_block_space); - b_blockwise_copy.RunStoreThreadBuffer(p_b_thread_buffer, - p_b_block_double + b_block_space); - - __syncthreads(); - - // LDS double buffer: GEMM on current data - c_thread_vec = blockwise_gemm.Run(p_a_block_double + a_block_space, - p_b_block_double + b_block_space, - c_thread_vec); - } - else // if has 1 iteration left - { - __syncthreads(); - - // LDS double buffer: GEMM on last data - c_thread_vec = blockwise_gemm.Run(p_a_block_double, p_b_block_double, c_thread_vec); - } - } - - // copy output: register to global memory - { - ///\todo inconsistent layout of xdlops and tensor - // xdlops layout - // M1 = num_groups; - // M0 = group_size; - // N1 = num_blks_per_wave; - // N0 = num_threads_per_blks; - constexpr auto CLayout = blockwise_gemm.GetOutputLayout(); - constexpr index_t M0 = CLayout.M1(); - constexpr index_t M1 = CLayout.N1(); - constexpr index_t M2 = CLayout.M0(); - - constexpr auto c_g_m0_m1_m2_n_global_desc = transform_tensor_descriptor( - c_g_m_n_global_desc, - make_tuple(PassThrough{}, UnMerge>{}, PassThrough{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1, 2, 3>{}, Sequence<4>{})); - - // src descriptor - constexpr auto c_g_m0_m1_m2_n_thread_desc = - make_native_tensor_descriptor_packed(Sequence<1, M0, 1, M2, 1>{}); - - using CThreadCopySliceLengths = Sequence<1, M0, 1, M2, 1>; - - constexpr index_t BlkSize = CLayout.GetBlkSize(); - constexpr index_t NumBlks = CLayout.GetNumBlks(); - - for(index_t i = 0; i < NumBlks; ++i) - { - // calculate origin of thread output tensor on global memory - // blockwise GEMM c matrix starting index - const auto c_thread_mtx_on_block = blockwise_gemm.GetBeginOfThreadMatrixC(i); - - const index_t m_thread_data_on_global = - m_block_data_on_global + c_thread_mtx_on_block.row; - - const index_t n_thread_data_on_global = - n_block_data_on_global + c_thread_mtx_on_block.col; - - ThreadwiseGenericTensorSliceCopy_v4r2::type, - 4, - 1, - 1, - AddressSpace::Vgpr, - AddressSpace::Global, - CGlobalMemoryDataOperation>( - {0, 0, 0, 0, 0}, - {group_id, - m_thread_data_on_global / (M2 * M1), - m_thread_data_on_global % (M2 * M1) / M2, - m_thread_data_on_global % M2, - n_thread_data_on_global}) - .Run(c_thread_vec.n + i * BlkSize, p_c_global); - } - } - } -}; - -} // namespace ck -#endif diff --git a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_threadwise_generic_tensor_slice_copy_deprecated.hpp b/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_threadwise_generic_tensor_slice_copy_deprecated.hpp deleted file mode 100644 index cf9a8bcbc7..0000000000 --- a/src/kernels/static_composable_kernel/include/tensor_operation/static_kernel_threadwise_generic_tensor_slice_copy_deprecated.hpp +++ /dev/null @@ -1,443 +0,0 @@ -#ifndef CK_THREADWISE_GENERIC_TENSOR_SLICE_COPY_DEPRECATED_HPP -#define CK_THREADWISE_GENERIC_TENSOR_SLICE_COPY_DEPRECATED_HPP - -#include "static_kernel_common_header.hpp" -#include "static_kernel_ConstantTensorDescriptor_deprecated.hpp" -#include "static_kernel_ConstantMergedTensorDescriptor_deprecated.hpp" -#include "static_kernel_tensor_coordinate_deprecated.hpp" - -namespace ck { - -// This threadwise copy allow vector access of src and dst. -// It allows the vector size to be different on src and dst. -// The dimensions of vector access should be the same on src and dst. -// The dimension access order should be the same on src and dst. -// It is designed for cases, where one of src and dst is register, and -// the other is device memory or LDS -template -struct ThreadwiseGenericTensorSliceCopy_v1r2_deprecated -{ - static constexpr index_t nDim = SliceLengths::GetSize(); - - __device__ constexpr ThreadwiseGenericTensorSliceCopy_v1r2_deprecated( - Array src_slice_origin, Array dst_slice_origin) - : mSrcSliceOrigin(src_slice_origin), mDstSliceOrigin(dst_slice_origin) - { - static_assert(nDim == SrcDesc::GetNumOfDimension() && - nDim == DstDesc::GetNumOfDimension() && nDim == SliceLengths::GetSize() && - nDim == DimAccessOrder::GetSize(), - "wrong! # of dimensions not the same"); - - static_assert(is_valid_sequence_map::value, "wrong! map is not valid"); - - static_assert( - SliceLengths{}[VectorAccessDim] % math::lcm(SrcDataPerAccess, DstDataPerAccess) == 0, - "wrong! cannot evenly divide"); - - // check vectorized memory access - constexpr auto vector_access_dim = Number{}; - - static_if{}([&](auto fwd) { - static_assert( - (fwd(SrcDesc{}).GetStride(vector_access_dim) == 1 || SrcDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }).Else([&](auto fwd) { - static_assert((fwd(SrcDesc{}).GetLastOriginalDimensionStride(vector_access_dim) == 1 || - SrcDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }); - - static_if{}([&](auto fwd) { - static_assert( - (fwd(DstDesc{}).GetStride(vector_access_dim) == 1 || DstDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }).Else([&](auto fwd) { - static_assert((fwd(DstDesc{}).GetLastOriginalDimensionStride(vector_access_dim) == 1 || - DstDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }); - } - - __device__ constexpr ThreadwiseGenericTensorSliceCopy_v1r2_deprecated() - : ThreadwiseGenericTensorSliceCopy_v1r2_deprecated(make_zero_array(), - make_zero_array()) - { - } - - __device__ void SetSrcSliceOrigin(Array src_slice_origin) - { - mSrcSliceOrigin = src_slice_origin; - } - - __device__ void SetDstSliceOrigin(Array dst_slice_origin) - { - mDstSliceOrigin = dst_slice_origin; - } - - template - __device__ void Run(const SrcData* p_src, DstData* p_dst) const - { - using src_vector_t = typename vector_type::MemoryType; - using dst_vector_t = typename vector_type::MemoryType; - - constexpr auto vector_access_dim = Number{}; - - constexpr auto src_data_per_access = Number{}; - constexpr auto dst_data_per_access = Number{}; - - constexpr auto long_vector_size = Number{}; - - constexpr auto long_vector_access_lengths = SliceLengths::Modify( - vector_access_dim, SliceLengths::Get(vector_access_dim) / long_vector_size); - - ford{}( - [&](auto long_vector_access_id) { - // data id w.r.t slicing-window - auto long_vector_data_begin_id = long_vector_access_id; - long_vector_data_begin_id(vector_access_dim) = - long_vector_size * long_vector_access_id[vector_access_dim]; - - // buffer to hold a long-vector - SrcData p_src_long_vector[long_vector_size]; - DstData p_dst_long_vector[long_vector_size]; - - // load data from src to the long-vector buffer - for(index_t i = 0; i < long_vector_size / src_data_per_access; ++i) - { - auto scalar_id = make_zero_array(); - scalar_id(vector_access_dim) = i * src_data_per_access; - - const index_t src_offset = SrcDesc::GetOffsetFromMultiIndex( - mSrcSliceOrigin + (long_vector_data_begin_id + scalar_id)); - - const index_t buffer_offset = i * src_data_per_access; - - *reinterpret_cast(&p_src_long_vector[buffer_offset]) = - *reinterpret_cast(&p_src[src_offset]); - } - - // type conversion - for(index_t i = 0; i < long_vector_size; ++i) - { - p_dst_long_vector[i] = type_convert{}(p_src_long_vector[i]); - } - - // store data from the long-vector buffer to dst - for(index_t i = 0; i < long_vector_size / dst_data_per_access; ++i) - { - auto scalar_id = make_zero_array(); - scalar_id(vector_access_dim) = i * dst_data_per_access; - - const index_t buffer_offset = i * dst_data_per_access; - - const index_t dst_offset = DstDesc::GetOffsetFromMultiIndex( - mDstSliceOrigin + (long_vector_data_begin_id + scalar_id)); - - *reinterpret_cast(&p_dst[dst_offset]) = - *reinterpret_cast(&p_dst_long_vector[buffer_offset]); - } - }); - } - -private: - Array mSrcSliceOrigin; - Array mDstSliceOrigin; -}; - -// This version use TensorCoordinate_deprecated -// This threadwise copy allow vector access of src and dst. -// It allows the dimensions of vector access to be different on src and dst. -// It also allows the vector size to be different on src and dst. -// It also allows order of access to be different on src and dst. -// It use register as buffer to hold all data moving from src to dst. -// It is designed for copying small amount of data, and src and dst are -// device memory or LDS. -// When copying large amout of data, let's hope compiler will reduce register -// used for the buffer. -template -struct ThreadwiseGenericTensorSliceCopy_v2r1_deprecated -{ - static constexpr index_t nDim = SliceLengths::GetSize(); - - using Index = MultiIndex; - - using SrcCoordinate = typename TensorCoordinate_deprecated::type; - using DstCoordinate = typename TensorCoordinate_deprecated::type; - - __device__ constexpr ThreadwiseGenericTensorSliceCopy_v2r1_deprecated( - const Index& src_slice_origin, const Index& dst_slice_origin) - : mSrcSliceOrigin(src_slice_origin), mDstSliceOrigin(dst_slice_origin) - { - static_assert(nDim == SrcDesc::GetNumOfDimension() && - nDim == DstDesc::GetNumOfDimension() && nDim == SliceLengths::GetSize() && - nDim == SrcDimAccessOrder::GetSize() && - nDim == DstDimAccessOrder::GetSize(), - "wrong! # of dimensions not the same"); - - static_assert(is_valid_sequence_map::value && - is_valid_sequence_map::value, - "wrong! map is not valid"); - - static_assert(SliceLengths{}[SrcVectorAccessDim] % SrcDataPerAccess == 0 && - SliceLengths{}[DstVectorAccessDim] % DstDataPerAccess == 0, - "wrong! cannot evenly divide"); - - // check vectorized memory access - constexpr auto src_vector_access_dim = Number{}; - constexpr auto dst_vector_access_dim = Number{}; - - static_if{}( - [&](auto fwd) { - static_assert( - (fwd(SrcDesc{}).GetStride(src_vector_access_dim) == 1 || SrcDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }) - .Else([&](auto fwd) { - static_assert( - (fwd(SrcDesc{}).GetLastOriginalDimensionStride(src_vector_access_dim) == 1 || - SrcDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }); - - static_if{}( - [&](auto fwd) { - static_assert( - (fwd(DstDesc{}).GetStride(dst_vector_access_dim) == 1 || DstDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }) - .Else([&](auto fwd) { - static_assert( - (fwd(DstDesc{}).GetLastOriginalDimensionStride(dst_vector_access_dim) == 1 || - DstDataPerAccess == 1), - "wrong! vectorized access is allowed only if stride == 1"); - }); - } - - __device__ constexpr ThreadwiseGenericTensorSliceCopy_v2r1_deprecated() - : ThreadwiseGenericTensorSliceCopy_v2r1_deprecated(make_zero_array(), - make_zero_array()) - { - } - - __device__ void SetSrcSliceOrigin(SrcCoordinate src_slice_origin) - { - mSrcSliceOrigin = src_slice_origin; - } - - __device__ void SetDstSliceOrigin(DstCoordinate dst_slice_origin) - { - mDstSliceOrigin = dst_slice_origin; - } - - template - struct IsolateMergedDimLengths - { - template - __device__ constexpr index_t operator()(IDim idim) const - { - return TDesc::ContainMultipleOriginalDimensions(idim) ? Lengths{}[idim] : 1; - } - }; - - template - __device__ void Run(const SrcData* p_src, - DstData* p_dst, - integral_constant, - integral_constant) const - { - constexpr auto buffer_desc = make_ConstantTensorDescriptor_packed(SliceLengths{}); - - SrcData p_src_buffer_[buffer_desc.GetElementSpace()]; - SrcData* p_src_buffer = p_src_buffer_; - - // copy data from src into buffer - { - constexpr auto src_vector_access_dim = Number{}; - constexpr auto src_data_per_access = Number{}; - - constexpr auto src_access_lengths = SliceLengths::Modify( - src_vector_access_dim, - SliceLengths::Get(src_vector_access_dim) / src_data_per_access); - - // Offset w.r.t merged dimensions need to be calculated at run-time. Offset w.r.t - // normal dimensions is known at compile time. - // Below is a hack to isolate merged dimension id from normal dimension id, so the - // corresponding offset can be calculated seperately at run-time and compile-time. - // src_merged_dim_access_lengths has the same value as src_access_lengths on src's - // merged dimensions, and has value = 1 on normal dimensions; - // src_merged_dim_access_lengths has the same value as src_access_lengths on src's - // normal dimensions, and has value = 1 on merged dimensions; - constexpr auto src_merged_dim_access_lengths = typename sequence_gen< - nDim, - IsolateMergedDimLengths>::type{}; - - constexpr auto src_normal_dim_access_lengths = - src_access_lengths + Number<1>{} - src_merged_dim_access_lengths; - - ford{}( - [&](auto src_merged_dim_access_id) { - auto src_merged_dim_data_id = src_merged_dim_access_id; - src_merged_dim_data_id(src_vector_access_dim) = - src_merged_dim_access_id[src_vector_access_dim] * src_data_per_access; - - // offset w.r.t. merged dimension need be computed at run-time, - const index_t src_merged_offset = - (mSrcSliceOrigin + src_merged_dim_data_id).GetOffset(); - - ford{}( - [&](auto src_normal_dim_access_id) { - auto src_normal_dim_data_id = src_normal_dim_access_id; - src_normal_dim_data_id(src_vector_access_dim) = - src_normal_dim_access_id[src_vector_access_dim] * - src_data_per_access; - - // offset w.r.t. normal dimension is known at compile-time - const index_t src_normal_offset = - SrcDesc::GetOffsetFromMultiIndex(src_normal_dim_data_id); - - SrcData p_src_vector_data[SrcDataPerAccess]; - - transfer_data( - p_src, src_normal_offset + src_merged_offset, p_src_vector_data, 0); - - // unpack vector into buffer - for(index_t i = 0; i < SrcDataPerAccess; ++i) - { - auto scalar_id = make_zero_array(); - scalar_id(src_vector_access_dim) = i; - - const index_t buffer_offset = buffer_desc.GetOffsetFromMultiIndex( - src_merged_dim_data_id + src_normal_dim_data_id + scalar_id); - - p_src_buffer[buffer_offset] = p_src_vector_data[i]; - } - }); - }); - } - - // type conversion - // TODO: would compiler do a good job reusing register for buffer? - DstData p_dst_buffer_[buffer_desc.GetElementSpace()]; - DstData* p_dst_buffer = p_dst_buffer_; - - ford{}([&](auto idx) { - p_dst_buffer[buffer_desc.GetOffsetFromMultiIndex(idx)] = - type_convert{}(p_src_buffer[buffer_desc.GetOffsetFromMultiIndex(idx)]); - }); - - // copy data from buffer into dst - { - constexpr auto dst_vector_access_dim = Number{}; - constexpr auto dst_data_per_access = Number{}; - - constexpr auto dst_access_lengths = SliceLengths::Modify( - dst_vector_access_dim, - SliceLengths::Get(dst_vector_access_dim) / dst_data_per_access); - - constexpr auto dst_merged_dim_access_lengths = typename sequence_gen< - nDim, - IsolateMergedDimLengths>::type{}; - - constexpr auto dst_normal_dim_access_lengths = - dst_access_lengths + Number<1>{} - dst_merged_dim_access_lengths; - - ford{}( - [&](auto dst_merged_dim_access_id) { - auto dst_merged_dim_data_id = dst_merged_dim_access_id; - dst_merged_dim_data_id(dst_vector_access_dim) = - dst_merged_dim_access_id[dst_vector_access_dim] * dst_data_per_access; - - // offset w.r.t. merged dimension need be computed at run-time, - const index_t dst_merged_offset = - (mDstSliceOrigin + dst_merged_dim_data_id).GetOffset(); - - ford{}( - [&](auto dst_normal_dim_access_id) { - auto dst_normal_dim_data_id = dst_normal_dim_access_id; - dst_normal_dim_data_id(dst_vector_access_dim) = - dst_normal_dim_access_id[dst_vector_access_dim] * - dst_data_per_access; - - DstData p_dst_vector_data[DstDataPerAccess]; - - // pack vector from buffer - for(index_t i = 0; i < DstDataPerAccess; ++i) - { - auto scalar_id = make_zero_array(); - scalar_id(dst_vector_access_dim) = i; - - const index_t buffer_offset = buffer_desc.GetOffsetFromMultiIndex( - dst_merged_dim_data_id + dst_normal_dim_data_id + scalar_id); - - p_dst_vector_data[i] = p_dst_buffer[buffer_offset]; - } - - // offset w.r.t. normal dimension is known at compile-time - const index_t dst_normal_offset = - DstDesc::GetOffsetFromMultiIndex(dst_normal_dim_data_id); - - transfer_data( - p_dst_vector_data, 0, p_dst, dst_normal_offset + dst_merged_offset); - }); - }); - } - } - - template - __device__ void Run(const SrcData* p_src, DstData* p_dst) const - { - constexpr auto generic_address_space = - integral_constant{}; - - Run(p_src, p_dst, generic_address_space, generic_address_space); - } - - // T can be Sequence or Array - template - __device__ void MoveSrcSliceWindow(T step_sizes, integral_constant) - { - static_if{}([&](auto) { - mSrcSliceOrigin += step_sizes; - }).Else([&](auto) { mSrcSliceOrigin -= step_sizes; }); - } - - template - __device__ void MoveDstSliceWindow(T step_sizes, integral_constant) - { - static_if{}([&](auto) { - mDstSliceOrigin += step_sizes; - }).Else([&](auto) { mDstSliceOrigin -= step_sizes; }); - } - -private: - SrcCoordinate mSrcSliceOrigin; - DstCoordinate mDstSliceOrigin; -}; - -} // namespace ck -#endif diff --git a/src/kernels/static_composable_kernel/src/kernel_wrapper/static_kernel_gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer.cpp b/src/kernels/static_composable_kernel/src/kernel_wrapper/static_kernel_gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer.cpp deleted file mode 100644 index 1c7fd21e3c..0000000000 --- a/src/kernels/static_composable_kernel/src/kernel_wrapper/static_kernel_gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer.cpp +++ /dev/null @@ -1,329 +0,0 @@ -#include "static_kernel_common_header.hpp" -#include "static_kernel_ConstantTensorDescriptor_deprecated.hpp" -#include "gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer.hpp" -#include "gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_fp16_bfp16_fwd_nchw_kcyx_nkhw_lds_double_buffer.hpp" -#include "gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_fp16_bfp16_wrw_nchw_kcyx_nkhw_lds_double_buffer.hpp" -#include "float_types.h" - -extern "C" __global__ - __launch_bounds__(CK_PARAM_TUNABLE_BLOCK_SIZE, 2) void gridwise_convolution_implicit_gemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer( - const FLOAT* const __restrict__ p_in_global, - const FLOAT* const __restrict__ p_wei_global, - FLOAT* const __restrict__ p_out_global) -{ - using namespace ck; - - // read params: problem decription - constexpr index_t N = CK_PARAM_PROBLEM_N; - constexpr index_t K = CK_PARAM_PROBLEM_K; - constexpr index_t C = CK_PARAM_PROBLEM_C; - constexpr index_t Hi = CK_PARAM_PROBLEM_HI; - constexpr index_t Wi = CK_PARAM_PROBLEM_WI; - constexpr index_t Ho = CK_PARAM_PROBLEM_HO; - constexpr index_t Wo = CK_PARAM_PROBLEM_WO; - constexpr index_t Y = CK_PARAM_PROBLEM_Y; - constexpr index_t X = CK_PARAM_PROBLEM_X; - - constexpr index_t ConvStrideH = CK_PARAM_PROBLEM_CONV_STRIDE_H; - constexpr index_t ConvStrideW = CK_PARAM_PROBLEM_CONV_STRIDE_W; - - constexpr index_t ConvDilationH = CK_PARAM_PROBLEM_CONV_DILATION_H; - constexpr index_t ConvDilationW = CK_PARAM_PROBLEM_CONV_DILATION_W; - - // read params: tunable params - constexpr index_t BlockSize = CK_PARAM_TUNABLE_BLOCK_SIZE; - - constexpr index_t GemmMPerBlock = CK_PARAM_TUNABLE_GEMM_M_PER_BLOCK; - constexpr index_t GemmNPerBlock = CK_PARAM_TUNABLE_GEMM_N_PER_BLOCK; - constexpr index_t GemmKPerBlock = CK_PARAM_TUNABLE_GEMM_K_PER_BLOCK; - constexpr index_t GemmKBlocks = CK_PARAM_TUNABLE_GEMM_K_BLOCKS; - - // read params: dependent params - constexpr index_t GridSize = CK_PARAM_DEPENDENT_GRID_SIZE; - - constexpr index_t LeftPadH = CK_PARAM_PROBLEM_LEFT_PAD_H; - constexpr index_t LeftPadW = CK_PARAM_PROBLEM_LEFT_PAD_W; - - constexpr index_t RightPadH = CK_PARAM_PROBLEM_RIGHT_PAD_H; - constexpr index_t RightPadW = CK_PARAM_PROBLEM_RIGHT_PAD_W; - - using LeftPads = Sequence; - using RightPads = Sequence; - -// calculate dependent params amd heuristic params -#if CK_PARAM_PROBLEM_DIRECTION == 2 - // In the WrW direction the filter is the output, while the output image is the input being - // convolved with the (original) input image. This requires that the tensordescriptors be - // swapped - // To reuse the fwd kernel for this operation we need to swap the n and c dimension of the - // input descriptor, the n and k dimension of the output descriptor - // This change is necessary so that reduction dimensions are consistent with the requirement - // of the wrw convolution when used in a fwd context - constexpr auto tmp_in_nchw_desc = - make_native_tensor_descriptor_packed(Sequence{}); - constexpr auto tmp_wei_kcyx_desc = make_native_tensor_descriptor_packed(Sequence{}); - constexpr auto tmp_out_nkhw_desc = - make_native_tensor_descriptor_packed(Sequence{}); - constexpr auto in_nchw_desc = - reorder_tensor_descriptor_given_upper2lower(tmp_in_nchw_desc, Sequence<1, 0, 2, 3>{}); - // wei and out are swapped in the solver - constexpr auto wei_kcyx_desc = - reorder_tensor_descriptor_given_upper2lower(tmp_out_nkhw_desc, Sequence<1, 0, 2, 3>{}); - constexpr auto out_nkhw_desc = - reorder_tensor_descriptor_given_upper2lower(tmp_wei_kcyx_desc, Sequence<1, 0, 2, 3>{}); - constexpr auto dir = ImplicitGemmDirection::BackwardWeight; - - // swap stride and dilation - using ConvDilations = Sequence; - using ConvStrides = Sequence; -#else - static_assert(GemmKBlocks == 1, "do not support GemmKBlocks > 1 for forward!"); - // calculate dependent params amd heuristic params - constexpr auto in_nchw_desc = make_native_tensor_descriptor_packed(Sequence{}); - constexpr auto wei_kcyx_desc = make_native_tensor_descriptor_packed(Sequence{}); - constexpr auto out_nkhw_desc = make_native_tensor_descriptor_packed(Sequence{}); - - constexpr auto dir = ImplicitGemmDirection::ForwardData; - using ConvStrides = Sequence; - using ConvDilations = Sequence; -#endif // CK_PARAM_PROBLEM_DIRECTION == 2 - - constexpr index_t GemmBBlockCopyClusterLengths_GemmK = - CK_PARAM_TUNABLE_GEMM_B_BLOCK_COPY_CLUSTER_LENGTHS_GEMM_K; - constexpr index_t GemmBBlockCopyClusterLengths_GemmN = - CK_PARAM_TUNABLE_GEMM_B_BLOCK_COPY_CLUSTER_LENGTHS_GEMM_N; - - constexpr index_t GemmBBlockCopyThreadSliceLengths_GemmK = - GemmKPerBlock / GemmBBlockCopyClusterLengths_GemmK; - constexpr index_t GemmBBlockCopyThreadSliceLengths_GemmN = - GemmNPerBlock / GemmBBlockCopyClusterLengths_GemmN; - - constexpr index_t GemmABlockCopyClusterLengths_GemmK = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_CLUSTER_LENGTHS_GEMM_K; - constexpr index_t GemmABlockCopyClusterLengths_GemmM = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_CLUSTER_LENGTHS_GEMM_M; - - constexpr index_t GemmABlockCopyThreadSliceLengths_GemmK = - GemmKPerBlock / GemmABlockCopyClusterLengths_GemmK; - constexpr index_t GemmABlockCopyThreadSliceLengths_GemmM = - GemmMPerBlock / GemmABlockCopyClusterLengths_GemmM; - -#if MIOPEN_USE_FP32 - using GemmABlockCopyThreadSliceLengths_GemmG_GemmK_GemmM = - Sequence<1, GemmABlockCopyThreadSliceLengths_GemmK, GemmABlockCopyThreadSliceLengths_GemmM>; - using GemmABlockCopyThreadClusterLengths_GemmG_GemmK_GemmM = - Sequence<1, GemmABlockCopyClusterLengths_GemmK, GemmABlockCopyClusterLengths_GemmM>; - - using GemmABlockCopyThreadClusterArrangeOrder = Sequence<0, 2, 1>; // [E0, K, E1] - using GemmABlockCopySrcAccessOrder = Sequence<0, 2, 1>; // [E0, K, E1] - using GemmABlockCopyDstAccessOrder = Sequence<0, 1, 2>; // [E0, E1, K] - - constexpr index_t GemmABlockCopySrcDataPerRead_GemmM = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_DST_DATA_PER_WRITE_GEMM_M; - - using GemmBBlockCopyThreadSliceLengths_GemmG_GemmK_GemmN = - Sequence<1, GemmBBlockCopyThreadSliceLengths_GemmK, GemmBBlockCopyThreadSliceLengths_GemmN>; - using GemmBBlockCopyThreadClusterLengths_GemmG_GemmK_GemmN = - Sequence<1, GemmBBlockCopyClusterLengths_GemmK, GemmBBlockCopyClusterLengths_GemmN>; - - using GemmBBlockCopyThreadClusterArrangeOrder = Sequence<0, 1, 2>; // [E0, E1, B] - using GemmBBlockCopySrcAccessOrder = Sequence<0, 1, 2>; // [E0, E1, B] - using GemmBBlockCopyDstAccessOrder = Sequence<0, 1, 2>; // [E0, E1, B] - - constexpr index_t GemmBBlockCopyDstDataPerWrite_GemmN = - CK_PARAM_TUNABLE_GEMM_B_BLOCK_COPY_DST_DATA_PER_WRITE_GEMM_N; - - constexpr index_t GemmABlockCopySrcDataPerRead_GemmK = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_SRC_DATA_PER_READ_GEMM_K; - -#elif MIOPEN_USE_FP16 || MIOPEN_USE_BFP16 - constexpr index_t GemmKPACK = CK_PARAM_GEMM_KPACK_LENGTH; - - using GemmABlockCopySubLengths_GemmG_GemmK_GemmM_GemmKPACK = - Sequence<1, - GemmABlockCopyThreadSliceLengths_GemmK, - GemmABlockCopyThreadSliceLengths_GemmM, - GemmKPACK>; - using GemmABlockCopyClusterLengths_GemmG_GemmK_GemmM_GemmKPACK = - Sequence<1, GemmABlockCopyClusterLengths_GemmK, GemmABlockCopyClusterLengths_GemmM, 1>; - - using GemmABlockCopyThreadClusterArrangeOrder = Sequence<0, 2, 1, 3>; // [G, M, K, GemmKPACK] - using GemmABlockCopySrcAccessOrder = Sequence<0, 2, 1, 3>; // [G, M, K, GemmKPACK] - using GemmABlockCopyDstAccessOrder = Sequence<0, 1, 2, 3>; // [G, K, M, GemmKPACK] - - constexpr index_t GemmABlockCopyDstDataPerWrite_GemmKPACK = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_DST_DATA_PER_WRITE_GEMM_KPACK; - - using GemmBBlockCopySubLengths_GemmG_GemmK_GemmN_GemmKPACK = - Sequence<1, - GemmBBlockCopyThreadSliceLengths_GemmK, - GemmBBlockCopyThreadSliceLengths_GemmN, - GemmKPACK>; - using GemmBBlockCopyClusterLengths_GemmG_GemmK_GemmN_GemmKPACK = - Sequence<1, GemmBBlockCopyClusterLengths_GemmK, GemmBBlockCopyClusterLengths_GemmN, 1>; - - using GemmBBlockCopyThreadClusterArrangeOrder = Sequence<0, 1, 3, 2>; // [G, K, GemmKPACK, B] - using GemmBBlockCopySrcAccessOrder = Sequence<0, 1, 3, 2>; // [G, K, GemmKPACK, B] - using GemmBBlockCopyDstAccessOrder = Sequence<0, 1, 2, 3>; // [G, K, B, GemmKPACK] - - constexpr index_t GemmBBlockCopyDstDataPerWrite_GemmKPACK = - CK_PARAM_TUNABLE_GEMM_B_BLOCK_COPY_DST_DATA_PER_WRITE_GEMM_KPACK; - -#if CK_PARAM_PROBLEM_DIRECTION == 2 - constexpr index_t GemmABlockCopySrcDataPerRead_GemmK = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_SRC_DATA_PER_READ_GEMM_K; -#else - constexpr index_t GemmABlockCopySrcDataPerRead_GemmKPACK = - CK_PARAM_TUNABLE_GEMM_A_BLOCK_COPY_SRC_DATA_PER_READ_GEMM_KPACK; -#endif // CK_PARAM_PROBLEM_DIRECTION - -#endif // MIOPEN_USE_FP16 || MIOPEN_USE_BFP16 - - constexpr index_t GemmBBlockCopySrcDataPerRead_GemmN = - CK_PARAM_TUNABLE_GEMM_B_BLOCK_COPY_SRC_DATA_PER_READ_GEMM_N; - - constexpr auto GemmMPerWave = CK_PARAM_GEMM_M_PER_WAVE; - constexpr auto GemmNPerWave = CK_PARAM_GEMM_N_PER_WAVE; - constexpr index_t GemmThreadGemmDataPerReadM = 1; - constexpr index_t GemmThreadGemmDataPerReadN = 1; - -#if MIOPEN_USE_FP32 - constexpr auto gridwise_conv = - GridwiseConvolutionImplicitGemm_v4r4_gen_xdlops_nchw_kcyx_nkhw_lds_double_buffer< - GridSize, - BlockSize, - FLOAT, - FLOAT_ACCUM, - decltype(in_nchw_desc), - decltype(wei_kcyx_desc), - decltype(out_nkhw_desc), - ConvStrides, - ConvDilations, - LeftPads, - RightPads, - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmKBlocks, - GemmMPerWave, - GemmNPerWave, - GemmThreadGemmDataPerReadM, - GemmThreadGemmDataPerReadN, - GemmABlockCopyThreadSliceLengths_GemmG_GemmK_GemmM, - GemmABlockCopyThreadClusterLengths_GemmG_GemmK_GemmM, - GemmABlockCopyThreadClusterArrangeOrder, - GemmABlockCopySrcAccessOrder, - GemmABlockCopyDstAccessOrder, - GemmABlockCopySrcDataPerRead_GemmK, - GemmABlockCopySrcDataPerRead_GemmM, - GemmBBlockCopyThreadSliceLengths_GemmG_GemmK_GemmN, - GemmBBlockCopyThreadClusterLengths_GemmG_GemmK_GemmN, - GemmBBlockCopyThreadClusterArrangeOrder, - GemmBBlockCopySrcAccessOrder, - GemmBBlockCopyDstAccessOrder, - GemmBBlockCopySrcDataPerRead_GemmN, - GemmBBlockCopyDstDataPerWrite_GemmN, - dir>{}; - gridwise_conv.Run(p_in_global, p_wei_global, p_out_global); - -#elif(MIOPEN_USE_FP16 || MIOPEN_USE_BFP16) && CK_PARAM_PROBLEM_DIRECTION == 2 - - // Backward weight in fp16/bfp16 uses atomic add to do reduction along K dimension - // It requires output blob to be of float as no atomic add exists for fp16/ushort - constexpr auto gridwise_conv = - GridwiseConvolutionImplicitGemm_v4r4_gen_xdlops_fp16_bfp16_wrw_nchw_kcyx_nkhw_lds_double_buffer< - GridSize, - BlockSize, - FLOAT, // Input data type = fp16 (fp16) or ushort (bfp16) - FLOAT_ACCUM, // Acc data type = float (see float_types.h) - float, // Output data type = float (not fp16/ushort) as no atomic add ISA exists for - // fp16/ushort. - decltype(in_nchw_desc), - decltype(wei_kcyx_desc), - decltype(out_nkhw_desc), - ConvStrides, - ConvDilations, - LeftPads, - RightPads, - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmKBlocks, - GemmKPACK, - GemmMPerWave, - GemmNPerWave, - GemmThreadGemmDataPerReadM, - GemmThreadGemmDataPerReadN, - GemmABlockCopySubLengths_GemmG_GemmK_GemmM_GemmKPACK, - GemmABlockCopyClusterLengths_GemmG_GemmK_GemmM_GemmKPACK, - GemmABlockCopyThreadClusterArrangeOrder, - GemmABlockCopySrcAccessOrder, - GemmABlockCopyDstAccessOrder, - GemmABlockCopySrcDataPerRead_GemmK, - GemmABlockCopyDstDataPerWrite_GemmKPACK, - GemmBBlockCopySubLengths_GemmG_GemmK_GemmN_GemmKPACK, - GemmBBlockCopyClusterLengths_GemmG_GemmK_GemmN_GemmKPACK, - GemmBBlockCopyThreadClusterArrangeOrder, - GemmBBlockCopySrcAccessOrder, - GemmBBlockCopyDstAccessOrder, - GemmBBlockCopySrcDataPerRead_GemmN, - GemmBBlockCopyDstDataPerWrite_GemmKPACK, - dir>{}; - - // Output blob is cast to float as no atomic add exists for fp16/ushort - gridwise_conv.Run(p_in_global, p_wei_global, reinterpret_cast(p_out_global)); -#elif(MIOPEN_USE_FP16 || MIOPEN_USE_BFP16) && CK_PARAM_PROBLEM_DIRECTION != 2 - // Forward data doesn't use any atomic add so output blob remains of the same type - // as input blob - - constexpr auto wkgrp_schd_order = -#if MIOPEN_USE_FP16 - NBlock1MBlock0; -#else - MBlock1NBlock0; -#endif // MIOPEN_USE_FP16 - - constexpr auto gridwise_conv = - GridwiseConvolutionImplicitGemm_v4r4_gen_xdlops_fp16_bfp16_fwd_nchw_kcyx_nkhw_lds_double_buffer< - GridSize, - BlockSize, - FLOAT, // Input data type = fp16 (fp16) or ushort (bfp16) - FLOAT_ACCUM, // Acc data type = float (see float_types.h) - FLOAT, // Input data type = fp16 (fp16) or ushort (bfp16) - decltype(in_nchw_desc), - decltype(wei_kcyx_desc), - decltype(out_nkhw_desc), - ConvStrides, - ConvDilations, - LeftPads, - RightPads, - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmKBlocks, - GemmKPACK, - GemmMPerWave, - GemmNPerWave, - GemmThreadGemmDataPerReadM, - GemmThreadGemmDataPerReadN, - GemmABlockCopySubLengths_GemmG_GemmK_GemmM_GemmKPACK, - GemmABlockCopyClusterLengths_GemmG_GemmK_GemmM_GemmKPACK, - GemmABlockCopyThreadClusterArrangeOrder, - GemmABlockCopySrcAccessOrder, - GemmABlockCopyDstAccessOrder, - GemmABlockCopySrcDataPerRead_GemmKPACK, - GemmABlockCopyDstDataPerWrite_GemmKPACK, - GemmBBlockCopySubLengths_GemmG_GemmK_GemmN_GemmKPACK, - GemmBBlockCopyClusterLengths_GemmG_GemmK_GemmN_GemmKPACK, - GemmBBlockCopyThreadClusterArrangeOrder, - GemmBBlockCopySrcAccessOrder, - GemmBBlockCopyDstAccessOrder, - GemmBBlockCopySrcDataPerRead_GemmN, - GemmBBlockCopyDstDataPerWrite_GemmKPACK, - dir, - wkgrp_schd_order>{}; - gridwise_conv.Run(p_in_global, p_wei_global, p_out_global); -#else - static_assert(false, "wrong! Only fp32, fp16 and bfp16 are supported."); -#endif // MIOPEN_USE_FP32 -} diff --git a/src/kthvalue.cpp b/src/kthvalue.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/kthvalue/problem_description.cpp b/src/kthvalue/problem_description.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/kthvalue_api.cpp b/src/kthvalue_api.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mlo_dir_conv.cpp b/src/mlo_dir_conv.cpp index 8fe63357cb..e69de29bb2 100644 --- a/src/mlo_dir_conv.cpp +++ b/src/mlo_dir_conv.cpp @@ -1,343 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#if MIOPEN_ENABLE_SQLITE -#include -#endif - -miopen::PerformanceDb miopen::GetDb(const miopen::ExecutionContext& ctx) -{ - return {DbKinds::PerfDb, ctx.GetPerfDbPath(), ctx.GetUserPerfDbPath()}; -} - -static auto GetGemmSolvers() -{ - return miopen::solver::SolverContainer{}; -} - -static auto GetDirectSolvers() -{ - return miopen::solver::SolverContainer{}; -} - -static auto GetImplicitGemmSolvers() -{ - return miopen::solver::SolverContainer< - miopen::solver::conv::ConvHipImplicitGemmForwardV4R5Xdlops, - miopen::solver::conv::ConvHipImplicitGemmForwardV4R4Xdlops, - miopen::solver::conv::ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm, - miopen::solver::conv::ConvHipImplicitGemmBwdDataV4R1Xdlops, - miopen::solver::conv::ConvHipImplicitGemmBwdDataV1R1Xdlops, - miopen::solver::conv::ConvHipImplicitGemmV4R1Fwd, - miopen::solver::conv::ConvHipImplicitGemmV4R4Fwd, - miopen::solver::conv::ConvMlirIgemmFwdXdlops, - miopen::solver::conv::ConvMlirIgemmFwd, - miopen::solver::conv::ConvMlirIgemmBwdXdlops, - miopen::solver::conv::ConvMlirIgemmBwd, - miopen::solver::conv::ConvHipImplicitGemmBwdDataV1R1, - miopen::solver::conv::ConvHipImplicitGemmBwdDataV4R1, - miopen::solver::conv::ConvAsmImplicitGemmV4R1DynamicFwd_1x1, - miopen::solver::conv::ConvAsmImplicitGemmV4R1DynamicFwd, - miopen::solver::conv::ConvAsmImplicitGemmV4R1DynamicBwd, - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicFwdXdlops, - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicBwdXdlops, - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC, - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC, - miopen::solver::conv::ConvCkIgemmFwdV6r1DlopsNchw, -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - miopen::solver::conv::ConvHipImplicitGemmFwdXdlops, - miopen::solver::conv::ConvHipImplicitGemmBwdXdlops, - miopen::solver::conv::ConvHipImplicitGemmGroupFwdXdlops, - miopen::solver::conv::ConvHipImplicitGemmGroupBwdXdlops, - miopen::solver::conv::ConvHipImplicitGemm3DGroupFwdXdlops, - miopen::solver::conv::ConvHipImplicitGemm3DGroupBwdXdlops, - miopen::solver::conv::ConvHipImplicitGemmF16F8F16FwdXdlops, - miopen::solver::conv::ConvHipImplicitGemmF16F8F16BwdXdlops, -#endif // MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC>{}; -} - -static auto GetWindogradSolvers() -{ - return miopen::solver::SolverContainer< - miopen::solver::conv::ConvBinWinograd3x3U, - miopen::solver::conv::ConvBinWinoRxS<3, 2>, - miopen::solver::conv::ConvBinWinoRxS<2, 3>, - miopen::solver::conv::ConvBinWinogradRxSf2x3g1, - miopen::solver::conv::ConvBinWinogradRxS, - miopen::solver::conv::ConvMPBidirectWinograd<3, 3>, - miopen::solver::conv::ConvMPBidirectWinograd<4, 3>, - miopen::solver::conv::ConvMPBidirectWinograd<5, 3>, - miopen::solver::conv::ConvMPBidirectWinograd<6, 3>, - miopen::solver::conv::ConvMPBidirectWinograd_xdlops<2, 3>, - miopen::solver::conv::ConvMPBidirectWinograd_xdlops<3, 3>, - miopen::solver::conv::ConvMPBidirectWinograd_xdlops<4, 3>, - miopen::solver::conv::ConvMPBidirectWinograd_xdlops<5, 3>, - miopen::solver::conv::ConvMPBidirectWinograd_xdlops<6, 3>, - miopen::solver::conv::ConvWinoFuryRxS<2, 3>>{}; -} - -static auto GetImplicitGemmWrWSolvers() -{ - return miopen::solver::SolverContainer< - miopen::solver::conv::ConvHipImplicitGemmWrwV4R4Xdlops, - miopen::solver::conv::ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm, - miopen::solver::conv::ConvHipImplicitGemmV4R1WrW, - miopen::solver::conv::ConvHipImplicitGemmV4R4WrW, - miopen::solver::conv::ConvAsmImplicitGemmV4R1DynamicWrw, - miopen::solver::conv::ConvMlirIgemmWrWXdlops, - miopen::solver::conv::ConvMlirIgemmWrW, - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicWrwXdlops, -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - miopen::solver::conv::ConvHipImplicitGemmGroupWrwXdlops, - miopen::solver::conv::ConvHipImplicitGemm3DGroupWrwXdlops, - miopen::solver::conv::ConvHipImplicitGemmF16F8F16WrwXdlops, -#endif // MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL - miopen::solver::conv::ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC>{}; -} - -static auto GetWindogradWrWSolvers() -{ - return miopen::solver::SolverContainer< - miopen::solver::conv::ConvBinWinogradRxS, - miopen::solver::conv::ConvBinWinoRxS<3, 2>, - miopen::solver::conv::ConvBinWinoRxS<2, 3>, - miopen::solver::conv::ConvBinWinogradRxSf2x3g1, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<3, 2>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<3, 3>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<3, 4>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<3, 5>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<3, 6>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<7, 2>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<7, 3>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<7, 3, 1, 1>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<7, 2, 1, 1>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<1, 1, 7, 2>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<1, 1, 7, 3>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<5, 3>, - miopen::solver::conv::ConvWinograd3x3MultipassWrW<5, 4>, - miopen::solver::conv::ConvWinoFuryRxS<2, 3>>{}; -} - -static auto GetBwdWrW2DSolvers() -{ - return miopen::solver::SolverContainer, - miopen::solver::conv::ConvOclBwdWrW2<2>, - miopen::solver::conv::ConvOclBwdWrW2<4>, - miopen::solver::conv::ConvOclBwdWrW2<8>, - miopen::solver::conv::ConvOclBwdWrW2<16>, - miopen::solver::conv::ConvOclBwdWrW2NonTunable, - miopen::solver::conv::ConvOclBwdWrW53, - miopen::solver::conv::ConvOclBwdWrW1x1, - miopen::solver::conv::ConvDirectNaiveConvFwd, - miopen::solver::conv::ConvDirectNaiveConvBwd, - miopen::solver::conv::ConvDirectNaiveConvWrw>{}; -} - -static auto GetFFTSolvers() { return miopen::solver::SolverContainer{}; } - -std::vector -FindAllGemmSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetGemmSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector> -AllGemmWorkspaceSize(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetGemmSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector -FindAllDirectSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetDirectSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector> -AllDirectForwardBackwardDataWorkspaceSize(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetDirectSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector> -FindAllWinogradWorkspaceSizes(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetWindogradSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector> -FindWinogradWrWWorkspaceSizes(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetWindogradWrWSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector> -FindAllImplicitGemmWorkspaceSizes(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetImplicitGemmSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector -FindAllImplicitGemmSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetImplicitGemmSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector -FindAllWinogradSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetWindogradSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector -FindWinogradWrWAllSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetWindogradWrWSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector> -AllDirectBwdWrW2DWorkspaceSize(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetBwdWrW2DSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector> -FindImplicitGemmWrWWorkspaceSizes(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetImplicitGemmWrWSolvers().GetWorkspaceSizes(ctx, problem); -} - -std::vector -FindImplicitGemmWrWAllSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetImplicitGemmWrWSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector -FindAllBwdWrW2DSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetBwdWrW2DSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector -FindAllFFTSolutions(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const miopen::AnyInvokeParams& invoke_ctx) -{ - return GetFFTSolvers().SearchForAllSolutions(ctx, problem, GetDb(ctx), invoke_ctx); -} - -std::vector> -AllFFTForwardBackwardDataWorkspaceSize(const miopen::ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) -{ - return GetFFTSolvers().GetWorkspaceSizes(ctx, problem); -} - -void mlo_construct_activ_lrn_pooling_common::setupFloats() -{ - if(_problem.GetInDataType() == miopenFloat && _problem.GetOutDataType() == miopenFloat) - { - _ctx.general_compile_options += " -DMIOPEN_USE_FP32=1 -DMIOPEN_USE_FP16=0"; - } - else if(_problem.GetInDataType() == miopenHalf && _problem.GetOutDataType() == miopenHalf) - { - _ctx.general_compile_options += " -DMIOPEN_USE_FP32=0 -DMIOPEN_USE_FP16=1"; - } - else - { - MIOPEN_LOG_W("Unsupported data types configuration: " - << miopen::GetDataTypeName(_problem.GetInDataType()) << "x" - << miopen::GetDataTypeName(_problem.GetOutDataType())); - } -} diff --git a/src/ocl/activ_ocl.cpp b/src/ocl/activ_ocl.cpp index fc301acadd..e69de29bb2 100644 --- a/src/ocl/activ_ocl.cpp +++ b/src/ocl/activ_ocl.cpp @@ -1,127 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace miopen { - -miopenStatus_t ActivationDescriptor::Forward(Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - size_t xOffset, - size_t yOffset) const -{ - if(!float_equal(*(static_cast(alpha)), 1.0) || - !float_equal(*(static_cast(beta)), 0)) - { - MIOPEN_THROW("Only alpha=1 and beta=0 is supported"); - } - - const auto problem = activ::ProblemDescription{*this, xDesc, yDesc}; - - const auto invoke_params = [&]() { - auto tmp = activ::InvokeParams{}; - tmp.type = InvokeType::Run; - tmp.alpha = GetAlpha(); - tmp.beta = GetBeta(); - tmp.gamma = GetGamma(); - tmp.x = x; - tmp.x_desc = xDesc; - tmp.y = y; - tmp.y_desc = yDesc; - tmp.x_offset = xOffset; - tmp.y_offset = yOffset; - return tmp; - }(); - - const auto algo = AlgorithmName{"miopenActivationForward"}; - const auto solvers = - solver::SolverContainer{}; - solvers.ExecutePrimitive(handle, problem, algo, invoke_params); - return miopenStatusSuccess; -} - -miopenStatus_t ActivationDescriptor::Backward(Handle& handle, - const void* alpha, - const TensorDescriptor& yDesc, - ConstData_t y, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - size_t yOffset, - size_t dyOffset, - size_t xOffset, - size_t dxOffset) const -{ - if(!float_equal(*(static_cast(alpha)), 1.0) || - !float_equal(*(static_cast(beta)), 0)) - { - MIOPEN_THROW("Only alpha=1 and beta=0 is supported"); - } - - const auto problem = activ::ProblemDescription{*this, xDesc, yDesc, dxDesc, dyDesc}; - - const auto invoke_params = [&]() { - auto tmp = activ::BwdInvokeParams{}; - tmp.type = InvokeType::Run; - tmp.alpha = GetAlpha(); - tmp.beta = GetBeta(); - tmp.gamma = GetGamma(); - tmp.x = x; - tmp.x_desc = xDesc; - tmp.y = y; - tmp.y_desc = yDesc; - tmp.dx = dx; - tmp.dx_desc = dxDesc; - tmp.dy = dy; - tmp.dy_desc = dyDesc; - tmp.x_offset = xOffset; - tmp.y_offset = yOffset; - tmp.dx_offset = dxOffset; - tmp.dy_offset = dyOffset; - return tmp; - }(); - - const auto algo = AlgorithmName{"miopenActivationBackward"}; - const auto solvers = solver::SolverContainer{}; - solvers.ExecutePrimitive(handle, problem, algo, invoke_params); - return miopenStatusSuccess; -} -} // namespace miopen diff --git a/src/ocl/convolutionocl.cpp b/src/ocl/convolutionocl.cpp index ee4c2839c7..e69de29bb2 100644 --- a/src/ocl/convolutionocl.cpp +++ b/src/ocl/convolutionocl.cpp @@ -1,1354 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_IMMED_FALLBACK) -MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DUMP_TENSOR_PATH) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_ENABLE_AI_IMMED_MODE_FALLBACK) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK) - -namespace miopen { - -static inline void ValidateWorkspace(Data_t workSpace, const size_t workSpaceSize) -{ - - [[maybe_unused]] bool x = (workSpace != nullptr); - [[maybe_unused]] bool y = (workSpaceSize != 0); - - assert(((x && y) || (!x && !y)) && "workspace pointer and size don't match. Either both should " - "be zero or both should be non-zero"); - - /// \todo could add a check here that workSpace points to GPU memory -} - -static Invoker PrepareInvoker(ExecutionContext ctx, - const conv::ProblemDescription& problem, - const NetworkConfig& config, - solver::Id solver_id) -{ - problem.SetupFloats(ctx); - ctx.do_search = false; - ctx.disable_search_enforce = true; - - const auto solver = solver_id.GetSolver(); - auto db = GetDb(ctx); - auto solution = solver.FindSolution(ctx, problem, db, {}); // auto tune is not expected here - auto& handle = ctx.GetStream(); - auto invoker = handle.PrepareInvoker(*solution.invoker_factory, solution.construction_params); - const auto algo = AlgorithmName{solver_id.GetAlgo(problem.GetDirection())}; - - handle.RegisterInvoker(invoker, config, solver_id.ToString(), algo); - return invoker; -} - -Invoker LoadOrPrepareInvoker(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - solver::Id solver_id) -{ - const auto& handle = ctx.GetStream(); - const auto config = problem.MakeNetworkConfig(); - auto invoker = handle.GetInvoker(config, solver_id); - if(invoker) - return *invoker; - return PrepareInvoker(ctx, problem, config, solver_id); -} - -static void -CompileSolution(solver::Id solver_id, ExecutionContext ctx, const conv::ProblemDescription& problem) -{ - if(!solver_id.IsValid()) - MIOPEN_THROW(miopenStatusBadParm, "solver_id = " + solver_id.ToString()); - - ctx.disable_search_enforce = true; - LoadOrPrepareInvoker(ctx, problem, solver_id); -} - -/// Keep only the best within algorithm, remove all others. -static void ShrinkToFind10Results(std::vector& found) -{ - std::sort(std::begin(found), std::end(found), [](auto&& l, auto&& r) { - return l.GetTime() < r.GetTime(); - }); - - std::vector out; - for(const auto& f : found) - { - // If an algo already resides in out, then skip solver. - auto algo_eq = [&](auto&& o) { return o.GetSolver().GetAlgo() == f.GetSolver().GetAlgo(); }; - if(std::find_if(std::begin(out), std::end(out), algo_eq) != std::end(out)) - continue; - out.emplace_back(f); - } - found = std::move(out); -} - -std::vector FindConvolution(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - const AnyInvokeParams& invoke_ctx, - int requestAlgoCount, - bool force_attach_binary) -{ - auto results = std::vector{}; - auto sol = boost::optional{}; - const auto& conv = problem.GetConv(); - const auto& findMode = conv.findMode; - - if(findMode.IsFast(ctx) || findMode.IsHybrid(ctx)) - { - auto fallback = bool{}; - auto sols = conv.GetSolutions(ctx, problem, 1, &fallback, &invoke_ctx); - // override the normal find with immed mode with env var - if(!sols.empty() && (!(findMode.IsHybrid(ctx) && fallback) || - env::enabled(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK))) - sol = sols.front(); - // In Hybrid Find mode, we use Normal Find instead of Immediate fallback kernels. - } - - if(sol.has_value()) - { - /// It is possible to measure actual execution time and return it to the caller. - /// \todo Consider if we need (and want to spend time) for this. - const auto id = solver::Id{sol->solution_id}; - const auto& s = id.GetSolver(); - CompileSolution(id, ctx, problem); - results.push_back({id, sol->time, s.GetWorkspaceSize(ctx, problem)}); - } - else - { - results = UserFindDbRecord::TryLoad(ctx.GetStream(), problem, [&]() { - auto ctx_copy = ctx; - ctx_copy.use_dynamic_solutions_only = findMode.IsDynamicHybrid(ctx); - const auto params = - conv::ConvFindParameters{conv.IsWinograd3x3SupportedAndFast(ctx_copy, problem)}; - - return FindCore(invoke_ctx, - ctx_copy, - problem, - params, - conv::GetConvSolverFinders(), - std::nullopt, - force_attach_binary); - }); - } - - if(env::enabled(MIOPEN_DEBUG_COMPILE_ONLY)) - { - MIOPEN_THROW( - miopenStatusGpuOperationsSkipped, - "MIOPEN_DEBUG_COMPILE_ONLY is enabled, escaping forward convolution. Search skipped."); - } - - ShrinkToFind10Results(results); - results.resize(std::min(results.size(), requestAlgoCount)); - - for(const auto& entry : results) - MIOPEN_LOG_I(entry.GetSolver().GetAlgo(problem.GetDirection()) - << "\t" << entry.GetTime() << "\t" << entry.GetWorkspaceSize()); - - return results; -} - -template -static inline void FillFindReturnParameters(const std::vector& results, - FieldType miopenConvAlgoPerf_t::*field, - const char* log_start, - int* const returned_algo_count, - miopenConvAlgoPerf_t* perf_results) -{ - *returned_algo_count = static_cast(results.size()); - - for(int i = 0; i < *returned_algo_count; i++) - { - perf_results[i].*field = static_cast(results[i].GetSolver().GetAlgo()); - perf_results[i].time = results[i].GetTime(); - perf_results[i].memory = results[i].GetWorkspaceSize(); - } - - MIOPEN_LOG_I(log_start << " Chosen Algorithm: " << results[0].GetSolver().ToString() << " , " - << results[0].GetWorkspaceSize() << ", " << results[0].GetTime()); -} - -void ConvolutionDescriptor::FindConvFwdAlgorithm(Handle& handle, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& yDesc, - Data_t y, - const int requestAlgoCount, - int* const returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) const -{ - MIOPEN_LOG_I("requestAlgoCount = " << requestAlgoCount << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - if(x == nullptr || w == nullptr || y == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Buffers cannot be NULL"); - if(returnedAlgoCount == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "returnedAlgoCount cannot be nullptr"); - if(perfResults == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "perfResults cannot be nullptr"); - if(requestAlgoCount < 1) - MIOPEN_THROW(miopenStatusBadParm, "requestAlgoCount cannot be < 1"); - - *returnedAlgoCount = 0; - - const auto problem = - conv::ProblemDescription(xDesc, wDesc, yDesc, *this, conv::Direction::Forward); - const auto ctx = [&] { - auto tmp = ExecutionContext{&handle}; - problem.SetupFloats(tmp); - tmp.do_search = exhaustiveSearch; - return tmp; - }(); - - const auto invoke_ctx = conv::DataInvokeParams{ - {xDesc, x, wDesc, w, yDesc, y}, workSpace, workSpaceSize, attribute.gfx90aFp16alt.GetFwd()}; - - const auto results = FindConvolution(ctx, problem, invoke_ctx, requestAlgoCount, false); - - if(results.empty()) - { - // Changes to this message lead to failures in test_conv_for_implicit_gemm - // To fix them check the test - // Two similar messages are in other convolution find methods - MIOPEN_THROW("No suitable algorithm was found to execute the required convolution"); - } - - FillFindReturnParameters( - results, &miopenConvAlgoPerf_t::fwd_algo, "FW", returnedAlgoCount, perfResults); -} - -namespace { - -// Currently 2D case only support default (alpha = 1.0 and beta = 0.0) -void ValidateAlphaBeta(const conv::ProblemDescription& problem) -{ - if(problem.Is2d() && problem.GetAlphaBetaCase() != DEFAULT) - { - MIOPEN_THROW(miopenStatusNotImplemented, - "Only alpha=1 and beta=0 is supported for 2D cases."); - } -} - -} // namespace - -void DumpTensorToFileFromDevice(const miopen::Handle& handle, - const miopen::TensorDescriptor& tDesc, - ConstData_t dData, - const fs::path& filename) -{ - if(dData == nullptr) - { - MIOPEN_LOG_E("Dereferencing nullptr when trying to dump tensor from gpu"); - return; - } - - fs::path path = filename.has_parent_path() ? filename : fs::current_path() / filename; - - if(!fs::is_directory(path.parent_path())) - { - MIOPEN_LOG_E("Directory does not exists : " << path); - return; - } - - std::ofstream file_stream{path, std::ios::binary}; - - if(!file_stream.is_open()) - { - MIOPEN_LOG_E("Cannot write to file : " << path); - return; - } - - // read tensor data from gpu - size_t num_bytes = tDesc.GetNumBytes(); - MIOPEN_LOG_I2("Start bringing tensor from device to host"); - std::vector hdata(num_bytes); - handle.ReadTo(hdata.data(), dData, num_bytes); - MIOPEN_LOG_I2("Done bringing tensor from device to host"); - // write tensor data to file - file_stream.write(hdata.data(), num_bytes); - file_stream.close(); - MIOPEN_LOG_I("Dumping tensor to file : " << path); -} - -static void ConvForwardCheckNumerics(const Handle& handle, - const ConvFwdTensors& tensors, - std::function&& worker) -{ - if(!miopen::CheckNumericsEnabled()) - { - worker(); - return; - } - - bool flag = false; - - flag |= miopen::checkNumericsInput(handle, tensors.xDesc, tensors.x); - flag |= miopen::checkNumericsInput(handle, tensors.wDesc, tensors.w); - - worker(); - - flag |= miopen::checkNumericsOutput(handle, tensors.yDesc, tensors.y); - - const auto& file_name = env::value(MIOPEN_DUMP_TENSOR_PATH); - if(flag && !file_name.empty()) - { - DumpTensorToFileFromDevice(handle, tensors.xDesc, tensors.x, file_name + "_x.bin"); - DumpTensorToFileFromDevice(handle, tensors.wDesc, tensors.w, file_name + "_w.bin"); - DumpTensorToFileFromDevice(handle, tensors.yDesc, tensors.y, file_name + "_y.bin"); - } -} - -void ConvolutionDescriptor::ValidateTensors(const ConvTensors& tensors) const -{ - - // Group stride in current TensorDescriptor is implicit. When invoking kernels, - // we need to add the group dimension G and compute its stride. We want the stride - // left of C to be a multiple of group count G. e.g. for NCHW, the stride for N - // should be a multiple of G so that we can compute the strides for NGCHW - auto bad_group_stride = [this](const TensorDescriptor& td) { - auto l = td.GetLayoutEnum(); - int g_stride_index = -1; - if(l == miopenTensorNCHW || l == miopenTensorNCDHW) - { - g_stride_index = 0; // stride index for N; - } - else if(l == miopenTensorNHWC || l == miopenTensorNDHWC) - { - // stride index for W. Normally this would be 2nd-last stride but we store - // strides in NCHW order for some weird reason. - g_stride_index = td.GetStrides().size() - 1; - } - else - { - MIOPEN_THROW(miopenStatusInternalError, "Layout not supported for grouped convolution"); - } - - if(g_stride_index != -1) - { - return (td.GetStrides()[g_stride_index] % this->group_count) != 0; - } - - return false; - }; - - // invalid_buffers - if(tensors.x == nullptr || tensors.w == nullptr || tensors.y == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm, "One of the convolution tensors is null"); - } - - // x_tensor_invalid = - if(tensors.xDesc.GetNumDims() < 3) - { - MIOPEN_THROW(miopenStatusBadParm, "input tensor's number of dimensions is wrong"); - } - - // tensor_sizes_not_matched = - if(tensors.xDesc.GetNumDims() != tensors.yDesc.GetNumDims() || - tensors.xDesc.GetNumDims() != tensors.wDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm, - "number of dimensions mismatch between input, output and weights tensors"); - } - - // trivial_tensor_types_not_matched = - if(tensors.xDesc.GetType() != tensors.yDesc.GetType() && tensors.xDesc.GetType() != miopenInt8) - { - MIOPEN_THROW(miopenStatusBadParm, "input/output tensor data types do not match"); - } - - // check for bad_group_stride. This applies for input and output only. There - // is no check for weight tensor currently. - // no need to check for group_count == 1 - - if((this->group_count > 1) && bad_group_stride(tensors.xDesc)) - { - MIOPEN_THROW( - miopenStatusBadParm, - "Invalid input tensor strides. Channel stride must be a multiple of group count"); - } - if((this->group_count > 1) && bad_group_stride(tensors.yDesc)) - { - MIOPEN_THROW( - miopenStatusBadParm, - "Invalid output tensor strides. Channel stride must be a multiple of group count"); - } - - // if(xDesc.GetLengths()[1] != wDesc.GetLengths()[1]) { - // MIOPEN_THROW(miopenStatusBadParm); - //} -} - -miopenDataType_t GetScalarDataType(const TensorDescriptor& xDesc) -{ - if(xDesc.GetType() == miopenDataType_t::miopenDouble) - { - return miopenDataType_t::miopenDouble; - } - else - { - return miopenDataType_t::miopenFloat; - } -} - -void ConvolutionDescriptor::ConvolutionForward(Handle& handle, - const void* alpha, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& wDesc, - ConstData_t w, - miopenConvFwdAlgorithm_t algo, - const void* beta, - const TensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - size_t workSpaceSize) const -{ - MIOPEN_LOG_I("algo = " << algo << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - - const auto tensors = ConvFwdTensors{xDesc, x, wDesc, w, yDesc, y}; - ValidateTensors(tensors); - Scalar alpha_val(alpha, GetScalarDataType(yDesc)); - Scalar beta_val(beta, GetScalarDataType(yDesc)); - const auto problem = conv::ProblemDescription{ - xDesc, wDesc, yDesc, *this, conv::Direction::Forward, 0, alpha_val, beta_val}; - ValidateAlphaBeta(problem); - - ConvForwardCheckNumerics(handle, tensors, [&]() { - Problem::ValidateGroupCount(xDesc, wDesc, *this); - - const auto algorithm_name = AlgorithmName{ConvolutionAlgoToDirectionalString( - static_cast(algo), conv::Direction::Forward)}; - const auto network_config = problem.MakeNetworkConfig(); - const auto& invoker = handle.GetInvoker(network_config, {}, algorithm_name); - - if(invoker) - { - const auto& invoke_ctx = conv::DataInvokeParams{tensors, - workSpace, - workSpaceSize, - this->attribute.gfx90aFp16alt.GetFwd(), - alpha_val, - beta_val}; - (*invoker)(handle, invoke_ctx); - return; - } - - MIOPEN_THROW("No invoker was registered for convolution forward. Was find executed?"); - }); -} - -static std::size_t GetSolutionCount(Handle& handle, const conv::ProblemDescription& problem) -{ - const FindDbRecord fdb_record{handle, problem}; - if(fdb_record.empty()) - return 0; - return std::distance(fdb_record.begin(), fdb_record.end()); -} - -static const char immFallbackFailed[] = - "Requested convolution is not supported or Immediate mode Fallback unsuccessful."; - -std::size_t -ConvolutionDescriptor::GetSolutionCountFallback(const ExecutionContext& ctx, - const conv::ProblemDescription& problem) const -{ - const auto maxSolutionCount = solver::GetSolversByPrimitive(solver::Primitive::Convolution) - .size(); // Simple and guarantees to provide enough space. - const auto n = GetSolutionsFallback(ctx, problem, maxSolutionCount).size(); - if(n > 0) - return n; - MIOPEN_LOG_I(immFallbackFailed); - /// When count=0 the reason could be: - /// * (1) Convolution is not implemented in the library at all, so Find() would fail as - /// well. This is case when rc = miopenStatusNotImplemented is correct. - /// * (2) Variant of the above: Convolution is implemented, but implementation is disabled, - /// for example, rocBLAS is not installed or some convolutions are disabled by the - /// environment setting. - /// * (3) There is none relevant record in the find-db and fallback path was unable to - /// choose suitable solution. - /// - /// We can't distinguish these three cases. - /// Let's do like Find() does: - MIOPEN_THROW(miopenStatusNotImplemented, immFallbackFailed); -} - -std::size_t ConvolutionDescriptor::GetSolutionCount(const ExecutionContext& ctx, - const conv::ProblemDescription& problem) const -{ - MIOPEN_LOG_I(""); - const auto n = miopen::GetSolutionCount(ctx.GetStream(), problem); - if(n > 0) - return n; - return GetSolutionCountFallback(ctx, problem); -} - -struct SolutionTimeComparator -{ - bool operator()(const miopenConvSolution_t& lhs, const miopenConvSolution_t& rhs) const - { - // Negative values are very coarse estimations. - // The more modulus, the "worse" (slower) is solution. - if(lhs.time < 0 && rhs.time < 0) - return !(lhs.time < rhs.time); - // Positive values are always "better" than negative (coarse) estimations. - if(lhs.time > 0 && rhs.time < 0) - return true; - if(lhs.time < 0 && rhs.time > 0) - return false; - // Both values are positive. The less is the better. - return (lhs.time < rhs.time); - } -}; - -namespace { - -std::ostream& operator<<(std::ostream& os, const miopenConvSolution_t& s) -{ - return os << "id: " << s.solution_id // - << ", algo: " << s.algorithm // - << ", time: " << s.time << ", ws: " << s.workspace_size // - << ", name: " << miopen::solver::Id(s.solution_id).ToString(); -} - -} // namespace - -std::vector -ConvolutionDescriptor::GetSolutionsFallback(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - const size_t maxSolutionCount, - const AnyInvokeParams* const invokeParams) const -{ - if(env::disabled(MIOPEN_DEBUG_CONV_IMMED_FALLBACK)) - { - MIOPEN_LOG_I("Disabled via environment"); - return {}; - } - - const auto& xDesc = - (problem.GetDirection() == conv::Direction::Forward) ? problem.GetIn() : problem.GetOut(); - const auto& weightsDesc = problem.GetWeights(); - // This check is needed on fallback path only. - // On regular path (find-db hit) this was checked during Find(). - Problem::ValidateGroupCount(xDesc, weightsDesc, *this); - - auto interim = std::vector{}; - interim.reserve(maxSolutionCount); // For speed. In most cases we have less entries than asked. - - // TunaNet Fallback -#if MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK - if(!env::disabled(MIOPEN_DEBUG_ENABLE_AI_IMMED_MODE_FALLBACK)) - { - const static std::string arch = ctx.GetStream().GetDeviceName(); - auto solvers = ai::immed_mode::PredictSolver(problem, ctx, arch); - if(!solvers.empty()) - { - MIOPEN_LOG_I2("Using TunaNet Fallback"); - const auto ai_time = [](const int& idx) { - return 10.0f * static_cast(idx); // Assume idx == 1 (best solver) is 10 ms. - }; - int idx = 1; - for(const auto kinder : solvers) - { - const auto solver_id = solver::Id{kinder}; - const auto sol = solver_id.GetSolver(); - const auto algo = solver_id.GetAlgo(); - if(conv::IsAlgorithmDisabled(algo)) - continue; - if(!sol.IsDynamic()) - continue; // branch should never be taken - if(!sol.IsApplicable(ctx, problem)) - continue; - const auto ws = sol.GetWorkspaceSize(ctx, problem); - if(!conv::IsEnoughWorkspace("GetSolutionsFallback AI", solver_id, ws, invokeParams)) - continue; - interim.emplace_back( - miopenConvSolution_t{ai_time(idx), ws, solver_id.Value(), algo}); - ++idx; - } - } - } -#endif // MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK - - // WTI Fallback - // if TunaNet is not enabled or produces no applicable solvers then fallback to WTI - if(interim.empty()) - { - MIOPEN_LOG_I2("Using WTI Fallback"); - const auto wti2time = [](const float& wti) { - assert(wti != 0.0f); - if(wti <= 0.0f) // Return negative values as is, avoid DIV/0. - return wti; - return 10.0f / wti; // Assume WTI == 1.0 (100%) is 10 ms. - }; - - for(const auto& solver_id : solver::GetSolversByPrimitive(solver::Primitive::Convolution)) - { - // solver_id is always valid here, because taken from registry. - // Validity check is not required. - const auto algo = solver_id.GetAlgo(); - if(conv::IsAlgorithmDisabled(algo)) // Algos can be disabled globally. - continue; - const auto& s = solver_id.GetSolver(); - // Let's allow non-dynamic later, if necessary. - if(s.IsEmpty() || !s.IsDynamic() || !s.IsApplicable(ctx, problem)) - continue; - const auto ws = s.GetWorkspaceSize(ctx, problem); - if(!conv::IsEnoughWorkspace("GetSolutionsFallback WTI", solver_id, ws, invokeParams)) - continue; - - const auto wti = s.GetWti(ctx, problem); - MIOPEN_LOG_I2(solver_id.ToString() << " Estimated WTI = " << wti); - if(wti < 0.0f) // Skip unknown WTIs. - continue; - interim.emplace_back(miopenConvSolution_t{wti2time(wti), ws, solver_id.Value(), algo}); - } - } - MIOPEN_LOG_I2("maxSolutionCount = " << maxSolutionCount << ", available = " << interim.size()); - for(const auto& s : interim) - MIOPEN_LOG_I2(s); - - std::sort(begin(interim), end(interim), SolutionTimeComparator{}); - interim.resize(std::min(maxSolutionCount, interim.size())); - - return interim; -} - -namespace { - -std::vector GetSolutions(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - const size_t maxSolutionCount, - const AnyInvokeParams* const invokeParams) -{ - auto algo_resolver = std::function{}; - - switch(problem.GetDirection()) - { - case conv::Direction::Forward: algo_resolver = &StringToConvolutionFwdAlgo; break; - case conv::Direction::BackwardData: algo_resolver = &StringToConvolutionBwdDataAlgo; break; - case conv::Direction::BackwardWeights: - algo_resolver = &StringToConvolutionBwdWeightsAlgo; - break; - } - - const FindDbRecord fdb_record{ctx.GetStream(), problem}; - - if(fdb_record.empty()) - return {}; - - auto interim = std::vector{}; - interim.reserve(20); // Heuristic for speed. - - for(const auto& pair : fdb_record) - { - const auto algo = static_cast(algo_resolver(pair.second.algorithm)); - if(conv::IsAlgorithmDisabled(algo)) - continue; - - const auto solver_id = solver::Id{pair.first}; - - // Wrong IDs can't be used to call IsApplicable(), so let's - // ignore obsolete or invalid IDs read from find-db first. - if(!solver_id.IsValid()) - { - // Do not disturb users with warnings unless detailed log is enabled. - MIOPEN_LOG_I("[Warning] incorrect solver_id: " << pair.first); - continue; - } - - interim.emplace_back( - miopenConvSolution_t{pair.second.time, pair.second.workspace, solver_id.Value(), algo}); - } - - /// Non-zero InvokeParams means that this function is used in Find to optimize host-side - /// performance (see Hybrid Find modes). Note that maxSolutionCount is usually 1 in this case. - /// - /// The size of the provided workspace in Hybrid Find modes is often smaller than necessary for - /// Normal Find, because GWSS in these modes return size suitable only for the "best" solver - /// \ref ffind_gwss_why_not_0. If we check IsEnoughWorkspace() for all solvers, then many false - /// warnings may be produced. That is why we have to check IsEnoughWorkspace for the - /// maxSolutionCount "best" solvers only. - /// - /// It is also highly desirable to avoid IsApplicable() checks for solutions that go beyond - /// maxSolutionCount, i.e. those that are not needed anyway. This optimization is important, for - /// example, to avoid applicability checks for MLIR solvers, since these may involve running the - /// MIIR compiler, which is very slow. - /// - /// The loop below does all the above at once. - std::sort(begin(interim), end(interim), SolutionTimeComparator{}); - auto out = std::vector{}; - out.reserve(maxSolutionCount); - auto n_copied = 0; - for(const auto& s : interim) - { - const auto solver_id = solver::Id{s.solution_id}; - if(!solver_id.GetSolver().IsApplicable(ctx, problem)) - continue; - if(!conv::IsEnoughWorkspace("GetSolutions", solver_id, s.workspace_size, invokeParams)) - continue; - out.push_back(s); - if(++n_copied >= maxSolutionCount) - break; - } - - for(const auto& s : out) - MIOPEN_LOG_I2(s); - - return out; -} - -} // namespace - -/// \todo Extend miopenConvSolution_t with an attribute indicating -/// how the solution was obtained (benchmarked on the current system, -/// taken from the System find-db, heuristically estimated, produced by -/// MLP classifier...) and then remove the fallbackPathTaken out param. -std::vector -ConvolutionDescriptor::GetSolutions(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - size_t maxSolutionCount, - bool* fallbackPathTaken, - const AnyInvokeParams* const invokeParams) const -{ - MIOPEN_LOG_I(""); - auto solutions = miopen::GetSolutions(ctx, problem, maxSolutionCount, invokeParams); - - if(fallbackPathTaken != nullptr) - *fallbackPathTaken = solutions.empty(); - - if(!solutions.empty()) - return solutions; - - return GetSolutionsFallback(ctx, problem, maxSolutionCount, invokeParams); -} - -std::size_t ConvolutionDescriptor::GetForwardSolutionWorkspaceSize(Handle& handle, - const TensorDescriptor& wDesc, - const TensorDescriptor& xDesc, - const TensorDescriptor& yDesc, - solver::Id solver_id) const -{ - MIOPEN_LOG_I("solver_id = " << solver_id.ToString()); - if(!solver_id.IsValid()) - MIOPEN_THROW(miopenStatusBadParm, "invalid solution id = " + solver_id.ToString()); - auto sol = solver_id.GetSolver(); - if(!sol.MayNeedWorkspace()) - return 0; - const auto problem = - conv::ProblemDescription{xDesc, wDesc, yDesc, *this, conv::Direction::Forward}; - auto ctx = ExecutionContext{}; - ctx.SetStream(&handle); - if(sol.IsApplicable(ctx, problem)) - return sol.GetWorkspaceSize(ctx, problem); - MIOPEN_THROW(miopenStatusBadParm, - "The supplied solution id: " + solver_id.ToString() + - " is not applicable to the current problem"); -} - -void ConvolutionDescriptor::CompileSolution(const ExecutionContext& ctx, - const conv::ProblemDescription& problem, - solver::Id solver_id) const -{ - MIOPEN_LOG_I("solver_id = " << solver_id.ToString()); - miopen::CompileSolution(solver_id, ctx, problem); -} - -void ConvolutionDescriptor::ConvolutionForwardImmediate(Handle& handle, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - Data_t workSpace, - const std::size_t workSpaceSize, - const solver::Id solver_id) const -{ - MIOPEN_LOG_I("solver_id = " << solver_id.ToString() << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - const auto tensors = ConvFwdTensors{xDesc, x, wDesc, w, yDesc, y}; - - ValidateTensors(tensors); - if(!solver_id.IsValid()) - MIOPEN_THROW(miopenStatusBadParm); - - ConvForwardCheckNumerics(handle, tensors, [&]() { - const auto problem = - conv::ProblemDescription{xDesc, wDesc, yDesc, *this, conv::Direction::Forward}; - const auto ctx = ExecutionContext{&handle}; - const auto invoker = LoadOrPrepareInvoker(ctx, problem, solver_id); - const auto invoke_ctx = conv::DataInvokeParams{ - tensors, workSpace, workSpaceSize, this->attribute.gfx90aFp16alt.GetFwd()}; - invoker(handle, invoke_ctx); - }); -} - -// FindBackwardDataAlgorithm() -// -void ConvolutionDescriptor::FindConvBwdDataAlgorithm(Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& dxDesc, - Data_t dx, - const int requestAlgoCount, - int* const returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) const -{ - MIOPEN_LOG_I("requestAlgoCount = " << requestAlgoCount << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - if(dx == nullptr || w == nullptr || dy == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Buffers cannot be NULL"); - if(returnedAlgoCount == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "returnedAlgoCount cannot be nullptr"); - if(perfResults == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "perfResults cannot be nullptr"); - if(requestAlgoCount < 1) - MIOPEN_THROW(miopenStatusBadParm, "requestAlgoCount cannot be < 1"); - - *returnedAlgoCount = 0; - - Problem::ValidateGroupCount(dxDesc, wDesc, *this); - - const auto problem = - conv::ProblemDescription{dyDesc, wDesc, dxDesc, *this, conv::Direction::BackwardData}; - - const auto ctx = [&] { - auto tmp = ExecutionContext{&handle}; - problem.SetupFloats(tmp); - tmp.do_search = exhaustiveSearch; - return tmp; - }(); - - const auto invoke_ctx = conv::DataInvokeParams{{dyDesc, dy, wDesc, w, dxDesc, dx}, - workSpace, - workSpaceSize, - this->attribute.gfx90aFp16alt.GetBwd()}; - - const auto results = FindConvolution(ctx, problem, invoke_ctx, requestAlgoCount, false); - - if(results.empty()) - { - // Changes to this message lead to failures in test_conv_for_implicit_gemm - // To fix them check the test - // Two similar messages are in other convolution find methods - MIOPEN_THROW("No suitable algorithm was found to execute the required convolution"); - } - - FillFindReturnParameters( - results, &miopenConvAlgoPerf_t::bwd_data_algo, "BWD", returnedAlgoCount, perfResults); -} - -static void ConvBwdCheckNumerics(const Handle& handle, - const ConvBwdTensors& tensors, - const void* beta, - std::function&& worker) -{ - if(!miopen::CheckNumericsEnabled()) - { - worker(); - return; - } - - bool flag = false; - - flag |= miopen::checkNumericsInput(handle, tensors.dyDesc, tensors.dy); - flag |= miopen::checkNumericsInput(handle, tensors.wDesc, tensors.w); - if(!float_equal(*(static_cast(beta)), 0)) - flag |= miopen::checkNumericsInput(handle, tensors.dxDesc, tensors.dx); - - worker(); - - flag |= miopen::checkNumericsOutput(handle, tensors.dxDesc, tensors.dx); - - const auto& file_name = env::value(MIOPEN_DUMP_TENSOR_PATH); - if(flag && !file_name.empty()) - { - DumpTensorToFileFromDevice(handle, tensors.dyDesc, tensors.dy, file_name + "_dy.bin"); - DumpTensorToFileFromDevice(handle, tensors.wDesc, tensors.w, file_name + "_w.bin"); - DumpTensorToFileFromDevice(handle, tensors.dxDesc, tensors.dx, file_name + "_dx.bin"); - } -} - -// BackwardDataAlgorithm() -void ConvolutionDescriptor::ConvolutionBackwardData(Handle& handle, - const void* alpha, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - miopenConvBwdDataAlgorithm_t algo, - const void* beta, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t workSpace, - size_t workSpaceSize) const -{ - MIOPEN_LOG_I("algo = " << algo << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - - auto tensors = ConvBwdTensors{dyDesc, dy, wDesc, w, dxDesc, dx}; - - ValidateTensors(tensors); - Scalar alpha_val(alpha, GetScalarDataType(dxDesc)); - Scalar beta_val(beta, GetScalarDataType(dxDesc)); - const auto problem = conv::ProblemDescription{ - dyDesc, wDesc, dxDesc, *this, conv::Direction::BackwardData, 0, alpha_val, beta_val}; - ValidateAlphaBeta(problem); - - ConvBwdCheckNumerics(handle, tensors, beta, [&]() { - if(dyDesc.GetLengths()[1] != wDesc.GetLengths()[0]) - { - MIOPEN_THROW(miopenStatusBadParm); - } - Problem::ValidateGroupCount(dxDesc, wDesc, *this); - - const auto algorithm_name = AlgorithmName{ConvolutionAlgoToDirectionalString( - static_cast(algo), conv::Direction::BackwardData)}; - - const auto network_config = problem.MakeNetworkConfig(); - const auto& invoker = handle.GetInvoker(network_config, {}, algorithm_name); - - if(!invoker) - MIOPEN_THROW("No invoker was registered for convolution backward. Was find executed?"); - - const auto& invoke_ctx = conv::DataInvokeParams{tensors, - workSpace, - workSpaceSize, - this->attribute.gfx90aFp16alt.GetBwd(), - alpha_val, - beta_val}; - (*invoker)(handle, invoke_ctx); - }); -} - -std::size_t ConvolutionDescriptor::GetBackwardSolutionWorkspaceSize(Handle& handle, - const TensorDescriptor& dyDesc, - const TensorDescriptor& wDesc, - const TensorDescriptor& dxDesc, - solver::Id solver_id) const -{ - MIOPEN_LOG_I2("solver_id = " << solver_id.ToString()); - if(!solver_id.IsValid()) - MIOPEN_THROW(miopenStatusBadParm, "invalid solution id = " + solver_id.ToString()); - - auto sol = solver_id.GetSolver(); - if(!sol.MayNeedWorkspace()) - return 0; - const auto problem = - conv::ProblemDescription{dyDesc, wDesc, dxDesc, *this, conv::Direction::BackwardData}; - auto ctx = ExecutionContext{}; - ctx.SetStream(&handle); - if(sol.IsApplicable(ctx, problem)) - { - return sol.GetWorkspaceSize(ctx, problem); - } - else - { - MIOPEN_THROW(miopenStatusBadParm, - "The supplied solution id: " + solver_id.ToString() + - " is not applicable to the current problem"); - } -} - -void ConvolutionDescriptor::ConvolutionBackwardImmediate(Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t workSpace, - std::size_t workSpaceSize, - solver::Id solver_id) const -{ - MIOPEN_LOG_I("solver_id = " << solver_id.ToString() << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - auto tensors = ConvBwdTensors{dyDesc, dy, wDesc, w, dxDesc, dx}; - - ValidateTensors(tensors); - - static const float beta = 0.0f; - ConvBwdCheckNumerics(handle, tensors, &beta, [&]() { - if(dyDesc.GetLengths()[1] != wDesc.GetLengths()[0]) - { - MIOPEN_THROW(miopenStatusBadParm); - } - Problem::ValidateGroupCount(dxDesc, wDesc, *this); - - const auto problem = - conv::ProblemDescription{dyDesc, wDesc, dxDesc, *this, conv::Direction::BackwardData}; - const auto ctx = ExecutionContext{&handle}; - const auto invoker = LoadOrPrepareInvoker(ctx, problem, solver_id); - const auto invoke_ctx = conv::DataInvokeParams{ - tensors, workSpace, workSpaceSize, this->attribute.gfx90aFp16alt.GetBwd()}; - invoker(handle, invoke_ctx); - }); -} - -// ConvolutionBackwardWeightsGetWorkSpaceSize -// FindBackwardWeightsAlgorithm() -// -void ConvolutionDescriptor::FindConvBwdWeightsAlgorithm(Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& dwDesc, - Data_t dw, - const int requestAlgoCount, - int* const returnedAlgoCount, - miopenConvAlgoPerf_t* perfResults, - Data_t workSpace, - size_t workSpaceSize, - bool exhaustiveSearch) const -{ - MIOPEN_LOG_I("requestAlgoCount = " << requestAlgoCount << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - if(x == nullptr || dw == nullptr || dy == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "Buffers cannot be NULL"); - if(returnedAlgoCount == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "returnedAlgoCount cannot be nullptr"); - if(perfResults == nullptr) - MIOPEN_THROW(miopenStatusBadParm, "perfResults cannot be nullptr"); - if(requestAlgoCount < 1) - MIOPEN_THROW(miopenStatusBadParm, "requestAlgoCount cannot be < 1"); - if(xDesc.GetType() == miopenInt8) - MIOPEN_THROW(miopenStatusBadParm); - - *returnedAlgoCount = 0; - - const auto problem = - conv::ProblemDescription{dyDesc, dwDesc, xDesc, *this, conv::Direction::BackwardWeights}; - const auto ctx = [&] { - auto tmp = ExecutionContext{&handle}; - problem.SetupFloats(tmp); - tmp.do_search = exhaustiveSearch; - return tmp; - }(); - - const auto invoke_ctx = conv::WrWInvokeParams{{dyDesc, dy, xDesc, x, dwDesc, dw}, - workSpace, - workSpaceSize, - attribute.gfx90aFp16alt.GetWrW()}; - - const auto results = FindConvolution(ctx, problem, invoke_ctx, requestAlgoCount, false); - - if(results.empty()) - { - // Changes to this message lead to failures in test_conv_for_implicit_gemm - // To fix them check the test - // Two similar messages are in other convolution find methods - MIOPEN_THROW("No suitable algorithm was found to execute the required convolution"); - } - - FillFindReturnParameters( - results, &miopenConvAlgoPerf_t::bwd_data_algo, "BWrW", returnedAlgoCount, perfResults); -} - -static void ConvWrwCheckNumerics(const Handle& handle, - const ConvWrwTensors& tensors, - const void* beta, - std::function&& worker) -{ - if(!miopen::CheckNumericsEnabled()) - { - worker(); - return; - } - - bool flag = false; - - flag |= miopen::checkNumericsInput(handle, tensors.dyDesc, tensors.dy); - flag |= miopen::checkNumericsInput(handle, tensors.xDesc, tensors.x); - if(!float_equal(*(static_cast(beta)), 0)) - flag |= miopen::checkNumericsInput(handle, tensors.dwDesc, tensors.dw); - - worker(); - - flag |= miopen::checkNumericsOutput(handle, tensors.dwDesc, tensors.dw); - - const auto& file_name = env::value(MIOPEN_DUMP_TENSOR_PATH); - if(flag && !file_name.empty()) - { - DumpTensorToFileFromDevice(handle, tensors.dyDesc, tensors.dy, file_name + "_dy.bin"); - DumpTensorToFileFromDevice(handle, tensors.xDesc, tensors.x, file_name + "_x.bin"); - DumpTensorToFileFromDevice(handle, tensors.dwDesc, tensors.dw, file_name + "_dw.bin"); - } -} - -// BackwardWeightsAlgorithm() -void ConvolutionDescriptor::ConvolutionBackwardWeights(const Handle& handle, - const void* alpha, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - miopenConvBwdWeightsAlgorithm_t algo, - const void* beta, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize) const -{ - MIOPEN_LOG_I("algo = " << algo << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - decltype(auto) tensors = ConvWrwTensors{dyDesc, dy, xDesc, x, dwDesc, dw}; - ValidateTensors(tensors); - decltype(auto) direction = conv::Direction::BackwardWeights; - Scalar alpha_val(alpha, GetScalarDataType(dwDesc)); - Scalar beta_val(beta, GetScalarDataType(dwDesc)); - decltype(auto) problem = - conv::ProblemDescription{dyDesc, dwDesc, xDesc, *this, direction, 0, alpha_val, beta_val}; - ValidateAlphaBeta(problem); - - if(xDesc.GetType() == miopenInt8) - MIOPEN_THROW(miopenStatusBadParm); - - ConvWrwCheckNumerics(handle, tensors, beta, [&]() { - Problem::ValidateGroupCount(xDesc, dwDesc, *this); - - decltype(auto) algorithm_name = AlgorithmName{ConvolutionAlgoToDirectionalString( - static_cast(algo), direction)}; - decltype(auto) network_config = problem.MakeNetworkConfig(); - decltype(auto) invoker = handle.GetInvoker(network_config, std::nullopt, algorithm_name); - - if(!invoker) - MIOPEN_THROW("No invoker was registered for convolution weights. Was find executed?"); - - const auto invoke_ctx = conv::WrWInvokeParams{tensors, - workSpace, - workSpaceSize, - this->attribute.gfx90aFp16alt.GetWrW(), - alpha_val, - beta_val}; - (*invoker)(handle, invoke_ctx); - }); -} - -std::size_t ConvolutionDescriptor::GetWrwSolutionWorkspaceSize(Handle& handle, - const TensorDescriptor& dyDesc, - const TensorDescriptor& xDesc, - const TensorDescriptor& dwDesc, - solver::Id solver_id) const -{ - MIOPEN_LOG_I2("solver_id = " << solver_id.ToString()); - if(!solver_id.IsValid()) - MIOPEN_THROW(miopenStatusBadParm, "invalid solution id = " + solver_id.ToString()); - - auto sol = solver_id.GetSolver(); - if(!sol.MayNeedWorkspace()) - return 0; - const auto problem = - conv::ProblemDescription{dyDesc, dwDesc, xDesc, *this, conv::Direction::BackwardWeights}; - auto ctx = ExecutionContext{}; - ctx.SetStream(&handle); - if(sol.IsApplicable(ctx, problem)) - { - return sol.GetWorkspaceSize(ctx, problem); - } - else - { - MIOPEN_THROW(miopenStatusBadParm, - "The supplied solution id: " + solver_id.ToString() + - " is not applicable to the current problem"); - } -} - -void ConvolutionDescriptor::ConvolutionWrwImmediate(Handle& handle, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - std::size_t workSpaceSize, - solver::Id solver_id) const -{ - MIOPEN_LOG_I("solver_id = " << solver_id.ToString() << ", workspace = " << workSpaceSize); - ValidateWorkspace(workSpace, workSpaceSize); - auto tensors = ConvWrwTensors{dyDesc, dy, xDesc, x, dwDesc, dw}; - ValidateTensors(tensors); - - if(xDesc.GetType() == miopenInt8) - MIOPEN_THROW(miopenStatusBadParm); - - float beta = 0; - ConvWrwCheckNumerics(handle, tensors, &beta, [&]() { - Problem::ValidateGroupCount(xDesc, dwDesc, *this); - - const auto problem = conv::ProblemDescription{ - dyDesc, dwDesc, xDesc, *this, conv::Direction::BackwardWeights}; - const auto ctx = ExecutionContext{&handle}; - const auto invoker = LoadOrPrepareInvoker(ctx, problem, solver_id); - const auto invoke_ctx = conv::WrWInvokeParams{ - tensors, workSpace, workSpaceSize, this->attribute.gfx90aFp16alt.GetWrW()}; - invoker(handle, invoke_ctx); - }); -} - -void ConvolutionBackwardBias(const Handle& handle, - const void* alpha, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const void* beta, - const TensorDescriptor& dbDesc, - Data_t db) -{ - if(dy == nullptr || db == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(dyDesc.GetLengths()[1] != dbDesc.GetLengths()[1]) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(!float_equal(*(static_cast(alpha)), 1.0) || - !float_equal(*(static_cast(beta)), 0)) - { - MIOPEN_THROW("Only alpha=1 and beta=0 is supported"); - } - if(miopen::CheckNumericsEnabled()) - { - miopen::checkNumericsInput(handle, dyDesc, dy); - } - - std::size_t out_n, out_k, stride_n, stride_k; - std::tie(out_n, out_k) = tie_pick<0, 1>()(dyDesc.GetLengths()); - std::tie(stride_n, stride_k) = tie_pick<0, 1>()(dyDesc.GetStrides()); - std::string algo_name = "miopenConvolutionBwdBias"; - std::string program_name = "MIOpenConvBwdBias.cl"; - std::string kernel_name = "MIOpenConvBwdB"; - std::string network_config = - "convbwdbias-" + - std::string(dyDesc.GetType() == miopenFloat - ? "fp32" - : (dyDesc.GetType() == miopenHalf - ? "fp16" - : (dyDesc.GetType() == miopenBFloat16 ? "bfloat16" : "int32"))); - - std::string params; - std::size_t lcl_grp_size0 = 256; - std::size_t lcl_grp_size1 = 1; - std::size_t local_mem_sz = 256; - - std::size_t map_size = std::accumulate(dyDesc.GetLengths().begin() + 2, - dyDesc.GetLengths().end(), - std::size_t(1), - std::multiplies()); - std::size_t read_unit = 4; - std::size_t map_size_aligned = (map_size + (read_unit - 1)) / read_unit; - std::size_t off_pix = map_size - (map_size / read_unit) * read_unit; - std::size_t total_work = map_size_aligned * out_n; - - params = " -DMLO_CONVBWD_GROUP_SZ0=" + std::to_string(lcl_grp_size0); - params += " -DMLO_CONVBWD_GROUP_SZ1=" + std::to_string(lcl_grp_size1); - params += " -DMLO_CONVBWDB_LCL_MEMSZ=" + std::to_string(local_mem_sz); - params += " -DMLO_CONVBWDB_UNITSIZE=" + std::to_string(read_unit); - - params += GetDataTypeKernelParams(dyDesc.GetType()); - - const std::vector vld = {lcl_grp_size0, size_t{1}, size_t{1}}; - const std::vector vgd = {lcl_grp_size0, size_t{256}, size_t{1}}; - - auto&& kernels = handle.GetKernels(algo_name, network_config); - if(!kernels.empty()) - { - kernels.front()(dy, - db, - static_cast(out_k), - static_cast(stride_k), - static_cast(stride_n), - static_cast(map_size_aligned), - static_cast(off_pix), - static_cast(total_work)); - } - else - { - handle.AddKernel(algo_name, network_config, program_name, kernel_name, vld, vgd, params)( - dy, - db, - static_cast(out_k), - static_cast(stride_k), - static_cast(stride_n), - static_cast(map_size_aligned), - static_cast(off_pix), - static_cast(total_work)); - } - - if(miopen::CheckNumericsEnabled()) - { - miopen::checkNumericsOutput(handle, dbDesc, db); - } -} - -} // namespace miopen diff --git a/src/ocl/ctcocl.cpp b/src/ocl/ctcocl.cpp index 35001e2855..e69de29bb2 100644 --- a/src/ocl/ctcocl.cpp +++ b/src/ocl/ctcocl.cpp @@ -1,310 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_ACTIVE_THREADS (64 * 4 * 64) -#define MAX_LOCAL_MEM 65536 - -namespace miopen { - -void CTCLossDescriptor::CTCLoss(Handle& handle, - const TensorDescriptor& probsDesc, - ConstData_t probs, - const int* labels, - const int* labelLengths, - const int* inputLengths, - Data_t losses, - const TensorDescriptor& gradientsDesc, - Data_t gradients, - miopenCTCLossAlgo_t algo, - Data_t workSpace, - size_t workSpaceSize) const -{ - (void)algo; - (void)workSpaceSize; - - if(probsDesc.GetType() != miopenFloat && probsDesc.GetType() != miopenHalf) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(probsDesc.GetLengths()[0] != gradientsDesc.GetLengths()[0] || - probsDesc.GetLengths()[1] != gradientsDesc.GetLengths()[1] || - probsDesc.GetLengths()[2] != gradientsDesc.GetLengths()[2]) - { - MIOPEN_THROW("probs tensor's dimension does not match gradients tensor's dimension"); - } - - int class_sz = probsDesc.GetLengths()[2]; - int batch_size = probsDesc.GetLengths()[1]; - int max_time_step = probsDesc.GetLengths()[0]; - std::vector repeat(batch_size, 0); - std::vector labels_offset(batch_size, 0); - int max_label_len = 0; - int total_label_len = 0; - - for(int i = 0; i < batch_size; i++) - { - if(inputLengths[i] > max_time_step) - { - MIOPEN_THROW("Wrong input time step"); - } - max_label_len = std::max(max_label_len, labelLengths[i]); - total_label_len += labelLengths[i]; - labels_offset[i] = i == 0 ? 0 : (labels_offset[i - 1] + labelLengths[i - 1]); - - for(int j = 0; j < labelLengths[i]; j++) - { - if(labels[labels_offset[i] + j] >= class_sz) - { - MIOPEN_THROW("Wrong label id"); - } - if(j > 0) - { - if(labels[labels_offset[i] + j] == labels[labels_offset[i] + j - 1]) - repeat[i]++; - } - } - - if(labelLengths[i] + repeat[i] > inputLengths[i]) - { - MIOPEN_THROW("Error: label length exceeds input time step"); - } - } - - int max_S_len = 2 * max_label_len + 1; - int lb_prime_offset = 4 * batch_size + total_label_len; - int problog_offset = lb_prime_offset + batch_size * max_S_len; - - if(probsDesc.GetType() == miopenHalf) - problog_offset *= 2; - - int alpha_offset = problog_offset + class_sz * batch_size * max_time_step; - int beta_offset = alpha_offset + max_time_step * batch_size * max_S_len; - int batch_bytes = 4 * batch_size; // batch size multiples sizeof(int) - -#if MIOPEN_BACKEND_OPENCL - auto q = handle.GetStream(); - - cl_context ctx; - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); - - clEnqueueWriteBuffer(q, workSpace, CL_FALSE, 0, batch_bytes, inputLengths, 0, nullptr, nullptr); - clEnqueueWriteBuffer( - q, workSpace, CL_FALSE, batch_bytes, batch_bytes, labelLengths, 0, nullptr, nullptr); - clEnqueueWriteBuffer(q, - workSpace, - CL_FALSE, - 2ULL * batch_bytes, - batch_bytes, - labels_offset.data(), - 0, - nullptr, - nullptr); - clEnqueueWriteBuffer(q, - workSpace, - CL_FALSE, - 3ULL * batch_bytes, - batch_bytes, - repeat.data(), - 0, - nullptr, - nullptr); - clEnqueueWriteBuffer(q, - workSpace, - CL_FALSE, - 4ULL * batch_bytes, - total_label_len * sizeof(int), - labels, - 0, - nullptr, - nullptr); - -#elif MIOPEN_BACKEND_HIP - - hipMemcpy(static_cast(workSpace), inputLengths, batch_bytes, hipMemcpyHostToDevice); - hipMemcpy(static_cast(workSpace) + batch_size, - labelLengths, - batch_bytes, - hipMemcpyHostToDevice); - hipMemcpy(static_cast(workSpace) + 2 * static_cast(batch_size), - labels_offset.data(), - batch_bytes, - hipMemcpyHostToDevice); - hipMemcpy(static_cast(workSpace) + 3 * static_cast(batch_size), - repeat.data(), - batch_bytes, - hipMemcpyHostToDevice); - hipMemcpy(static_cast(workSpace) + 4 * static_cast(batch_size), - labels, - total_label_len * sizeof(int), - hipMemcpyHostToDevice); -#endif - - std::string program_name = "MIOpenCTCLoss.cl"; - std::string kernel_name = "CTCLossGPU"; - - std::string network_config = - "t" + std::to_string(max_time_step) + "n" + std::to_string(batch_size) + "a" + - std::to_string(class_sz) + "mlb" + std::to_string(max_label_len) + "tlb" + - std::to_string(total_label_len) + "sfm" + - std::to_string(static_cast(apply_softmax_layer)) + "b" + - std::to_string(blank_label_id); // max timestep, batch, alphabet, max label length, total - // label length, softmax layer indicator, blank ID - - auto&& kernels = handle.GetKernels(kernel_name, network_config); - - float time = 0.; - if(apply_softmax_layer) - { - std::vector sfm_size(4, 1); - sfm_size[0] = max_time_step * batch_size; - sfm_size[1] = class_sz; - auto sfm_desc = miopen::TensorDescriptor(probsDesc.GetType(), sfm_size); - - float alpha = 1; - float beta = 0; - SoftmaxForward(handle, - &alpha, - &beta, - sfm_desc, - probs, - sfm_desc, - workSpace, - MIOPEN_SOFTMAX_LOG, - MIOPEN_SOFTMAX_MODE_CHANNEL, - 0, - problog_offset); - if(handle.IsProfilingEnabled()) - time += handle.GetKernelTime(); - } - - if(!kernels.empty()) - { - auto kernel = kernels.front(); - - kernel(probs, workSpace, workSpace, losses, gradients); - } - else - { - std::string params; - - size_t work_per_grp = batch_size <= 64 ? 256 : batch_size <= 128 ? 128 : 64; - assert(512 >= work_per_grp && work_per_grp > 0); - size_t glb_sz = batch_size < static_cast(MAX_ACTIVE_THREADS) / work_per_grp - ? batch_size * work_per_grp - : static_cast(MAX_ACTIVE_THREADS); - size_t grp_num = glb_sz / work_per_grp; - - size_t lcl_mem_per_grp = MAX_LOCAL_MEM / 2 / (512 / work_per_grp); - - params += " -DCLASS_SZ=" + std::to_string(class_sz) + - " -DBATCH_SZ=" + std::to_string(batch_size) + - " -DMAX_TSTEP=" + std::to_string(max_time_step) + - " -DMAX_LB_LEN=" + std::to_string(max_label_len) + - " -DTOTAL_LB_LEN=" + std::to_string(total_label_len) + - " -DMAX_S_LEN=" + std::to_string(max_S_len) + - " -DLB_PRIME_OFFSET=" + std::to_string(lb_prime_offset) + - " -DPROBLOG_OFFSET=" + std::to_string(problog_offset) + - " -DALPHA_OFFSET=" + std::to_string(alpha_offset) + - " -DBETA_OFFSET=" + std::to_string(beta_offset) + - " -DWORK_PER_GRP=" + std::to_string(work_per_grp) + - " -DGRP_NUM=" + std::to_string(grp_num) + - " -DBLANK_LB_ID=" + std::to_string(blank_label_id); - - if(!probsDesc.IsPacked()) - { - params += " -DPROBS_STRIDE0=" + std::to_string(probsDesc.GetStrides()[0]) + - " -DPROBS_STRIDE1=" + std::to_string(probsDesc.GetStrides()[1]); - } - - if(!gradientsDesc.IsPacked()) - { - params += " -DGRADS_STRIDE0=" + std::to_string(gradientsDesc.GetStrides()[0]) + - " -DGRADS_STRIDE1=" + std::to_string(gradientsDesc.GetStrides()[1]); - } - - params += " -DSOFTMAX_APPLIED=" + std::to_string(static_cast(apply_softmax_layer)) + - " -DSOFTMAX_LEN=" + std::to_string(class_sz); - -#if MIOPEN_BACKEND_OPENCL - if(class_sz <= lcl_mem_per_grp) - params += " -DOPT_LCL_MEM_GRAD"; -#endif - - if(static_cast(max_S_len) * 2 -#if MIOPEN_BACKEND_OPENCL - + class_sz -#endif - <= lcl_mem_per_grp) - { - params += " -DOPT_LCL_MEM_BETA"; - } - - if(static_cast(max_S_len) * 3 -#if MIOPEN_BACKEND_OPENCL - + class_sz -#endif - <= lcl_mem_per_grp) - { - params += " -DOPT_LCL_MEM_LB"; - } - - if(probsDesc.GetType() == miopenHalf) - params += " -DMIOPEN_USE_FP16=1"; - else - params += " -DMIOPEN_USE_FP32=1 -DOPT_ATOMIC_LOGADDEXP"; - -#if MIOPEN_BACKEND_HIP - params += " -DUSE_HIP_BACKEND=1"; -#elif MIOPEN_BACKEND_OPENCL - params += " -DUSE_OCL_BACKEND=1"; -#endif - - const std::vector vld{work_per_grp, 1, 1}; - const std::vector vgd{glb_sz, 1, 1}; - - handle.AddKernel(kernel_name, network_config, program_name, kernel_name, vld, vgd, params)( - probs, workSpace, workSpace, losses, gradients); - } - if(handle.IsProfilingEnabled()) - handle.AccumKernelTime(time); -} - -} // namespace miopen diff --git a/src/ocl/dropoutocl.cpp b/src/ocl/dropoutocl.cpp index 28fbe01a08..e69de29bb2 100644 --- a/src/ocl/dropoutocl.cpp +++ b/src/ocl/dropoutocl.cpp @@ -1,580 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../../test/dropout_util.hpp" - -#define DROPOUT_DEBUG 0 - -namespace miopen { - -template -inline void SquashPairedTensor(const std::vector x_len, - const std::vector x_str, - const std::vector y_len, - const std::vector y_str, - std::vector& in_len, - std::vector& in_str, - std::vector& out_len, - std::vector& out_str) -{ - if(!std::equal(x_len.begin(), x_len.end(), y_len.begin())) - { - MIOPEN_THROW("Input/Output tensor lengths do not match"); - } - - in_len.back() = x_len.back(); - in_str.back() = x_str.back(); - out_len.back() = y_len.back(); - out_str.back() = y_str.back(); - - int xl_idx = x_len.size() - 2; - int yl_idx = y_len.size() - 2; - int xs_idx = x_str.size() - 2; - int ys_idx = y_str.size() - 2; - - int il_idx = in_len.size() - 1; - int ol_idx = out_len.size() - 1; - int is_idx = in_str.size() - 2; - int os_idx = out_str.size() - 2; - - while(xl_idx >= 0 && x_str[xs_idx] == x_len[xl_idx + 1] * x_str[xs_idx + 1] && - y_str[ys_idx] == y_len[yl_idx + 1] * y_str[ys_idx + 1]) - { - in_len[il_idx] *= x_len[xl_idx--]; - out_len[ol_idx] *= y_len[yl_idx--]; - - xs_idx--; - ys_idx--; - } - - if(xl_idx < 0 && is_idx >= 0) - { - in_str[is_idx--] = in_len[il_idx]; - out_str[os_idx--] = out_len[ol_idx]; - } - else if(xl_idx >= 0) - { - il_idx--; - ol_idx--; - - while(xl_idx >= 0 && il_idx >= 0) - { - in_len[il_idx--] = x_len[xl_idx--]; - in_str[is_idx--] = x_str[xs_idx--]; - } - - while(yl_idx >= 0 && ol_idx >= 0) - { - out_len[ol_idx--] = y_len[yl_idx--]; - out_str[os_idx--] = y_str[ys_idx--]; - } - } - - while(is_idx >= 0) - in_str[is_idx--] = in_str[is_idx + 1] * in_len[is_idx + 1]; - - while(os_idx >= 0) - out_str[os_idx--] = out_str[os_idx + 1] * out_len[os_idx + 1]; - - if(!std::equal(in_len.begin(), in_len.end(), out_len.begin())) - { - MIOPEN_THROW("Input/Output tensor lengths do not match"); - } -} - -void DropoutDescriptor::InitPRNGState(Handle& handle, - Data_t prng_states, - size_t prng_stateSizeInBytes, - unsigned long long prng_seed) const -{ -#if DROPOUT_DEBUG - std::cout << "Check memory and threads info of dropout PRNG states in debug mode:" << std::endl; -#endif - std::string program_name = "MIOpenDropoutHIP.cpp"; - std::string kernel_name = "InitKernelStateHIP"; - - if(prng_stateSizeInBytes > handle.GetMaxMemoryAllocSize()) - { - MIOPEN_THROW("PRNG state size should not exceed system maximum memory allocation size."); - } - - unsigned long long states_num = prng_stateSizeInBytes / sizeof(rocrand_state_xorwow); - size_t wk_grp_num = - std::min(static_cast(MAX_PRNG_STATE / 256), (states_num + 255) / 256); - - std::string network_config = "initprngs-" + std::to_string(sizeof(rocrand_state_xorwow)) + "x" + - std::to_string(rng_mode) + "x" + std::to_string(wk_grp_num); - - auto&& kernels = handle.GetKernels(kernel_name, network_config); - if(!kernels.empty()) - { - kernels.front()(prng_states, prng_seed, states_num); - } - else - { - const std::vector vld{256, 1, 1}; - const std::vector vgd{wk_grp_num * 256, 1, 1}; - - std::string params; - params += "-DRUN_FORWARD=0 -DRUN_INIT_PRNG=1"; -#if DROPOUT_DEBUG - std::cout << "Threads allocated for PRNG states: " << vgd[0] << std::endl; - std::cout << "Memory allocated for PRNG states: " << stateSizeInBytes << std::endl; -#endif - handle.AddKernel(kernel_name, network_config, program_name, kernel_name, vld, vgd, params)( - prng_states, prng_seed, states_num); -#if DROPOUT_DEBUG - std::cout << "Succeeded in launching InitPRNGState()." << stateSizeInBytes << std::endl; -#endif - } -} - -void DropoutDescriptor::DropoutForward(const Handle& handle, - const TensorDescriptor& noise_shape, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - Data_t reserveSpace, - size_t reserveSpaceSizeInBytes, - size_t in_offset, - size_t out_offset, - size_t rsvsp_offset) const -{ - if(x == nullptr || y == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(xDesc.GetNumDims() != yDesc.GetNumDims()) - { - MIOPEN_THROW("Input/Output dimension does not match"); - } - - if(xDesc.GetNumDims() > 5) - { - MIOPEN_THROW("Only support 1D to 5D tensors"); - } - - if(xDesc.GetElementSize() != yDesc.GetElementSize()) - { - MIOPEN_THROW("Input/Output element size does not match"); - } - - if(xDesc.GetElementSize() != noise_shape.GetElementSize() || - xDesc.GetNumDims() != noise_shape.GetNumDims()) - { - MIOPEN_THROW("Only support dropout with regular noise shape currently"); - } - - if(xDesc.GetType() != yDesc.GetType()) - { - MIOPEN_THROW("Input/Output datatype does not match"); - } - - if(dropout < 0.0 || dropout > 1.0) - { - MIOPEN_THROW("Invalid dropout rate"); - } - - bool use_rsvsp = !(reserveSpace == nullptr); - if(((use_rsvsp || use_mask) && - reserveSpaceSizeInBytes < xDesc.GetElementSize() * sizeof(bool)) || - (use_mask && reserveSpace == nullptr)) - { - MIOPEN_THROW("Insufficient reservespace size"); - } - - if(stateSizeInBytes + reserveSpaceSizeInBytes + - xDesc.GetElementSize() * GetTypeSize(xDesc.GetType()) + - yDesc.GetElementSize() * GetTypeSize(yDesc.GetType()) > - handle.GetGlobalMemorySize()) - { - MIOPEN_THROW("Memory required by dropout forward configs exceeds GPU memory range."); - } - - if(miopen::CheckNumericsEnabled()) - { - std::cout << "Dropout forward input numerics check at dropout rate " << dropout - << std::endl; - miopen::checkNumericsInput(handle, xDesc, x); - } - - // support up to 5D tensor - std::vector in_len(5, 1); - std::vector in_str(5, 1); - std::vector out_len(5, 1); - std::vector out_str(5, 1); - - SquashPairedTensor(xDesc.GetLengths(), - xDesc.GetStrides(), - yDesc.GetLengths(), - yDesc.GetStrides(), - in_len, - in_str, - out_len, - out_str); - - size_t RD_BLCK = /* (in_len[4] % 4 == 0) ? 4 : */ (in_len[2] % 2 == 0) ? 2 : 1; - size_t total_work = (in_len[4] / RD_BLCK) * in_len[3] * in_len[2] * in_len[1] * in_len[0]; - - size_t max_wk_grp = use_mask ? size_t(MAX_WORKITEM_NUM) - : std::min(size_t(MAX_PRNG_STATE), handle.GetImage3dMaxWidth()); - size_t wk_grp_num = - std::min(max_wk_grp / 256, - ((in_len[4] * in_len[3] * in_len[2] * in_len[1] * in_len[0] + 255) / 256)); - - size_t states_num = stateSizeInBytes / sizeof(rocrand_state_xorwow); - if(states_num < wk_grp_num * 256 && !use_mask) - { - MIOPEN_THROW("Insufficient state size for parallel PRNG"); - } - - std::string program_name = "MIOpenDropoutHIP.cpp"; - std::string kernel_name = "DropoutFW"; - - std::string network_config = - "fwd-" + std::string(xDesc.GetType() == miopenHalf ? "fp16-" : "fp32-") + "-seed" + - std::to_string(seed) + "-rng" + std::to_string(rng_mode) + "-rsvsp" + - std::to_string(static_cast(use_rsvsp)) + "-mask" + - std::to_string(static_cast(use_mask)) + "-evo" + - std::to_string(static_cast(state_evo)) + "-blk" + std::to_string(RD_BLCK) + "-wg" + - std::to_string(wk_grp_num) /* + "-noise" + std::to_string(noise_shape.GetLengths()[0])*/; - - // TODO: Add noise shape - // for(int i = 1; i < noise_shape.GetNumDims(); i++) - // network_config += "x" + std::to_string(noise_shape.GetLengths()[i]); - - auto&& kernels = handle.GetKernels(kernel_name, network_config); - - float amp_scale = float_equal(dropout, 1.0) ? 0 : 1 / (1 - dropout); - if(!kernels.empty()) - { - kernels.front()(pstates, - dropout, - amp_scale, - static_cast(in_len[1]), - static_cast(in_len[2]), - static_cast(in_len[3]), - static_cast(in_len[4]), - y, - static_cast(out_str[0]), - static_cast(out_str[1]), - static_cast(out_str[2]), - static_cast(out_str[3]), - x, - static_cast(in_str[0]), - static_cast(in_str[1]), - static_cast(in_str[2]), - static_cast(in_str[3]), - reserveSpace, - static_cast(total_work), - static_cast(in_offset), - static_cast(out_offset), - static_cast(rsvsp_offset)); - } - else - { - std::string params; - - const std::string data_type = GetDataType(xDesc.GetType()); - const std::string READ_DAT_TYPE = - RD_BLCK == 1 ? data_type : data_type + std::to_string(RD_BLCK); - - params += " -DRD_BLCK=" + std::to_string(RD_BLCK) + " -DREAD_DAT_TYPE=" + READ_DAT_TYPE + - " -DREAD_BOOL_TYPE=" + - std::string(RD_BLCK == 4 ? "uint" - : RD_BLCK == 2 ? "ushort" - : "uchar"); - - if(xDesc.GetType() == miopenHalf) - params += " -DMIOPEN_USE_FP16=1"; - else - params += " -DMIOPEN_USE_FP32=1"; - - params += " -DRUN_FORWARD=1"; - - params += " -DUSE_RSVSP=" + std::to_string(static_cast(use_rsvsp)); - params += " -DUSE_MASK=" + std::to_string(static_cast(use_mask)); - - const std::vector vld{256, 1, 1}; - const std::vector vgd{wk_grp_num * 256, 1, 1}; - - handle.AddKernel(kernel_name, network_config, program_name, kernel_name, vld, vgd, params)( - pstates, - dropout, - amp_scale, - static_cast(in_len[1]), - static_cast(in_len[2]), - static_cast(in_len[3]), - static_cast(in_len[4]), - y, - static_cast(out_str[0]), - static_cast(out_str[1]), - static_cast(out_str[2]), - static_cast(out_str[3]), - x, - static_cast(in_str[0]), - static_cast(in_str[1]), - static_cast(in_str[2]), - static_cast(in_str[3]), - reserveSpace, - static_cast(total_work), - static_cast(in_offset), - static_cast(out_offset), - static_cast(rsvsp_offset)); - } - - if(miopen::CheckNumericsEnabled()) - { - std::cout << "Dropout forward output numerics check at dropout rate " << dropout - << std::endl; - miopen::checkNumericsOutput(handle, yDesc, y); - } -} - -void DropoutDescriptor::DropoutBackward(const Handle& handle, - const TensorDescriptor& noise_shape, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t reserveSpace, - size_t reserveSpaceSizeInBytes, - size_t in_offset, - size_t out_offset, - size_t rsvsp_offset) const -{ - if(dx == nullptr || dy == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(dxDesc.GetNumDims() != dyDesc.GetNumDims()) - { - MIOPEN_THROW("Input/Output dimension does not match"); - } - - if(dyDesc.GetNumDims() > 5) - { - MIOPEN_THROW("Only support 1D to 5D tensors"); - } - - if(dxDesc.GetElementSize() != dyDesc.GetElementSize()) - { - MIOPEN_THROW("Input/Output element size does not match"); - } - - if(dxDesc.GetElementSize() != noise_shape.GetElementSize() || - dxDesc.GetNumDims() != noise_shape.GetNumDims()) - { - MIOPEN_THROW("Only support dropout with regular noise shape currently"); - } - - if(dxDesc.GetType() != dyDesc.GetType()) - { - MIOPEN_THROW("Input/Output datatype does not match"); - } - - if(dropout < 0.0 || dropout > 1.0) - { - MIOPEN_THROW("Invalid dropout rate"); - } - - bool use_prng = reserveSpace == nullptr; - if(((!use_prng || use_mask) && - reserveSpaceSizeInBytes < dyDesc.GetElementSize() * sizeof(bool)) || - (use_mask && use_prng)) - { - MIOPEN_THROW("Insufficient reservespace size"); - } - - if(reserveSpaceSizeInBytes + dxDesc.GetElementSize() * GetTypeSize(dxDesc.GetType()) + - dyDesc.GetElementSize() * GetTypeSize(dyDesc.GetType()) > - handle.GetGlobalMemorySize()) - { - MIOPEN_THROW("Memory required by dropout backward configs exceeds GPU memory range."); - } - - if(miopen::CheckNumericsEnabled()) - { - std::cout << "Dropout backward input numerics check at dropout rate " << dropout - << std::endl; - miopen::checkNumericsInput(handle, dyDesc, dy); - } - - // support up to 5D tensor - std::vector in_len(5, 1); - std::vector in_str(5, 1); - std::vector out_len(5, 1); - std::vector out_str(5, 1); - - SquashPairedTensor(dxDesc.GetLengths(), - dxDesc.GetStrides(), - dyDesc.GetLengths(), - dyDesc.GetStrides(), - in_len, - in_str, - out_len, - out_str); - - size_t RD_BLCK = /* (in_len[4] % 4 == 0) ? 4 : */ (in_len[2] % 2 == 0) ? 2 : 1; - size_t total_work = (in_len[4] / RD_BLCK) * in_len[3] * in_len[2] * in_len[1] * in_len[0]; - - size_t max_wk_grp = use_prng ? std::min(size_t(MAX_PRNG_STATE), handle.GetImage3dMaxWidth()) - : size_t(MAX_WORKITEM_NUM); - size_t wk_grp_num = - std::min(max_wk_grp / 256, - ((in_len[4] * in_len[3] * in_len[2] * in_len[1] * in_len[0] + 255) / 256)); - - if(use_prng) - { - size_t states_num = stateSizeInBytes / sizeof(rocrand_state_xorwow); - if(states_num < wk_grp_num * 256) - { - MIOPEN_THROW("Insufficient state size for parallel PRNG"); - } - } - - std::string program_name = "MIOpenDropoutHIP.cpp"; - std::string kernel_name = "DropoutBW"; - - std::string network_config = - "bwd-" + std::string(dyDesc.GetType() == miopenHalf ? "fp16-" : "fp32-") + "-seed" + - std::to_string(seed) + "-rng" + std::to_string(rng_mode) + "-prng" + - std::to_string(static_cast(use_prng)) + "-evo" + - std::to_string(static_cast(state_evo)) + "-blk" + std::to_string(RD_BLCK) + "-wg" + - std::to_string(wk_grp_num) /* + "-noise" + std::to_string(noise_shape.GetLengths()[0]) */; - - // TODO: Add noise shape - // for(int i = 1; i < noise_shape.GetNumDims(); i++) - // network_config += "x" + std::to_string(noise_shape.GetLengths()[i]); - - auto&& kernels = handle.GetKernels(kernel_name, network_config); - - float amp_scale = float_equal(dropout, 1.0) ? 0 : 1 / (1 - dropout); - if(!kernels.empty()) - { - kernels.front()(pstates, - dropout, - amp_scale, - static_cast(in_len[1]), - static_cast(in_len[2]), - static_cast(in_len[3]), - static_cast(in_len[4]), - dy, - static_cast(out_str[0]), - static_cast(out_str[1]), - static_cast(out_str[2]), - static_cast(out_str[3]), - dx, - static_cast(in_str[0]), - static_cast(in_str[1]), - static_cast(in_str[2]), - static_cast(in_str[3]), - reserveSpace, - static_cast(total_work), - static_cast(in_offset), - static_cast(out_offset), - static_cast(rsvsp_offset)); - } - else - { - std::string params; - - const std::string data_type = GetDataType(dyDesc.GetType()); - const std::string READ_DAT_TYPE = - RD_BLCK == 1 ? data_type : data_type + std::to_string(RD_BLCK); - - params += " -DRD_BLCK=" + std::to_string(RD_BLCK) + " -DREAD_DAT_TYPE=" + READ_DAT_TYPE + - " -DREAD_BOOL_TYPE=" + - std::string(RD_BLCK == 4 ? "uint" - : RD_BLCK == 2 ? "ushort" - : "uchar"); - - if(use_prng) - { - params += " -DUSE_PRNG=1"; - } - - if(dyDesc.GetType() == miopenHalf) - params += " -DMIOPEN_USE_FP16=1"; - else - params += " -DMIOPEN_USE_FP32=1"; - - params += " -DRUN_FORWARD=0"; - - const std::vector vld{256, 1, 1}; - const std::vector vgd{wk_grp_num * 256, 1, 1}; - - handle.AddKernel(kernel_name, network_config, program_name, kernel_name, vld, vgd, params)( - pstates, - dropout, - amp_scale, - static_cast(in_len[1]), - static_cast(in_len[2]), - static_cast(in_len[3]), - static_cast(in_len[4]), - dy, - static_cast(out_str[0]), - static_cast(out_str[1]), - static_cast(out_str[2]), - static_cast(out_str[3]), - dx, - static_cast(in_str[0]), - static_cast(in_str[1]), - static_cast(in_str[2]), - static_cast(in_str[3]), - reserveSpace, - static_cast(total_work), - static_cast(in_offset), - static_cast(out_offset), - static_cast(rsvsp_offset)); - } - - if(miopen::CheckNumericsEnabled()) - { - std::cout << "Dropout backward output numerics check at dropout rate " << dropout - << std::endl; - miopen::checkNumericsOutput(handle, dxDesc, dx); - } -} - -} // namespace miopen diff --git a/src/ocl/rnnocl.cpp b/src/ocl/rnnocl.cpp index 3a07eeb0ff..e69de29bb2 100644 --- a/src/ocl/rnnocl.cpp +++ b/src/ocl/rnnocl.cpp @@ -1,6613 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_RNNFWD_EXP) -MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_RNNWRW_EXP) -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_RNNFWD_MS_DISPATCH) -MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_RNN_MS_STREAM_CNT) - -namespace miopen { - -namespace { - -#if MIOPEN_USE_ROCBLAS - -bool RNNForwardMSIsSupported([[maybe_unused]] const RNNDescriptor& desctiptor, - [[maybe_unused]] bool use_dropout) -{ -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - if(desctiptor.rnnMode == miopenLSTM && desctiptor.algoMode == miopenRNNdefault && - !use_dropout && desctiptor.nLayers > 1 && desctiptor.dirMode == miopenRNNunidirection && - desctiptor.inputMode != miopenRNNskip) - { - return true; - } -#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP - return false; -} - -bool RNNForwardMSIsFast(const int seqLen) -{ - if(env::enabled(MIOPEN_RNNFWD_EXP)) - return true; - - if(seqLen >= 32 && !env::disabled(MIOPEN_RNNFWD_EXP)) - return true; - return false; -} - -void checkGemmStatusAndLog(miopenStatus_t gemm_status) -{ - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } -} - -miopenStatus_t ReducAddBias(miopen::Handle& handle, - Data_t dw, - const Data_t workSpace, - const miopen::TensorDescriptor& dw_desc, - const miopen::TensorDescriptor& ws_desc, - size_t dw_bias_offset, - size_t ws_bias_offset, - Data_t red_workSpace, - size_t red_workSpace_size) -{ - if(ws_desc.GetLengths()[1] != 1) - { - - int algo = getReductionAlgo(); - - switch(algo) - { - case 0: { - float alpha0 = 0; - float alpha1 = 1; - float beta_t = 1; - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - dw_desc, - dw, - &alpha1, - ws_desc, - workSpace, - &beta_t, - dw_desc, - dw, - dw_bias_offset, - ws_bias_offset, - dw_bias_offset, - true); - } - break; - case 1: { - float alpha1 = 1; - float beta1 = 1; - - miopen::ReduceTensorDescriptor red_add{ - miopenReduceTensorOp_t::MIOPEN_REDUCE_TENSOR_ADD, - miopenDataType_t::miopenFloat, - miopenNanPropagation_t::MIOPEN_PROPAGATE_NAN, - miopenReduceTensorIndices_t::MIOPEN_REDUCE_TENSOR_NO_INDICES, - miopenIndicesType_t::MIOPEN_32BIT_INDICES}; - - Data_t srcA_with_offset = - static_cast(workSpace) + ws_bias_offset * GetTypeSize(dw_desc.GetType()); - - Data_t dstC_with_offset = - static_cast(dw) + dw_bias_offset * GetTypeSize(dw_desc.GetType()); - - // WA CK bug - Data_t red_workSpace_bugfix = red_workSpace; - if(dw_desc.GetType() == miopenDataType_t::miopenHalf) - { - if(std::align( - 4, red_workSpace_size - 4, red_workSpace_bugfix, red_workSpace_size) == - nullptr) - MIOPEN_THROW(miopenStatusInternalError, "failed alignment."); - } - - red_add.ReduceTensor(handle, - nullptr, - 0, - red_workSpace_bugfix, - red_workSpace_size, - &alpha1, - ws_desc, - srcA_with_offset, - &beta1, - dw_desc, - dstC_with_offset); - } - break; - case 2: - case 3: { - float alpha1 = 1.; - auto red_type = ws_desc.GetType(); - int m = 1, n = ws_desc.GetLengths()[2], k = ws_desc.GetLengths()[1]; - int lda = k, ldb = ws_desc.GetStrides()[1], ldc = n; - - const miopen::TensorDescriptor red_matrix{ - red_type, std::vector{1, 1, k}, std::vector{k, k, 1}}; - - SetTensor(handle, red_matrix, red_workSpace, &alpha1); - - float alpha = 1, beta = 1; - if(algo == 2) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - false, - m, - n, - k, - lda, - ldb, - ldc, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - alpha, // alpha - beta, // beta - red_type, - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - red_workSpace, - 0, - workSpace, - ws_bias_offset, - dw, - dw_bias_offset, - GemmBackend_t::rocblas); - checkGemmStatusAndLog(gemm_status); - } - else - { - if(dw_desc.GetType() != miopenDataType_t::miopenFloat) - MIOPEN_THROW(miopenStatusInternalError, "rocblas_sgemv wrong Type"); - - Data_t srcA_with_offset = - static_cast(workSpace) + ws_bias_offset * GetTypeSize(dw_desc.GetType()); - - Data_t dstY_with_offset = - static_cast(dw) + dw_bias_offset * GetTypeSize(dw_desc.GetType()); - - rocblas_sgemv(handle.rhandle().get(), - rocblas_operation::rocblas_operation_none, - n, - k, - &alpha, - static_cast(srcA_with_offset), - ldb, - static_cast(red_workSpace), - 1, - &beta, - static_cast(dstY_with_offset), - 1); - } - } - break; - default: break; - } - } - else - { - // nothing to reduce - // just copy data from workspace to dw - CopyTensor(handle, ws_desc, workSpace, dw_desc, dw, ws_bias_offset, dw_bias_offset); - } - - return miopenStatusSuccess; -} - -#endif // MIOPEN_USE_ROCBLAS - -} // namespace - -void RNNDescriptor::RNNForwardMS(Handle& handle, - std::vector& seq_array, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& yDesc, - Data_t y, - Data_t hy, - Data_t cy, - Data_t extra_space, - size_t extra_space_size, - miopenRNNFWDMode_t fwd_mode) const -{ -#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP - std::vector in_n; - int in_vec = xDesc.GetLengths()[1]; // input vector size - int out_vec = yDesc.GetLengths()[1]; // output vector size - - int seq_len = seq_array.size(); - int max_batch = seq_array[0]; - int hidden_size; - - std::tie(std::ignore, max_batch, hidden_size) = miopen::tien<3>(hxDesc.GetLengths()); - - const int extra_stream_cnt = env::value_or(MIOPEN_RNN_MS_STREAM_CNT, 4); - - MultiStreamController ms_controller{handle, extra_stream_cnt}; - - constexpr auto root_stream_id = MultiStreamController::rootStreamId; - ms_controller.ChangeActiveStream(root_stream_id); - - int total_batch_size = 0; - std::vector bacc_per_time(seq_len + 1); - - for(int i = 0; i < seq_len; i++) - { - bacc_per_time[i] = total_batch_size; - total_batch_size += seq_array[i]; - in_n.push_back(seq_array[i]); - } - bacc_per_time[seq_len] = total_batch_size; - - const struct - { - int batch; - } InBuff_strides{in_vec}; - - auto get_HxBuff_offset = [&](int layer_id) { - return layer_id * (static_cast(hidden_size) * max_batch); - }; - - int gates_cnt = 4; - int save_points_cnt = 6; - - struct WeightsBufferHelper - { - private: - auto hidden_xinput_size(int hidden_sz, int bidirect_mode) const - { - if(bidirect_mode == 0) - return hidden_sz; - MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); - } - - auto matrix_lin_layer_size(int input_vector_sz, int hidden_vec_sz, int gates) const - { - return (input_vector_sz + hidden_vec_sz) * hidden_vec_sz * gates; - } - size_t bias_start_offset(int input_vector_sz, - int hidden_vec_sz, - int layers_cnt, - int gates, - int bidirect_mode) const - { - if(bidirect_mode == 0) - { - return matrix_lin_layer_size(input_vector_sz, hidden_vec_sz, gates) + - static_cast(hidden_vec_sz + hidden_xinput_size(hidden_vec_sz, 0)) * - hidden_vec_sz * static_cast(layers_cnt - 1) * gates; - } - - MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); - } - - public: - WeightsBufferHelper( - int input_vector_sz, int hidden_vec_sz, int layers_cnt, int bias_mode, int gates) - : in_vec(input_vector_sz), - h_vec(hidden_vec_sz), - x_in_vec(hidden_xinput_size(hidden_vec_sz, 0)), - layers(layers_cnt), - gates_cnt(gates), - bias_cnt(bias_mode), - matrix_normal_start_off(matrix_lin_layer_size(input_vector_sz, hidden_vec_sz, gates)), - bias_start_off( - bias_start_offset(input_vector_sz, hidden_vec_sz, layers_cnt, gates, 0)) - { - } - - const int in_vec, h_vec; - const int x_in_vec; // for bidirect TODO - - const int layers; - const int gates_cnt; - const int - bias_cnt; // 0 - no bisa; 1 - one bias; 2 - separate bias for x_vec and for hidden_vec - private: - const size_t matrix_normal_start_off; - const size_t bias_start_off; - - public: - auto get_matrix_x_size(int layer_id) const - { - return (layer_id > 0 ? x_in_vec : in_vec) * h_vec; - } - auto get_matrix_h_size() const { return h_vec * h_vec; } - auto get_matrix_layer_size(int layer_id) const - { - return get_matrix_x_size(layer_id) * gates_cnt + get_matrix_h_size() * gates_cnt; - } - - size_t get_matrix_x_off(int layer_id) const - { - if(layer_id > 0) - { - return matrix_normal_start_off + - static_cast(layer_id - 1) * get_matrix_layer_size(layer_id); - } - else - { - return 0; - } - }; - - size_t get_matrix_h_off(int layer_id) const - { - if(layer_id > 0) - { - return get_matrix_x_off(layer_id) + - static_cast(h_vec * x_in_vec * gates_cnt); - } - else - { - return get_matrix_x_off(layer_id) + static_cast(h_vec * in_vec) * gates_cnt; - } - }; - - int bias_vector_size() const { return h_vec; } - int bias_vector_mul_gate() const { return bias_vector_size() * gates_cnt; } - int bias_stride() const { return bias_vector_mul_gate(); } - - size_t bias_relative_off(int layer_id, int bias_id) const - { - return static_cast(layer_id * bias_cnt + bias_id) * gates_cnt * h_vec; - } - - size_t get_bias_off(int layer_id, int bias_id) const - { - return bias_start_off + bias_relative_off(layer_id, bias_id); - } - - } WeiBuf(in_vec, hidden_size, nLayers, biasMode * 2, gates_cnt); - - struct ReserveBufferHelper - { - struct RBuffHelper - { - int element, save_point, batch; - size_t layer; - }; - - private: - auto Reserve_Buffer_strides(int save_point_sz, - int batches_per_layer, - int save_points, - int bidirect_mode = 0) const - { - const auto element_st = 1; - const auto save_point_st = element_st * save_point_sz; - const auto batch_st = save_point_st * save_points; - const auto layer_st = static_cast(batch_st) * batches_per_layer; - if(bidirect_mode == 0) - return RBuffHelper{element_st, save_point_st, batch_st, layer_st}; - MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); - } - - public: - enum save_point - { - F = 1, - I = 0, - G = 3, - O = 2, - St = 4, - Ht = 5 - }; - - ReserveBufferHelper(int hidden_vec_sz, - int save_point_sz, - int layers_cnt, - int batches_per_layer, - int save_points, - int gates_cnt) - : h_vec(hidden_vec_sz), - save_point_size(save_point_sz), - layers(layers_cnt), - batches(batches_per_layer), - save_points_cnt(save_points), - gates(gates_cnt), - strides(Reserve_Buffer_strides(save_point_sz, batches, save_points, 0)) - { - } - - const int h_vec; - const int save_point_size; // for bidirect TODO - - const int layers; - const int batches; - const int save_points_cnt; - const int gates; - const RBuffHelper strides; - - size_t layer_offset(int layer) const { return static_cast(layer) * strides.layer; } - auto layer_stride() const { return strides.layer; } - - auto gemm_write_size() const { return h_vec * gates; } - auto gemm_write_stride() const - { - return strides.batch; - } // save_point_size * save_points_cnt - - size_t gemm_write_relative_offset(int batch_id) const - { - return static_cast(gemm_write_stride()) * batch_id; - } - - size_t gemm_write_offset(int layer, int batch_id) const - { - return layer_offset(layer) + static_cast(gemm_write_stride()) * batch_id; - } - - auto ht_relative_offset() const { return save_point::Ht * save_point_size; } - - auto ct_relative_offset() const { return save_point::St * save_point_size; } - - auto get_gate_relative_offset(int gate_id) const { return gate_id * save_point_size; } - - size_t ht_offset(int layer_id, int batch_id) const - { - return layer_offset(layer_id) + gemm_write_relative_offset(batch_id) + - ht_relative_offset(); - } - - size_t extra_save_point_offset(int layer_id, int batch_id) const - { - return (static_cast(batches) * layers * gemm_write_stride()) // all data offset - + (static_cast(batches) * layer_id) * h_vec + - static_cast(batch_id * h_vec); - } - - } RBuff(hidden_size, hidden_size, nLayers, total_batch_size, save_points_cnt, gates_cnt); - - auto call_x_gemm = [&RBuff, - &WeiBuf, - &InBuff_strides, - &bacc_per_time, - &handle, - &xDesc, - extra_space, - x, - w, - hidden_size, - in_vec](int layer, int start_time, int time_cnt, float beta_t = 1) { - const auto start_b = bacc_per_time[start_time]; - const auto batch_sz = bacc_per_time[start_time + time_cnt] - start_b; - - const int m = batch_sz, n = RBuff.gemm_write_size(), k = layer > 0 ? hidden_size : in_vec; - const int lda = layer > 0 ? RBuff.gemm_write_stride() : InBuff_strides.batch, ldb = k, - ldc = RBuff.gemm_write_stride(); - - const miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - m, - n, - k, - lda, - ldb, - ldc, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - beta_t, // beta - xDesc.GetType(), - false}; - - const auto wx_off = WeiBuf.get_matrix_x_off(layer); - const auto out_offset = RBuff.gemm_write_offset(layer, start_b); - - const auto x_in_offset = layer > 0 ? RBuff.ht_offset(layer - 1, start_b) - : static_cast(start_b * InBuff_strides.batch); - const auto in_ptr = layer > 0 ? extra_space : x; - - const miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - in_ptr, - x_in_offset, - w, - wx_off, - extra_space, - out_offset, - GemmBackend_t::rocblas); - if(gemm_status != miopenStatusSuccess) - MIOPEN_THROW("GEMM execution failure"); - }; - - auto call_bias_add = [&RBuff, &WeiBuf, &handle, &wDesc, extra_space, w](int layer, - float beta_t = 0) { - float alpha0 = 1; - float alpha1 = 1; - const auto bias_stride = WeiBuf.bias_stride(); - - const auto bias_desc = - miopen::TensorDescriptor(wDesc.GetType(), - std::vector{1, 1, WeiBuf.bias_vector_mul_gate()}, - std::vector{bias_stride, bias_stride, 1}); - - const auto hidden_interim_desc = miopen::TensorDescriptor( - wDesc.GetType(), - std::vector{1, RBuff.batches, WeiBuf.bias_vector_mul_gate()}, - std::vector{ - RBuff.batches * RBuff.gemm_write_stride(), RBuff.gemm_write_stride(), 1}); - - const auto RB_layer_out_off = RBuff.layer_offset(layer); - const auto w_bias_layer_start_off = WeiBuf.get_bias_off(layer, 0); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hidden_interim_desc, - extra_space, // A - &alpha1, - bias_desc, - w, // B - &beta_t, - hidden_interim_desc, - extra_space, // C - RB_layer_out_off, // A offset - w_bias_layer_start_off, // B offset - RB_layer_out_off, // C offset - true); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hidden_interim_desc, - extra_space, - &alpha1, - bias_desc, - w, - &beta_t, - hidden_interim_desc, - extra_space, - RB_layer_out_off, - w_bias_layer_start_off + bias_stride, - RB_layer_out_off, - true); - }; - - auto call_hx_gemm = [&RBuff, - &WeiBuf, - &get_HxBuff_offset, - &bacc_per_time, - &in_n, - &handle, - &xDesc, - extra_space, - hx, - w, - hidden_size](int layer, int cur_time) { - const int m = in_n.at(cur_time), n = RBuff.gemm_write_size(), k = hidden_size; - - const int lda = (cur_time != 0) ? RBuff.gemm_write_stride() : hidden_size, - ldb = hidden_size, ldc = RBuff.gemm_write_stride(); - - const auto hx_ptr_offset = (cur_time == 0) - ? get_HxBuff_offset(layer) - : RBuff.ht_offset(layer, bacc_per_time[cur_time - 1]); - - if(cur_time == 0) - { - if(hx == nullptr) - return; - } - - const miopen::GemmDescriptor gemm_desc_hx = GemmDescriptor{false, - false, - true, - m, - n, - k, - lda, - ldb, - ldc, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc.GetType(), - false}; - - const auto RB_layer_save_points_off = - RBuff.gemm_write_offset(layer, bacc_per_time[cur_time]); - - const auto hx_ptr = cur_time > 0 ? extra_space : hx; - - const miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc_hx, - hx_ptr, - hx_ptr_offset, - w, - WeiBuf.get_matrix_h_off(layer), - extra_space, - RB_layer_save_points_off, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - MIOPEN_THROW("GEMM execution failure"); - }; - - auto call_hidden_state_update = [&RBuff, - &get_HxBuff_offset, - &bacc_per_time, - &in_n, - &handle, - &wDesc, - fwd_mode, - extra_space, - cx, - max_batch, - hidden_size](int layer_id, int time_id) { - auto RB_layer_save_points_off = - RBuff.layer_offset(layer_id) + RBuff.gemm_write_relative_offset(bacc_per_time[time_id]); - - auto is_seq_begin = time_id == 0; - - const int direction = 0; - const int cur_batch = in_n.at(time_id), use_batch = in_n.at(time_id); - - const int hy_stride = RBuff.gemm_write_stride(), wei_len = RBuff.gemm_write_size(), - wei_stride = RBuff.gemm_write_size(); - - const size_t cx_offset = get_HxBuff_offset(layer_id); - - const size_t i_offset = RB_layer_save_points_off + RBuff.get_gate_relative_offset(0), - f_offset = RB_layer_save_points_off + RBuff.get_gate_relative_offset(1), - o_offset = RB_layer_save_points_off + RBuff.get_gate_relative_offset(2), - c_offset = RB_layer_save_points_off + RBuff.get_gate_relative_offset(3); - - const size_t cell_offset = RB_layer_save_points_off + RBuff.ct_relative_offset(), - hidden_offset = RB_layer_save_points_off + RBuff.ht_relative_offset(); - - const size_t cell_offset_pre = - (time_id == 0) ? 0 - : RBuff.layer_offset(layer_id) + - RBuff.gemm_write_relative_offset(bacc_per_time[time_id - 1]) + - RBuff.ct_relative_offset(); - - const size_t activ_cell_offset = - RBuff.extra_save_point_offset(layer_id, bacc_per_time[time_id]); - - LSTMForwardHiddenStateUpdate(handle, - wDesc.GetType(), - fwd_mode == miopenRNNFWDMode_t::miopenRNNTraining ? false - : true, - is_seq_begin, - direction, - max_batch, - cur_batch, - use_batch, - - hidden_size, - hy_stride, - wei_len, - wei_stride, - cx, - cx_offset, - extra_space, - i_offset, - f_offset, - o_offset, - c_offset, - cell_offset, - cell_offset_pre, - activ_cell_offset, - hidden_offset); - }; - - auto call_hy_cy_update = [&RBuff, - &get_HxBuff_offset, - &bacc_per_time, - &in_n, - &handle, - &wDesc, - &ms_controller, - extra_space, - hy, - cy, - max_batch, - hidden_size, - seq_len](int layer_id, int extra_stream_id) { - if(hy != nullptr || (cy != nullptr)) - { - ms_controller.ChangeActiveStream(extra_stream_id); - - auto hcy_layer_offset = get_HxBuff_offset(layer_id); - - const std::vector hcy_src_stride{ - RBuff.layer_stride(), static_cast(RBuff.gemm_write_stride()), 1}; - const std::vector hcy_dst_stride{ - static_cast(hidden_size * max_batch), static_cast(hidden_size), 1}; - - if(in_n.at(0) < max_batch) - { - float beta = 0.; - const std::vector zero_set_size{1, - static_cast(max_batch - in_n.at(0)), - static_cast(hidden_size)}; - auto set_batch_offset = in_n.at(0) * hidden_size; - - auto set_desc = - miopen::TensorDescriptor(wDesc.GetType(), zero_set_size, hcy_dst_stride); - if(hy != nullptr) - { - SetTensor(handle, set_desc, hy, &beta, hcy_layer_offset + set_batch_offset); - } - if(cy != nullptr) - { - SetTensor(handle, set_desc, cy, &beta, hcy_layer_offset + set_batch_offset); - } - } - - for(int time_i = seq_len - 1; time_i >= 0; time_i--) - { - auto copy_batch = (time_i == seq_len - 1) ? in_n.at(time_i) - : in_n.at(time_i) - in_n.at(time_i + 1); - if(copy_batch > 0) - { - auto batch_id_relative = in_n.at(time_i) - copy_batch; - auto batch_id_abs = bacc_per_time[time_i] + batch_id_relative; - - auto hcy_batch_offset = batch_id_relative * hidden_size; - - auto src_batch_offset = RBuff.layer_offset(layer_id) + - RBuff.gemm_write_relative_offset(batch_id_abs); - - const std::vector hcy_copy_size{ - 1, static_cast(copy_batch), static_cast(hidden_size)}; - - auto src_desc = - miopen::TensorDescriptor(wDesc.GetType(), hcy_copy_size, hcy_src_stride); - auto dst_desc = - miopen::TensorDescriptor(wDesc.GetType(), hcy_copy_size, hcy_dst_stride); - - if(hy != nullptr) - { - CopyTensor(handle, - src_desc, - extra_space, - dst_desc, - hy, - src_batch_offset + RBuff.ht_relative_offset(), - hcy_layer_offset + hcy_batch_offset); - } - - if(cy != nullptr) - { - CopyTensor(handle, - src_desc, - extra_space, - dst_desc, - cy, - src_batch_offset + RBuff.ct_relative_offset(), - hcy_layer_offset + hcy_batch_offset); - } - } - } - } - }; - - if(seq_len == 0) - return; - - constexpr int try_chunks_cnt = 16; - const int time_chunk_sz = ((seq_len + try_chunks_cnt - 1) / try_chunks_cnt); - const int chunks_cnt = (seq_len + time_chunk_sz - 1) / time_chunk_sz; - - std::vector layer_inx_cur_time(nLayers, 0); - std::vector layer_hx_cur_time(nLayers, 0); - std::vector layer_upd_cur_time(nLayers, 0); - - std::vector> layer_chunk_end_event; - - layer_chunk_end_event.resize(nLayers); - for(int layer_id = 0; layer_id < nLayers; layer_id++) - { - layer_chunk_end_event[layer_id].resize(chunks_cnt); - for(int chunk_id = 0; chunk_id < chunks_cnt; chunk_id++) - layer_chunk_end_event[layer_id][chunk_id] = make_hip_fast_event(); - } - - auto call_inx_next_chunk_preload = [&](int layer_id) { - auto start_time = layer_inx_cur_time[layer_id]; - auto time_cnt = std::min(time_chunk_sz, seq_len - start_time); - - call_x_gemm(layer_id, start_time, time_cnt); - layer_inx_cur_time[layer_id] += time_cnt; - }; - - auto call_hx_next_gemm = [&](int layer_id) { - auto cur_time = layer_hx_cur_time[layer_id]; - if(cur_time < seq_len) - { - call_hx_gemm(layer_id, cur_time); - layer_hx_cur_time[layer_id]++; - } - }; - - auto call_next_hidden_state_update = [&](int layer_id) { - auto cur_time = layer_upd_cur_time[layer_id]; - if(cur_time < seq_len) - { - call_hidden_state_update(layer_id, cur_time); - layer_upd_cur_time[layer_id]++; - } - }; - - auto call_next_chunk_compute = [&call_next_hidden_state_update, - &call_hx_next_gemm, - &call_inx_next_chunk_preload, - &layer_upd_cur_time, - &layer_chunk_end_event, - &ms_controller, - time_chunk_sz, - seq_len](int layer_id, int stream_id) { - ms_controller.ChangeActiveStream(stream_id); - - const int chunk_id = layer_upd_cur_time[layer_id] / time_chunk_sz; - const int chunk_time = std::min(time_chunk_sz, seq_len - chunk_id * time_chunk_sz); - - if(!(layer_id == 0 && chunk_id == 1)) - { - call_inx_next_chunk_preload(layer_id); - } - - for(int time_id = 0; time_id < chunk_time; time_id++) - { - call_hx_next_gemm(layer_id); - call_next_hidden_state_update(layer_id); - } - ms_controller.RecordEvent(layer_chunk_end_event[layer_id][chunk_id].get(), stream_id); - }; - - auto sync_next_chunk_across_time = [&layer_chunk_end_event, - &ms_controller](int stream_id, int layer_id, int chunk_id) { - if(chunk_id > 0) - { - ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id][chunk_id - 1].get(), - stream_id); - } - }; - - auto sync_next_chunk_across_layers = - [&layer_chunk_end_event, &ms_controller](int stream_id, int layer_id, int chunk_id) { - if(layer_id > 0) - { - ms_controller.SetWaitEvent(layer_chunk_end_event[layer_id - 1][chunk_id].get(), - stream_id); - } - }; - - { // extra_space clean set 0 - const int fill_val = 0; - // if(biasMode == 0u) req - hipMemsetAsync(extra_space, fill_val, extra_space_size, handle.GetStream()); - } - - // stage 0 bias and input preload - // stage 0.2 first chunk compute and preload - { - ms_controller.AllStreamsWaitRoot(); - const auto first_layer_id = 0; - const auto stream_id = 1; // 1 - const auto extra_stream_id = 2; - - ms_controller.ChangeActiveStream(stream_id); - - if(biasMode != 0u) - call_bias_add(first_layer_id); - - call_next_chunk_compute(first_layer_id, stream_id); - - ms_controller.ChangeActiveStream(extra_stream_id); - - if(biasMode != 0u) - { - for(int layer_id = 1; layer_id < nLayers; layer_id++) - call_bias_add(layer_id); - } - - call_inx_next_chunk_preload(first_layer_id); - - // sync first to second stream - const miopen::HipEventPtr next_chunk_inx = make_hip_fast_event(); - ms_controller.RecordEvent(next_chunk_inx.get(), extra_stream_id); - ms_controller.SetWaitEvent(next_chunk_inx.get(), stream_id); - } - - auto spiral_dispatch = [&](int first_stream, int last_stream) { - auto layers_last_state = layer_upd_cur_time; - - auto update_last_state = [&layer_upd_cur_time, &layers_last_state]() { - std::copy( - layer_upd_cur_time.begin(), layer_upd_cur_time.end(), layers_last_state.begin()); - }; - - auto is_dispatchable = [&layers_last_state, seq_len, time_chunk_sz](int layer, - int dispatch_chunks) { - auto cur_seq_time = layers_last_state[layer]; - return seq_len <= cur_seq_time ? false - : layer == 0 - ? true - : layers_last_state[layer - 1] >= - std::min(cur_seq_time + (dispatch_chunks * time_chunk_sz), seq_len); - }; - - auto try_dispatch_next_chunk = - [&layer_upd_cur_time, - &sync_next_chunk_across_time, - &sync_next_chunk_across_layers, - &call_next_chunk_compute, - &is_dispatchable, - time_chunk_sz](int layer_id, int stream_id, int chunk_to_dispatch) -> bool { - if(!is_dispatchable(layer_id, chunk_to_dispatch)) - return false; - - auto chunk_id = layer_upd_cur_time[layer_id] / time_chunk_sz; - - sync_next_chunk_across_time(stream_id, layer_id, chunk_id); - sync_next_chunk_across_layers(stream_id, layer_id, chunk_id); - - call_next_chunk_compute(layer_id, stream_id); - return true; - }; - - auto try_dispatch_hy_cy_printout = [&layer_upd_cur_time, &call_hy_cy_update, seq_len]( - int layer_id, int stream_id) -> bool { - if(layer_upd_cur_time[layer_id] < seq_len) - return false; - - call_hy_cy_update(layer_id, stream_id); - return true; - }; - - const auto stream_round = last_stream - first_stream + 1; - bool nothing_to_dispatch = false; - while(!nothing_to_dispatch) - { - update_last_state(); - nothing_to_dispatch = true; - int stream_it = 0; - - for(int cur_layer = 0; cur_layer < nLayers; cur_layer++) - { - const auto dispatch_stream = first_stream + stream_it; - if(try_dispatch_next_chunk(cur_layer, dispatch_stream, 1)) - { - try_dispatch_hy_cy_printout(cur_layer, dispatch_stream); - stream_it = (stream_it + 1) % stream_round; - nothing_to_dispatch = false; - } - } - } - }; - - enum class DispatchStrategy - { - OldMasterSlave = 0, - Spiral = 1, - } dispatch_strategy = - static_cast(env::value_or( - MIOPEN_RNNFWD_MS_DISPATCH, - static_cast(DispatchStrategy::Spiral))); // what am I doing wrong? - - if(dispatch_strategy == DispatchStrategy::Spiral) - { - const auto first_stream = extra_stream_cnt > 0 ? 1 : 0; - const auto last_stream = extra_stream_cnt > 0 ? extra_stream_cnt : 0; - - spiral_dispatch(first_stream, last_stream); - } - else - { - std::vector layer_stream_id(nLayers, 2); - layer_stream_id[0] = 1; - - auto dispatch_next_chunk = [&layer_upd_cur_time, - sync_next_chunk_across_layers, - call_next_chunk_compute, - time_chunk_sz](int layer_id, int stream_id) { - auto chunk_id = layer_upd_cur_time[layer_id] / time_chunk_sz; - - sync_next_chunk_across_layers(stream_id, layer_id, chunk_id); - - call_next_chunk_compute(layer_id, stream_id); - }; - - for(int layer_id = 0; layer_id < nLayers; layer_id++) - { - const auto main_stream_id = 1; - ms_controller.ChangeActiveStream(main_stream_id); - - // check for wich stream was assigned this layer. If it differs from current - set - // stream wait event - if(layer_stream_id[layer_id] != main_stream_id) - { - auto chunk_id = layer_upd_cur_time[layer_id] / time_chunk_sz; - - sync_next_chunk_across_time(main_stream_id, layer_id, chunk_id); - - layer_stream_id[layer_id] = main_stream_id; - } - - const int start_chunk = layer_upd_cur_time[layer_id] / time_chunk_sz; - - const int extra_layer_max_chunks = - start_chunk + - ((layer_id + 1 < nLayers - 1) ? (chunks_cnt - start_chunk) / 2 : chunks_cnt); - - for(int chunk_id = start_chunk; chunk_id < chunks_cnt; chunk_id++) - { - dispatch_next_chunk(layer_id, layer_stream_id[layer_id]); - - int extra_compute_layer = layer_id + 1; - for(; extra_compute_layer < nLayers; extra_compute_layer++) - { - auto extra_chunk_id = layer_upd_cur_time[extra_compute_layer] / time_chunk_sz; - if(extra_chunk_id < extra_layer_max_chunks && extra_chunk_id <= chunk_id) - break; - } - - if(extra_compute_layer < nLayers) - dispatch_next_chunk(extra_compute_layer, layer_stream_id[extra_compute_layer]); - } - - // update hy, cy - call_hy_cy_update(layer_id, main_stream_id); - } - } - - ms_controller.ChangeActiveStream(root_stream_id); - - ms_controller.SetWaitEvent(layer_chunk_end_event[nLayers - 1][chunks_cnt - 1].get(), - root_stream_id); - - // output tensor copy - { - const std::vector y_copy_size{ - 1, static_cast(total_batch_size), static_cast(out_vec)}; - - const std::vector y_src_stride{ - RBuff.layer_stride(), static_cast(RBuff.gemm_write_stride()), 1}; - - const std::vector y_dst_stride{ - static_cast(out_vec * total_batch_size), static_cast(out_vec), 1}; - - auto src_desc = miopen::TensorDescriptor(wDesc.GetType(), y_copy_size, y_src_stride); - auto y_dst_desc = miopen::TensorDescriptor(wDesc.GetType(), y_copy_size, y_dst_stride); - - CopyTensor( - handle, src_desc, extra_space, y_dst_desc, y, RBuff.ht_offset(nLayers - 1, 0), 0); - } - - ms_controller.RootWaitToAllStreams(); -#else - (void)handle; - (void)seq_array; - (void)xDesc; - (void)x; - (void)hxDesc; - (void)hx; - (void)cx; - (void)wDesc; - (void)w; - (void)yDesc; - (void)y; - (void)hy; - (void)cy; - - MIOPEN_THROW("GEMM is not supported"); -#endif -} - -// Assuming sequence length is set to > 0 otherwise throw exception. -void RNNDescriptor::RNNForwardInference(Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize) const -{ - if(x == nullptr || w == nullptr || y == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(hxDesc.GetNumDims() != cxDesc.GetNumDims() || hxDesc.GetNumDims() != hyDesc.GetNumDims() || - hxDesc.GetNumDims() != cyDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(workSpaceSize < GetWorkspaceSize(handle, seqLen, xDesc)) - { - MIOPEN_THROW("Workspace is required"); - } - -#if MIOPEN_BACKEND_HIP - RnnHipAutoProfiler kernel_profiler{handle}; - - try - { -#endif - - if(paddingMode == miopenRNNIONotPadded) - { - return RNNForwardInferencePacked(handle, - seqLen, - xDesc, - x, - hxDesc, - hx, - cxDesc, - cx, - wDesc, - w, - yDesc, - y, - hyDesc, - hy, - cyDesc, - cy, - workSpace, - workSpaceSize); - } - else - { - Data_t packedXIn = workSpace; - size_t packedXInSize, packedYOutSize; - std::tie(packedXInSize, packedYOutSize) = - RNNTensorPaddingConverter::GetTempPackedBuffersSpace(*this, xDesc); - - Data_t packedYOut = - static_cast(reinterpret_cast(workSpace) + packedXInSize); - - auto shifted_workSpace = static_cast(reinterpret_cast(workSpace) + - (packedXInSize + packedYOutSize)); - auto shifted_workSpace_size = workSpaceSize - (packedXInSize + packedYOutSize); - // std::vector packed_desc; - // std::vector packed_desc_ptrs; - // RNNTensorPaddingConverter::CreatePackedDescriptor() - // for future developments: as long as we don't use strides from xDesc and yDesc - // we ignoring conversion of this descriptors. - std::vector in_n(seqLen); - - for(int i = 0; i < seqLen; i++) - { - int batchval, batchvalout; - std::tie(batchval, std::ignore) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, std::ignore) = miopen::tien<2>(yDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - in_n[i] = batchval; - } - - RNNTensorPaddingConverter::ConvertTensorData( - handle, xDesc[0], in_n, x, packedXIn, true); - - RNNDescriptor packedRnnDesc(*this); - packedRnnDesc.SetPaddingmode(miopenRNNIONotPadded); - - packedRnnDesc.RNNForwardInferencePacked(handle, - seqLen, - xDesc, - packedXIn, - hxDesc, - hx, - cxDesc, - cx, - wDesc, - w, - yDesc, - packedYOut, - hyDesc, - hy, - cyDesc, - cy, - shifted_workSpace, - shifted_workSpace_size); - - RNNTensorPaddingConverter::ConvertTensorData( - handle, yDesc[0], in_n, packedYOut, y, false); - } - -#if MIOPEN_BACKEND_HIP - } - catch(...) - { - kernel_profiler.abortProfiling(); - throw; - } -#endif -} -void RNNDescriptor::RNNForwardInferencePacked(Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize) const -{ - (void)cyDesc; - (void)hxDesc; - (void)cxDesc; - -#if MIOPEN_USE_GEMM - - float ctime = 0.; - // reset kernel timer - profileRNNkernels(handle, 0, ctime); - - std::vector in_n; - int in_h = xDesc[0].GetLengths()[1]; // input vector size - int hy_d = hyDesc.GetLengths()[0]; // biNumLayers - int hy_n = hyDesc.GetLengths()[1]; // max batch size - int hy_h = hyDesc.GetLengths()[2]; // hidden size - int out_h = yDesc[0].GetLengths()[1]; // output vector size - int bi = dirMode != 0u ? 2 : 1; - - if(in_h <= 0 || hy_h <= 0 || hy_n <= 0 || hy_d <= 0 || out_h <= 0 || seqLen <= 0) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(out_h != (bi * hy_h)) - { - MIOPEN_THROW(miopenStatusBadParm, "Output size doesn't match hidden state size!"); - } - - if(inputMode == miopenRNNskip) - { - if(in_h != hy_h) - { - MIOPEN_THROW(miopenStatusBadParm, - "The input tensor size must equal to the hidden " - "state size of the network in SKIP_INPUT mode!"); - } - in_h = 0; - } - - int batch_n = 0; - for(int i = 0; i < seqLen; i++) - { - int batchval, inputvec, batchvalout, outputvec; - std::tie(batchval, inputvec) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, outputvec) = miopen::tien<2>(yDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - if(i == 0) - { - if(batchval <= 0) - { - MIOPEN_THROW(miopenStatusBadParm, "Input batch is ZERO!"); - } - } - else - { - if(batchval > in_n.back() || batchval < 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "Incorrect input batch size at time " + std::to_string(i) + - "! Batch size must not ascend!"); - } - } - in_n.push_back(batchval); - batch_n += batchval; - } - // input check end - - if(RNNForwardMSIsSupported(*this, false) && RNNForwardMSIsFast(seqLen)) - { - return RNNForwardMS(handle, - in_n, - xDesc[0], - x, - hxDesc, - hx, - cx, - wDesc, - w, - yDesc[0], - y, - hy, - cy, - workSpace, - workSpaceSize, - miopenRNNFWDMode_t::miopenRNNInference); - } - - int in_stride = xDesc[0].GetLengths()[1]; - int hy_stride = hy_h * bi * static_cast(workspaceScale); - int out_stride = out_h; - int wei_stride = hy_h * bi * static_cast(nHiddenTensorsPerLayer); - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - size_t wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (nLayers - 1)) * wei_stride; - size_t offset; - float alpha0, alpha1, beta_t; - float alpha = 1, beta = 0; - - std::vector sp_size(3, 1), sp_stride(3, 1), w_size(3, 1), w_stride(3, 1), x_size(3, 1), - x_stride(3, 1), y_size(3, 1), y_stride(3, 1), hx_size(3, 1), hx_stride(3, 1); - miopen::TensorDescriptor sp_desc, w_desc, x_desc, y_desc, hx_desc; - - sp_size[2] = workSpaceSize / GetTypeSize(wDesc.GetType()); - sp_stride[0] = sp_size[2]; - sp_stride[1] = sp_size[2]; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - SetTensor(handle, sp_desc, workSpace, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - sp_stride[0] = batch_n * hy_stride; - sp_stride[1] = hy_stride; - sp_size[2] = 1; - w_stride[0] = wei_stride; - w_stride[1] = wei_stride; - x_stride[0] = batch_n * in_stride; - x_stride[1] = in_stride; - y_stride[0] = batch_n * out_stride; - y_stride[1] = out_stride; - if(hy != nullptr || (rnnMode == miopenLSTM && cy != nullptr)) - { - hx_size[2] = hy_d * hy_n * hy_h; - hx_stride[0] = hx_size[2]; - hx_stride[1] = hx_size[2]; - hx_desc = miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - if(hy != nullptr) - { - SetTensor(handle, hx_desc, hy, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - if(rnnMode == miopenLSTM && cy != nullptr) - { - SetTensor(handle, hx_desc, cy, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - hx_stride[0] = in_n.at(0) * uni_stride; - hx_stride[1] = uni_stride; - - int wei_shift, prelayer_shift; - int wei_len = 0; - int hid_off = 0; - - switch(rnnMode) - { - case miopenRNNRELU: - case miopenRNNTANH: - // printf("run rnn gpu inference \n"); - wei_len = hy_h; - hid_off = 0; - break; - case miopenLSTM: - // printf("run lstm gpu inference \n"); - wei_len = hy_h * 4; - hid_off = bi * hy_h * 5; - break; - case miopenGRU: - // printf("run gru gpu inference \n"); - wei_len = hy_h * 3; - hid_off = bi * hy_h * 3; - break; - } - - ActivationDescriptor tanhDesc, sigDesc, activDesc; - sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; - tanhDesc = {miopenActivationTANH, 1, 1, 1}; - if(rnnMode == miopenRNNRELU) - { - activDesc = {miopenActivationRELU, 1, 0, 1}; - } - else if(rnnMode == miopenRNNTANH) - { - activDesc = {miopenActivationTANH, 1, 1, 1}; - } - - for(int li = 0; li < nLayers; li++) - { - int hid_shift = li * batch_n * hy_stride; - int hx_shift = li * hy_n * bi_stride; - int wei_shift_bias_temp = static_cast(wei_shift_bias) + li * 2 * wei_stride; - - // from input - if(li == 0) - { - if(inputMode == miopenRNNskip) - { - x_size[1] = batch_n; - x_size[2] = hy_h; - sp_size[1] = batch_n; - sp_size[2] = hy_h; - x_desc = miopen::TensorDescriptor(wDesc.GetType(), x_size, x_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - for(int gi = 0; gi < nHiddenTensorsPerLayer * bi; gi++) - { - CopyTensor(handle, x_desc, x, sp_desc, workSpace, 0, gi * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - false, - true, - batch_n, - wei_len * bi, - in_h, - in_stride, - in_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; // RNN does not support determinism - - miopenStatus_t gemm_status = CallGemm( - handle, gemm_desc, x, 0, w, 0, workSpace, hid_shift, GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - prelayer_shift = (li - 1) * batch_n * hy_stride + hid_off; - - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - batch_n, - wei_len * bi, - hy_h * bi, - hy_stride, - bi_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - workSpace, - prelayer_shift, - w, - wei_shift, - workSpace, - hid_shift, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(biasMode != 0u) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - w_size[1] = 1; - w_size[2] = wei_stride; - sp_size[1] = batch_n; - sp_size[2] = wei_stride; - w_desc = miopen::TensorDescriptor(wDesc.GetType(), w_size, w_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - workSpace, - hid_shift, - wei_shift_bias_temp, - hid_shift, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(rnnMode == miopenGRU) - { - sp_size[1] = batch_n; - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - alpha0 = 0; - alpha1 = 0; - beta_t = 0; - for(int bs = 0; bs < bi; bs++) - { - CopyTensor(handle, - sp_desc, - workSpace, - sp_desc, - workSpace, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + hid_off + bs * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + bs * wei_len + 2 * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - if(biasMode != 0u) - { - wei_shift_bias_temp += wei_stride; - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - if(hx != nullptr) - { - sp_size[1] = batch_n; - sp_size[2] = wei_stride; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - workSpace, - hid_shift, - wei_shift_bias_temp, - hid_shift, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else - { - if(batch_n - in_n.at(0) > 0) - { - sp_size[1] = batch_n - in_n.at(0); - sp_size[2] = wei_len; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - w_size[1] = 1; - w_size[2] = wei_len; - w_desc = miopen::TensorDescriptor(wDesc.GetType(), w_size, w_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - workSpace, - hid_shift + in_n.at(0) * hy_stride, - wei_shift_bias_temp, - hid_shift + in_n.at(0) * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(dirMode != 0u) - { - if(in_n.at(0) == in_n.at(seqLen - 1)) - { - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - workSpace, - hid_shift + wei_len, - wei_shift_bias_temp + wei_len, - hid_shift + wei_len, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else - { - int cur_batch = 0; - for(int ti = 0; ti < seqLen; ti++) - { - if(ti != (seqLen - 1)) - { - offset = hid_shift + cur_batch * hy_stride; - - sp_size[1] = in_n.at(ti + 1); - sp_size[2] = wei_len; - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - workSpace, - offset + wei_len, - wei_shift_bias_temp + wei_len, - offset + wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - cur_batch += in_n.at(ti); - } - } - } - } - } - } - - // from hidden state - int bacc = 0; - int baccbi = batch_n; - for(int ti = 0; ti < seqLen; ti++) - { - baccbi -= in_n.at(seqLen - 1 - ti); - wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int pretime_shift = 0; - int use_time = 0; - - for(int ri = 0; ri < bi; ri++) - { - int cur_time = ri == 0 ? ti : seqLen - 1 - ti; - int cur_batch = ri == 0 ? bacc : baccbi; - offset = hid_shift + cur_batch * hy_stride; - if(ti > 0) - { - pretime_shift = - ri == 0 ? hid_shift + (bacc - in_n.at(ti - 1)) * hy_stride - : hid_shift + (baccbi + in_n.at(seqLen - 1 - ti)) * hy_stride; - use_time = ri == 0 ? ti : seqLen - ti; - } - - if(in_n.at(cur_time) > 0) - { - if(ti == 0) - { - if(hx != nullptr) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - in_n.at(cur_time), - wei_len, - hy_h, - uni_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - hx, - hx_shift + ri * hy_n * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - workSpace, - static_cast(offset) + ri * wei_len, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - false, - true, - (in_n.at(cur_time) - in_n.at(use_time)), - wei_len, - hy_h, - uni_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - hx, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - workSpace, - static_cast(offset) + ri * wei_len + - in_n.at(use_time) * hy_stride, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(in_n.at(use_time) > 0) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - in_n.at(use_time), - wei_len, - hy_h, - hy_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - pretime_shift + hid_off + ri * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - workSpace, - static_cast(offset) + ri * wei_len, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - // update hidden status - sp_size[1] = in_n.at(cur_time); - if(rnnMode == miopenRNNRELU || rnnMode == miopenRNNTANH) - { - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - activDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenLSTM) - { - if(algoMode == miopenRNNdefault) - { - LSTMForwardHiddenStateUpdate( - handle, - wDesc.GetType(), - true, - ti == 0, - ri, - in_n.at(0), - in_n.at(cur_time), - in_n.at(use_time), - hy_h, - hy_stride, - wei_len, - wei_stride, - cx, - hx_shift + ri * hy_n * hy_h, - workSpace, - offset + static_cast(ri) * wei_len, - offset + hy_h + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - 0, - offset + hid_off + static_cast(ri) * hy_h); - - // Update time - profileRNNkernels(handle, 1, ctime); - continue; - } - - // active gate i, f, o - sp_size[2] = hy_h * 3; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - sigDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // active gate c - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - tanhDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update cell state - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(ti == 0) - { - if(cx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - workSpace, - offset + hy_h + static_cast(ri) * wei_len, - hx_shift + ri * hy_n * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && cx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - workSpace, - offset + hy_h + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time)) * hy_stride, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - } - - if(in_n.at(use_time) > 0) - { - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(use_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + hy_h + static_cast(ri) * wei_len, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(cur_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - } - } - - // active cell state - tanhDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update hidden state - beta_t = 0; - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hid_off + static_cast(ri) * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenGRU) - { - // active z, r gate - sp_size[2] = 2 * hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - sigDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // calculate c gate - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + hy_h + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hid_off + static_cast(ri) * hy_h, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // active c gate - tanhDesc.Forward(handle, - &alpha, - sp_desc, - workSpace, - &beta, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // calculate hidden state - alpha0 = -1; - alpha1 = 1; - beta_t = 0; - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hid_off + static_cast(ri) * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - if(ti == 0) - { - if(hx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - hx, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - hx_shift + ri * hy_n * hy_h, - offset + hid_off + static_cast(ri) * hy_h, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - hx, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time)) * hy_stride, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + hid_off + static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - } - - if(in_n.at(use_time) > 0) - { - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(use_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len, - pretime_shift + hid_off + ri * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - } - } - - bacc += in_n.at(ti); - } - - // update hy, cy - if(hy != nullptr || (rnnMode == miopenLSTM && cy != nullptr)) - { - hx_size[2] = hy_h; - sp_size[2] = hy_h; - - bacc = batch_n; - baccbi = 0; - for(int ti = seqLen - 1; ti >= 0; ti--) - { - bacc -= in_n.at(ti); - for(int ri = 0; ri < bi; ri++) - { - int cur_time = ri == 0 ? ti : seqLen - 1 - ti; - int cur_batch = ri == 0 ? bacc : baccbi; - int use_batch = 0; - - if(ti < seqLen - 1) - { - int use_time = ri == 0 ? ti + 1 : seqLen - 2 - ti; - use_batch = in_n.at(use_time); - } - - if(in_n.at(cur_time) > use_batch) - { - offset = hid_shift + cur_batch * hy_stride; - - sp_size[1] = in_n.at(cur_time) - use_batch; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - hx_size[1] = sp_size[1]; - hx_desc = miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - if(hy != nullptr) - { - CopyTensor(handle, - sp_desc, - workSpace, - hx_desc, - hy, - static_cast(offset) + hid_off + ri * hy_h + - use_batch * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(rnnMode == miopenLSTM && cy != nullptr) - { - CopyTensor(handle, - sp_desc, - workSpace, - hx_desc, - cy, - static_cast(offset) + bi * wei_len + ri * hy_h + - use_batch * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - baccbi += in_n.at(seqLen - 1 - ti); - } - } - } - - // output - prelayer_shift = (static_cast(nLayers) - 1) * batch_n * hy_stride + hid_off; - - sp_size[1] = batch_n; - sp_size[2] = hy_h * bi; - y_size[1] = batch_n; - y_size[2] = out_h; - y_desc = miopen::TensorDescriptor(wDesc.GetType(), y_size, y_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - CopyTensor(handle, sp_desc, workSpace, y_desc, y, prelayer_shift, 0); - // Update time - profileRNNkernels(handle, 2, ctime); - -#else - (void)hx; - (void)cx; - (void)handle; - (void)seqLen; - (void)xDesc; - (void)x; - (void)w; - (void)y; - (void)hyDesc; - (void)hy; - (void)yDesc; - (void)wDesc; - (void)workSpaceSize; - (void)workSpace; - MIOPEN_THROW("GEMM is not supported"); -#endif -} - -void RNNDescriptor::RNNForwardTraining(Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const -{ - - if(x == nullptr || w == nullptr || y == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(hxDesc.GetNumDims() != cxDesc.GetNumDims() || hxDesc.GetNumDims() != hyDesc.GetNumDims() || - hxDesc.GetNumDims() != cyDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(reserveSpaceSize < GetReserveSize(handle, seqLen, xDesc)) - { - MIOPEN_THROW("Reservespace is required"); - } - - if(workSpaceSize < GetWorkspaceSize(handle, seqLen, xDesc)) - { - MIOPEN_THROW("Workspace is required"); - } - -#if MIOPEN_BACKEND_HIP - RnnHipAutoProfiler kernel_profiler{handle}; - try - { -#endif - - if(paddingMode == miopenRNNIONotPadded) - { - return RNNForwardTrainingPackedTensors(handle, - seqLen, - xDesc, - x, - hxDesc, - hx, - cxDesc, - cx, - wDesc, - w, - yDesc, - y, - hyDesc, - hy, - cyDesc, - cy, - reserveSpace, - reserveSpaceSize); - } - else - { - Data_t packedXIn = workSpace; - size_t packedXInSize; - std::tie(packedXInSize, std::ignore) = - RNNTensorPaddingConverter::GetTempPackedBuffersSpace(*this, xDesc); - - Data_t packedYOut = - static_cast(reinterpret_cast(workSpace) + packedXInSize); - - // std::vector packed_desc; - // std::vector packed_desc_ptrs; - // RNNTensorPaddingConverter::CreatePackedDescriptor() - // for future developments: as long as we don't use strides from xDesc and yDesc - // we ignoring conversion of this descriptors. - std::vector in_n(seqLen); - - for(int i = 0; i < seqLen; i++) - { - int batchval, batchvalout; - std::tie(batchval, std::ignore) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, std::ignore) = miopen::tien<2>(yDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - in_n[i] = batchval; - } - - RNNTensorPaddingConverter::ConvertTensorData( - handle, xDesc[0], in_n, x, packedXIn, true); - - RNNDescriptor packedRnnDesc(*this); - packedRnnDesc.SetPaddingmode(miopenRNNIONotPadded); - - packedRnnDesc.RNNForwardTrainingPackedTensors(handle, - seqLen, - xDesc, - packedXIn, - hxDesc, - hx, - cxDesc, - cx, - wDesc, - w, - yDesc, - packedYOut, - hyDesc, - hy, - cyDesc, - cy, - reserveSpace, - reserveSpaceSize); - - RNNTensorPaddingConverter::ConvertTensorData( - handle, yDesc[0], in_n, packedYOut, y, false); - } - -#if MIOPEN_BACKEND_HIP - } - catch(...) - { - kernel_profiler.abortProfiling(); - throw; - } - -#endif -}; - -void RNNDescriptor::RNNForwardTrainingPackedTensors( - Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - const TensorDescriptor& wDesc, - ConstData_t w, - c_array_view yDesc, - Data_t y, - const TensorDescriptor& hyDesc, - Data_t hy, - const TensorDescriptor& cyDesc, - Data_t cy, - Data_t reserveSpace, - size_t reserveSpaceSize) const -{ - (void)cxDesc; - (void)cyDesc; -#if MIOPEN_USE_GEMM - - // OCL legacy - float ctime = 0.; - // reset kernel timer - profileRNNkernels(handle, 0, ctime); - - int in_h = xDesc[0].GetLengths()[1]; // input vector size - int hy_d = hyDesc.GetLengths()[0]; // biNumLayers - int hy_n = hyDesc.GetLengths()[1]; // max batch size - int hy_h = hyDesc.GetLengths()[2]; // hidden size - int out_h = yDesc[0].GetLengths()[1]; // output vector size - int bi = dirMode != 0u ? 2 : 1; - - if(in_h <= 0 || hy_h <= 0 || hy_n <= 0 || hy_d <= 0 || out_h <= 0 || seqLen <= 0) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - if(out_h != (bi * hy_h)) - { - MIOPEN_THROW(miopenStatusBadParm, "Output size doesn't match hidden state size!"); - } - - if(inputMode == miopenRNNskip) - { - if(in_h != hy_h) - { - MIOPEN_THROW(miopenStatusBadParm, - "The input tensor size must equal to the hidden " - "state size of the network in SKIP_INPUT mode!"); - } - in_h = 0; - } - - int batch_n = 0; - std::vector in_n; - for(int i = 0; i < seqLen; i++) - { - int batchval, batchvalout; - std::tie(batchval, std::ignore) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, std::ignore) = miopen::tien<2>(yDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - if(i == 0) - { - if(batchval <= 0) - { - MIOPEN_THROW(miopenStatusBadParm, "Input batch is ZERO!"); - } - } - else - { - if(batchval > in_n.back() || batchval < 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "Incorrect input batch size at time " + std::to_string(i) + - "! Batch size must not ascend!"); - } - } - in_n.push_back(batchval); - batch_n += batchval; - } - // input check end - bool use_dropout = !float_equal(miopen::deref(dropoutDesc).dropout, 0); - - if(RNNForwardMSIsSupported(*this, false) && RNNForwardMSIsFast(seqLen)) - { - return RNNForwardMS(handle, - in_n, - xDesc[0], - x, - hxDesc, - hx, - cx, - wDesc, - w, - yDesc[0], - y, - hy, - cy, - reserveSpace, - reserveSpaceSize, - miopenRNNFWDMode_t::miopenRNNTraining); - } - - int in_stride = xDesc[0].GetLengths()[1]; - int hy_stride = hy_h * bi * static_cast(workspaceScale); - int out_stride = out_h; - int wei_stride = hy_h * bi * static_cast(nHiddenTensorsPerLayer); - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - size_t wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (nLayers - 1)) * wei_stride; - size_t offset; - float alpha0, alpha1, beta_t; - float alpha = 1, beta = 0; - - std::vector sp_size(3, 1), sp_stride(3, 1), w_size(3, 1), w_stride(3, 1), x_size(3, 1), - x_stride(3, 1), y_size(3, 1), y_stride(3, 1), hx_size(3, 1), hx_stride(3, 1); - miopen::TensorDescriptor sp_desc, w_desc, x_desc, y_desc, hx_desc; - - sp_size[2] = reserveSpaceSize / GetTypeSize(wDesc.GetType()); - sp_stride[0] = sp_size[2]; - sp_stride[1] = sp_size[2]; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - SetTensor(handle, sp_desc, reserveSpace, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - sp_stride[0] = batch_n * hy_stride; - sp_stride[1] = hy_stride; - sp_size[2] = 1; - w_stride[0] = wei_stride; - w_stride[1] = wei_stride; - x_stride[0] = batch_n * in_stride; - x_stride[1] = in_stride; - y_stride[0] = batch_n * out_stride; - y_stride[1] = out_stride; - if(hy != nullptr || (rnnMode == miopenLSTM && cy != nullptr)) - { - hx_size[2] = hy_d * hy_n * hy_h; - hx_stride[0] = hx_size[2]; - hx_stride[1] = hx_size[2]; - hx_desc = miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - if(hy != nullptr) - { - SetTensor(handle, hx_desc, hy, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - if(rnnMode == miopenLSTM && cy != nullptr) - { - SetTensor(handle, hx_desc, cy, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - hx_stride[0] = in_n.at(0) * uni_stride; - hx_stride[1] = uni_stride; - - int wei_shift, prelayer_shift; - int wei_len = 0; - int hid_off = 0; - - switch(rnnMode) - { - case miopenRNNRELU: - case miopenRNNTANH: - // printf("run rnn gpu fwd \n"); - wei_len = hy_h; - hid_off = static_cast(nLayers) * batch_n * hy_stride; - break; - case miopenLSTM: - // printf("run lstm gpu fwd \n"); - wei_len = hy_h * 4; - hid_off = bi * hy_h * 5; - break; - case miopenGRU: - // printf("run gru gpu fwd \n"); - wei_len = hy_h * 3; - hid_off = bi * hy_h * 3; - break; - } - - ActivationDescriptor tanhDesc, sigDesc, activDesc; - sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; - tanhDesc = {miopenActivationTANH, 1, 1, 1}; - if(rnnMode == miopenRNNRELU) - { - activDesc = {miopenActivationRELU, 1, 0, 1}; - } - else if(rnnMode == miopenRNNTANH) - { - activDesc = {miopenActivationTANH, 1, 1, 1}; - } - - for(int li = 0; li < nLayers; li++) - { - int hid_shift = li * batch_n * hy_stride; - int hx_shift = li * hy_n * bi_stride; - int wei_shift_bias_temp = static_cast(wei_shift_bias) + li * 2 * wei_stride; - - // from input - if(li == 0) - { - if(inputMode == miopenRNNskip) - { - x_size[1] = batch_n; - x_size[2] = hy_h; - sp_size[1] = batch_n; - sp_size[2] = hy_h; - x_desc = miopen::TensorDescriptor(wDesc.GetType(), x_size, x_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - for(int gi = 0; gi < nHiddenTensorsPerLayer * bi; gi++) - { - CopyTensor(handle, x_desc, x, sp_desc, reserveSpace, 0, gi * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - batch_n, - wei_len * bi, - in_h, - in_stride, - in_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = CallGemm( - handle, gemm_desc, x, 0, w, 0, reserveSpace, hid_shift, GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - prelayer_shift = (li - 1) * batch_n * hy_stride + hid_off; - - if(use_dropout) - { - std::vector drop_size(2), drop_in_str(2, 1), drop_out_str(2, 1); - drop_size[0] = batch_n; - drop_size[1] = hy_h * bi; - drop_in_str[0] = hy_stride; - drop_out_str[0] = hy_h * bi; - - auto drop_in_desc = - miopen::TensorDescriptor(wDesc.GetType(), drop_size, drop_in_str); - auto drop_out_desc = - miopen::TensorDescriptor(wDesc.GetType(), drop_size, drop_out_str); - - size_t drop_rsv_size = drop_out_desc.GetElementSize(); - size_t drop_rsv_start = - algoMode == miopenRNNdefault && rnnMode == miopenLSTM - ? nLayers * batch_n * hy_stride + nLayers * batch_n * hy_h * bi - : 2 * nLayers * batch_n * hy_stride; - - size_t drop_in_offset = prelayer_shift; - size_t drop_out_offset = - drop_rsv_start + (static_cast(li) - 1) * batch_n * hy_h * bi; - size_t drop_rsv_offset = (drop_rsv_start + (nLayers - 1) * batch_n * hy_h * bi) * - (wDesc.GetType() == miopenFloat ? 4 : 2) + - (li - 1) * drop_rsv_size; - - miopen::deref(dropoutDesc) - .DropoutForward(handle, - drop_in_desc, - drop_in_desc, - reserveSpace, - drop_out_desc, - reserveSpace, - reserveSpace, - drop_rsv_size, - drop_in_offset, - drop_out_offset, - drop_rsv_offset); - // Update time - profileRNNkernels(handle, 1, ctime); - prelayer_shift = drop_out_offset; - } - - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - batch_n, - wei_len * bi, - hy_h * bi, - use_dropout ? hy_h * bi : hy_stride, - bi_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - reserveSpace, - prelayer_shift, - w, - wei_shift, - reserveSpace, - hid_shift, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(biasMode != 0u) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - w_size[1] = 1; - w_size[2] = wei_stride; - sp_size[1] = batch_n; - sp_size[2] = wei_stride; - w_desc = miopen::TensorDescriptor(wDesc.GetType(), w_size, w_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - reserveSpace, - hid_shift, - wei_shift_bias_temp, - hid_shift, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(rnnMode == miopenGRU) - { - sp_size[1] = batch_n; - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - alpha0 = 0; - alpha1 = 0; - beta_t = 0; - for(int bs = 0; bs < bi; bs++) - { - CopyTensor(handle, - sp_desc, - reserveSpace, - sp_desc, - reserveSpace, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + hid_off + bs * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + bs * wei_len + 2 * hy_h, - hid_shift + bs * wei_len + 2 * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - if(biasMode != 0u) - { - wei_shift_bias_temp += wei_stride; - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - if(hx != nullptr) - { - sp_size[1] = batch_n; - sp_size[2] = wei_stride; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - reserveSpace, - hid_shift, - wei_shift_bias_temp, - hid_shift, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else - { - if(batch_n - in_n.at(0) > 0) - { - sp_size[1] = batch_n - in_n.at(0); - sp_size[2] = wei_len; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - w_size[1] = 1; - w_size[2] = wei_len; - w_desc = miopen::TensorDescriptor(wDesc.GetType(), w_size, w_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - reserveSpace, - hid_shift + in_n.at(0) * hy_stride, - wei_shift_bias_temp, - hid_shift + in_n.at(0) * hy_stride, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(dirMode != 0u) - { - if(in_n.at(0) == in_n.at(seqLen - 1)) - { - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - reserveSpace, - hid_shift + wei_len, - wei_shift_bias_temp + wei_len, - hid_shift + wei_len, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else - { - int cur_batch = 0; - for(int ti = 0; ti < seqLen; ti++) - { - if(ti != (seqLen - 1)) - { - offset = hid_shift + cur_batch * hy_stride; - - sp_size[1] = in_n.at(ti + 1); - sp_size[2] = wei_len; - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - w_desc, - w, - &beta_t, - sp_desc, - reserveSpace, - static_cast(offset) + wei_len, - wei_shift_bias_temp + wei_len, - static_cast(offset) + wei_len, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - cur_batch += in_n.at(ti); - } - } - } - } - } - } - - // from hidden state - int bacc = 0; - int baccbi = batch_n; - for(int ti = 0; ti < seqLen; ti++) - { - baccbi -= in_n.at(seqLen - 1 - ti); - wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int pretime_shift = 0; - int use_time = 0; - - for(int ri = 0; ri < bi; ri++) - { - int cur_time = ri == 0 ? ti : seqLen - 1 - ti; - int cur_batch = ri == 0 ? bacc : baccbi; - offset = hid_shift + cur_batch * hy_stride; - if(ti > 0) - { - pretime_shift = - ri == 0 ? hid_shift + (bacc - in_n.at(ti - 1)) * hy_stride - : hid_shift + (baccbi + in_n.at(seqLen - 1 - ti)) * hy_stride; - use_time = ri == 0 ? ti : seqLen - ti; - } - - if(in_n.at(cur_time) > 0) - { - if(ti == 0) - { - if(hx != nullptr) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - in_n.at(cur_time), - wei_len, - hy_h, - uni_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - hx, - hx_shift + ri * hy_n * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - reserveSpace, - static_cast(offset) + ri * wei_len, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - false, - true, - (in_n.at(cur_time) - in_n.at(use_time)), - wei_len, - hy_h, - uni_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - hx, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - reserveSpace, - static_cast(offset) + ri * wei_len + - in_n.at(use_time) * hy_stride, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(in_n.at(use_time) > 0) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - true, - in_n.at(use_time), - wei_len, - hy_h, - hy_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - xDesc[0].GetType(), - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - reserveSpace, - pretime_shift + hid_off + ri * hy_h, - w, - wei_shift + ri * wei_len * uni_stride, - reserveSpace, - static_cast(offset) + ri * wei_len, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - // update hidden status - sp_size[1] = in_n.at(cur_time); - if(rnnMode == miopenRNNRELU || rnnMode == miopenRNNTANH) - { - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - activDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenLSTM) - { - if(algoMode == miopenRNNdefault) - { - LSTMForwardHiddenStateUpdate( - handle, - wDesc.GetType(), - false, - ti == 0, - ri, - in_n.at(0), - in_n.at(cur_time), - in_n.at(use_time), - hy_h, - hy_stride, - wei_len, - wei_stride, - cx, - hx_shift + ri * hy_n * hy_h, - reserveSpace, - offset + static_cast(ri) * wei_len, - offset + hy_h + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - (li * batch_n + cur_batch) * bi * hy_h + ri * hy_h + - nLayers * batch_n * hy_stride, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - continue; - } - - // active gate i, f, o - sp_size[2] = hy_h * 3; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - sigDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride); - - // active gate c - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - tanhDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len + - nLayers * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update cell state - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(ti == 0) - { - if(cx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - reserveSpace, - offset + hy_h + static_cast(ri) * wei_len + - nLayers * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && cx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - reserveSpace, - offset + hy_h + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time)) * hy_stride + - nLayers * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - } - - if(in_n.at(use_time) > 0) - { - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(use_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + hy_h + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(cur_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - } - } - - // active cell state - tanhDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - nLayers * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update hidden state - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(nLayers) * batch_n * hy_stride, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenGRU) - { - // active z, r gate - sp_size[2] = 2 * hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - sigDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - // calculate c gate - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - CopyTensor(handle, - sp_desc, - reserveSpace, - sp_desc, - reserveSpace, - static_cast(offset) + 2 * hy_h + ri * wei_len, - static_cast(offset) + hid_off + ri * hy_h + - static_cast(nLayers) * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + hy_h + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hid_off + static_cast(ri) * hy_h, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // active c gate - tanhDesc.Forward(handle, - &alpha, - sp_desc, - reserveSpace, - &beta, - sp_desc, - reserveSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - // calculate hidden state - alpha0 = -1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + hid_off + static_cast(ri) * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - - if(ti == 0) - { - if(hx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - hx_desc, - hx, - &beta_t, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h, - offset + hid_off + static_cast(ri) * hy_h, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - hx_desc, - hx, - &beta_t, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time)) * hy_stride + - static_cast(nLayers) * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + hid_off + static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - } - - if(in_n.at(use_time) > 0) - { - if(in_n.at(use_time) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(use_time); - sp_desc = miopen::TensorDescriptor( - wDesc.GetType(), sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - pretime_shift + hid_off + ri * hy_h, - offset + hid_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - } - } - - bacc += in_n.at(ti); - } - - // update hy, cy - if(hy != nullptr || (rnnMode == miopenLSTM && cy != nullptr)) - { - hx_size[2] = hy_h; - sp_size[2] = hy_h; - - bacc = batch_n; - baccbi = 0; - for(int ti = seqLen - 1; ti >= 0; ti--) - { - bacc -= in_n.at(ti); - for(int ri = 0; ri < bi; ri++) - { - int cur_time = ri == 0 ? ti : seqLen - 1 - ti; - int cur_batch = ri == 0 ? bacc : baccbi; - int use_batch = 0; - - if(ti < seqLen - 1) - { - int use_time = ri == 0 ? ti + 1 : seqLen - 2 - ti; - use_batch = in_n.at(use_time); - } - - if(in_n.at(cur_time) > use_batch) - { - offset = hid_shift + cur_batch * hy_stride; - - sp_size[1] = in_n.at(cur_time) - use_batch; - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - hx_size[1] = sp_size[1]; - hx_desc = miopen::TensorDescriptor(wDesc.GetType(), hx_size, hx_stride); - - if(hy != nullptr) - { - CopyTensor(handle, - sp_desc, - reserveSpace, - hx_desc, - hy, - static_cast(offset) + hid_off + ri * hy_h + - use_batch * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(rnnMode == miopenLSTM && cy != nullptr) - { - CopyTensor(handle, - sp_desc, - reserveSpace, - hx_desc, - cy, - static_cast(offset) + bi * wei_len + ri * hy_h + - use_batch * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - baccbi += in_n.at(seqLen - 1 - ti); - } - } - } - - // output - prelayer_shift = (static_cast(nLayers) - 1) * batch_n * hy_stride + hid_off; - - sp_size[1] = batch_n; - sp_size[2] = hy_h * bi; - y_size[1] = batch_n; - y_size[2] = out_h; - y_desc = miopen::TensorDescriptor(wDesc.GetType(), y_size, y_stride); - sp_desc = miopen::TensorDescriptor(wDesc.GetType(), sp_size, sp_stride); - - CopyTensor(handle, sp_desc, reserveSpace, y_desc, y, prelayer_shift, 0); - // Update time - profileRNNkernels(handle, 2, ctime); - -#else - (void)handle; - (void)seqLen; - (void)xDesc; - (void)x; - (void)w; - (void)hx; - (void)cx; - (void)y; - (void)hyDesc; - (void)hy; - (void)yDesc; - (void)cy; - (void)hxDesc; - (void)wDesc; - (void)reserveSpace; - (void)reserveSpaceSize; - MIOPEN_THROW("GEMM is not supported"); -#endif -}; - -void RNNDescriptor::RNNBackwardData(Handle& handle, - const int seqLen, - c_array_view yDesc, - ConstData_t y, - c_array_view dyDesc, - ConstData_t dy, - const TensorDescriptor& dhyDesc, - ConstData_t dhy, - const TensorDescriptor& dcyDesc, - ConstData_t dcy, - const TensorDescriptor& wDesc, - ConstData_t w, - const TensorDescriptor& hxDesc, - ConstData_t hx, - const TensorDescriptor& cxDesc, - ConstData_t cx, - c_array_view dxDesc, - Data_t dx, - const TensorDescriptor& dhxDesc, - Data_t dhx, - const TensorDescriptor& dcxDesc, - Data_t dcx, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const -{ - // Suppress warning - (void)y; - (void)yDesc; - (void)wDesc; - - if(dx == nullptr || w == nullptr || dy == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(dhyDesc.GetNumDims() != dcyDesc.GetNumDims() || - dhyDesc.GetNumDims() != hxDesc.GetNumDims() || dhyDesc.GetNumDims() != cxDesc.GetNumDims() || - dhyDesc.GetNumDims() != dhxDesc.GetNumDims() || dhyDesc.GetNumDims() != dcxDesc.GetNumDims()) - { - MIOPEN_THROW(miopenStatusBadParm); - } - -#if MIOPEN_BACKEND_HIP - RnnHipAutoProfiler kernel_profiler{handle}; - - try - { -#endif - - if(paddingMode == miopenRNNIONotPadded) - { - return RNNBackwardDataPackedTensors(handle, - seqLen, - dyDesc, - dy, - dhy, - dcy, - w, - hx, - cx, - dxDesc, - dx, - dhxDesc, - dhx, - dcxDesc, - dcx, - workSpace, - workSpaceSize, - reserveSpace, - reserveSpaceSize); - } - else - { - Data_t packedDYIn = workSpace; - size_t packedDXSize, packedDYSize; - std::tie(packedDXSize, packedDYSize) = - RNNTensorPaddingConverter::GetTempPackedBuffersSpace(*this, dxDesc); - - Data_t packedDXOut = - static_cast(reinterpret_cast(workSpace) + packedDYSize); - - auto shifted_workSpace = static_cast(reinterpret_cast(workSpace) + - (packedDYSize + packedDXSize)); - auto shifted_workSpace_size = workSpaceSize - (packedDYSize + packedDXSize); - - std::vector in_n(seqLen); - - for(int i = 0; i < seqLen; i++) - { - int batchval, batchvalout; - std::tie(batchval, std::ignore) = miopen::tien<2>(dxDesc[i].GetLengths()); - std::tie(batchvalout, std::ignore) = miopen::tien<2>(dyDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - in_n[i] = batchval; - } - - RNNTensorPaddingConverter::ConvertTensorData( - handle, dyDesc[0], in_n, dy, packedDYIn, true); - - RNNDescriptor packedRnnDesc(*this); - packedRnnDesc.SetPaddingmode(miopenRNNIONotPadded); - - packedRnnDesc.RNNBackwardDataPackedTensors(handle, - seqLen, - dyDesc, - packedDYIn, - dhy, - dcy, - w, - hx, - cx, - dxDesc, - packedDXOut, - dhxDesc, - dhx, - dcxDesc, - dcx, - shifted_workSpace, - shifted_workSpace_size, - reserveSpace, - reserveSpaceSize); - - RNNTensorPaddingConverter::ConvertTensorData( - handle, dxDesc[0], in_n, packedDXOut, dx, false); - } - -#if MIOPEN_BACKEND_HIP - } - catch(...) - { - kernel_profiler.abortProfiling(); - throw; - } -#endif -} - -void RNNDescriptor::RNNBackwardDataPackedTensors( - Handle& handle, - const int seqLen, - c_array_view dyDesc, - ConstData_t dy, - ConstData_t dhy, - ConstData_t dcy, - ConstData_t w, - ConstData_t hx, - ConstData_t cx, - c_array_view dxDesc, - Data_t dx, - const TensorDescriptor& dhxDesc, - Data_t dhx, - const TensorDescriptor& dcxDesc, - Data_t dcx, - Data_t workSpace, - size_t workSpaceSize, - Data_t reserveSpace, - size_t reserveSpaceSize) const -{ -#if MIOPEN_USE_GEMM - - float ctime = 0.; - // reset kernel timer - profileRNNkernels(handle, 0, ctime); - - // if projections supported, dcxDesc.GetLengths()[2] should be used for hidden_size, - // dhxDesc.GetLengths()[2] for proj_size. - if(dhxDesc.GetNumDims() != dcxDesc.GetNumDims() || - dhxDesc.GetLengths()[2] != dcxDesc.GetLengths()[2]) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(paddingMode != miopenRNNIONotPadded) - { - MIOPEN_THROW("Padded IO is not supported by this solver"); - } - if(workSpaceSize < GetWorkspaceSize(handle, seqLen, dxDesc)) - { - MIOPEN_THROW("Workspace is required"); - } - if(reserveSpaceSize < GetReserveSize(handle, seqLen, dxDesc)) - { - MIOPEN_THROW("Reservespace is required"); - } - - auto rnn_data_type = dhxDesc.GetType(); - - std::vector in_n; - int in_h = dxDesc[0].GetLengths()[1]; - int hy_d = dhxDesc.GetLengths()[0]; - int hy_n = dhxDesc.GetLengths()[1]; - int hy_h = dhxDesc.GetLengths()[2]; - int out_h = dyDesc[0].GetLengths()[1]; - - if(in_h <= 0 || hy_h <= 0 || hy_n <= 0 || hy_d <= 0 || out_h <= 0 || seqLen <= 0) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - int batch_n = 0; - for(int i = 0; i < seqLen; i++) - { - int batchval, inputvec, batchvalout, outputvec; - std::tie(batchval, inputvec) = miopen::tien<2>(dxDesc[i].GetLengths()); - std::tie(batchvalout, outputvec) = miopen::tien<2>(dyDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(i == 0) - { - if(batchval <= 0) - { - MIOPEN_THROW(miopenStatusBadParm, "Input batch is ZERO!"); - } - } - else - { - if(batchval > in_n.back() || batchval < 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "Incorrect input batch size at time " + std::to_string(i) + - "! Batch size must not ascend!"); - } - } - in_n.push_back(batchval); - batch_n += dxDesc[i].GetLengths()[0]; - } - - int bi = dirMode != 0u ? 2 : 1; - if(out_h != (bi * hy_h)) - { - MIOPEN_THROW(miopenStatusBadParm, "Output size doesn't match hidden state size!"); - } - - bool use_dropout = !float_equal(miopen::deref(dropoutDesc).dropout, 0); - - if(dirMode == 0 && inputMode == miopenRNNlinear && rnnMode == miopenLSTM && !use_dropout && - algoMode == miopenRNNdefault) - { - SeqTensorDescriptor dx_seq = - makeSeqTensorDescriptor(dxDesc, seqLen, miopenRNNDataSeqMajorNotPadded); - - SeqTensorDescriptor dy_seq = - makeSeqTensorDescriptor(dyDesc, seqLen, miopenRNNDataSeqMajorNotPadded); - - return this->ModularBackward(handle, - dy_seq, - dy, - dhxDesc, - hx, - dhy, - dhx, - dcxDesc, - cx, - dcy, - dcx, - dx_seq, - dx, - w, - workSpace, - workSpaceSize, - reserveSpace, - reserveSpaceSize); - } - - int in_stride = in_h; - int hy_stride = hy_h * bi * static_cast(workspaceScale); - int out_stride = out_h; - int wei_stride = hy_h * bi * static_cast(nHiddenTensorsPerLayer); - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - if(inputMode == miopenRNNskip) - { - if(in_h != hy_h) - { - MIOPEN_THROW(miopenStatusBadParm, - "The input tensor size must equal to the hidden " - "state size of the network in SKIP_INPUT mode!"); - } - in_h = 0; - } - - size_t offset; - float alpha0, alpha1, beta_t; - float alpha = 1, beta = 0; - - std::vector sp_size(3, 1), sp_stride(3, 1), x_size(3, 1), x_stride(3, 1), y_size(3, 1), - y_stride(3, 1), hx_size(3, 1), hx_stride(3, 1); - miopen::TensorDescriptor sp_desc, x_desc, y_desc, hx_desc; - - sp_size[2] = workSpaceSize / GetTypeSize(rnn_data_type); - sp_stride[0] = sp_size[2]; - sp_stride[1] = sp_size[2]; - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - SetTensor(handle, sp_desc, workSpace, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - sp_stride[0] = batch_n * hy_stride; - sp_stride[1] = hy_stride; - sp_size[2] = 1; - x_stride[0] = batch_n * in_stride; - x_stride[1] = in_stride; - y_stride[0] = batch_n * out_stride; - y_stride[1] = out_stride; - if(dhx != nullptr || (rnnMode == miopenLSTM && dcx != nullptr)) - { - hx_size[2] = hy_d * hy_n * hy_h; - hx_stride[0] = hx_size[2]; - hx_stride[1] = hx_size[2]; - hx_desc = miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - if(dhx != nullptr) - { - SetTensor(handle, hx_desc, dhx, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - if(rnnMode == miopenLSTM && dcx != nullptr) - { - SetTensor(handle, hx_desc, dcx, &beta); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - hx_stride[0] = in_n.at(0) * uni_stride; - hx_stride[1] = uni_stride; - - int prelayer_shift, pretime_shift, cur_time, cur_batch; - int wei_len = 0; - int wei_len_t = 0; - int dhd_off = 0; - int use_time = 0; - int pre_batch = 0; - int use_time2 = 0; - int pre_batch2 = 0; - - switch(rnnMode) - { - case miopenRNNRELU: - case miopenRNNTANH: - // printf("run rnn gpu bwd data \n"); - // wei_len = hy_h * nHiddenTensorsPerLayer; // hy_h; - // wei_len_t = hy_h * nHiddenTensorsPerLayer; // hy_h; - // dhd_off = bi * hy_h * (workspaceScale - 1); // 0; - // break; - case miopenLSTM: - // printf("run lstm gpu bwd data \n"); - wei_len = hy_h * static_cast(nHiddenTensorsPerLayer); // hy_h * 4; - wei_len_t = hy_h * static_cast(nHiddenTensorsPerLayer); // hy_h * 4; - dhd_off = bi * hy_h * static_cast(workspaceScale - 1); // bi* hy_h * 5; - break; - case miopenGRU: - // printf("run gru gpu bwd data \n"); - wei_len = hy_h * static_cast(nHiddenTensorsPerLayer); // hy_h * 3; - wei_len_t = hy_h * static_cast(nHiddenTensorsPerLayer - 1); // hy_h * 2; - dhd_off = bi * hy_h * static_cast(workspaceScale - 1); // bi * hy_h * 3; - break; - } - - ActivationDescriptor tanhDesc, sigDesc, activDesc; - sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; - tanhDesc = {miopenActivationTANH, 1, 1, 1}; - if(rnnMode == miopenRNNRELU) - { - activDesc = {miopenActivationRELU, 1, 0, 1}; - } - else if(rnnMode == miopenRNNTANH) - { - activDesc = {miopenActivationTANH, 1, 1, 1}; - } - - for(int li = static_cast(nLayers) - 1; li >= 0; li--) - { - int wei_shift = (in_h + hy_h) * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - int hid_shift = li * batch_n * hy_stride; - int hx_shift = li * hy_n * bi_stride; - int weitime_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - // feedback from output - if(li == nLayers - 1) - { - y_size[1] = batch_n; - y_size[2] = out_h; - sp_size[1] = batch_n; - sp_size[2] = hy_h * bi; - y_desc = miopen::TensorDescriptor(rnn_data_type, y_size, y_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - CopyTensor(handle, y_desc, dy, sp_desc, workSpace, 0, hid_shift + dhd_off); - // Update time - profileRNNkernels(handle, 1, ctime); // start timing - } - else - { - prelayer_shift = (li + 1) * batch_n * hy_stride; - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - false, - batch_n, - hy_h * bi, - wei_len * bi, - hy_stride, - bi_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_type, - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - workSpace, - prelayer_shift, - w, - wei_shift, - workSpace, - hid_shift + dhd_off, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - - if(use_dropout) - { - std::vector drop_size(2), drop_in_str(2, 1); - drop_size[0] = batch_n; - drop_size[1] = hy_h * bi; - drop_in_str[0] = hy_stride; - - auto drop_in_desc = miopen::TensorDescriptor(rnn_data_type, drop_size, drop_in_str); - - size_t drop_rsv_size = drop_in_desc.GetElementSize(); - size_t drop_rsv_start = - algoMode == miopenRNNdefault && rnnMode == miopenLSTM - ? nLayers * batch_n * hy_stride + nLayers * batch_n * hy_h * bi - : 2 * nLayers * batch_n * hy_stride; - - size_t drop_rsv_offset = (drop_rsv_start + (nLayers - 1) * batch_n * hy_h * bi) * - (rnn_data_type == miopenFloat ? 4 : 2) + - li * drop_rsv_size; - - miopen::deref(dropoutDesc) - .DropoutBackward(handle, - drop_in_desc, - drop_in_desc, - workSpace, - drop_in_desc, - workSpace, - reserveSpace, - drop_rsv_size, - hid_shift + dhd_off, - hid_shift + dhd_off, - drop_rsv_offset); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - // from hidden state - int bacc = batch_n; - int baccbi = 0; - for(int ti = seqLen - 1; ti >= 0; ti--) - { - bacc -= in_n.at(ti); - - // from post state - for(int ri = 0; ri < bi; ri++) - { - cur_time = ri == 0 ? ti : seqLen - 1 - ti; - cur_batch = ri == 0 ? bacc : baccbi; - offset = hid_shift + cur_batch * hy_stride; - if(ti < seqLen - 1) - { - use_time = ri == 0 ? ti + 1 : seqLen - 1 - ti; - pre_batch = ri == 0 ? bacc + in_n.at(ti) : baccbi - in_n.at(seqLen - 2 - ti); - } - if(ti > 0) - { - use_time2 = ri == 0 ? ti : seqLen - ti; - pre_batch2 = - ri == 0 ? bacc - in_n.at(ti - 1) : baccbi + in_n.at(seqLen - 1 - ti); - } - - if(in_n.at(cur_time) > 0) - { - if(ti == seqLen - 1) - { - if(dhy != nullptr) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - sp_size[1] = in_n.at(cur_time); - sp_size[2] = hy_h; - hx_desc = miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hx_desc, - dhy, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h, - offset + dhd_off + static_cast(ri) * hy_h, - offset + dhd_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 0 && dhy != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_size[2] = hy_h; - hx_desc = miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hx_desc, - dhy, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + dhd_off + static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride, - offset + dhd_off + static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - pretime_shift = - li * batch_n * hy_stride + pre_batch * hy_stride + ri * wei_len; - - if(in_n.at(use_time) > 0) - { - if(rnnMode == miopenGRU) - { - sp_size[1] = in_n.at(use_time); - sp_size[2] = hy_h; - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - pretime_shift - ri * 2 * hy_h + dhd_off, - pretime_shift + nLayers * batch_n * hy_stride, - offset + dhd_off + static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - CopyTensor(handle, - sp_desc, - workSpace, - sp_desc, - workSpace, - pretime_shift + 2 * hy_h, - static_cast(offset) + ri * wei_len + 2 * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - CopyTensor(handle, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - pretime_shift - ri * 2 * hy_h + dhd_off + - static_cast(nLayers) * batch_n * hy_stride, - pretime_shift + 2 * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - false, - in_n.at(use_time), - hy_h, - wei_len, - hy_stride, - uni_stride, - hy_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_type, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - pretime_shift, - w, - weitime_shift + ri * wei_len * uni_stride, - workSpace, - static_cast(offset) + dhd_off + ri * hy_h, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - - if(rnnMode == miopenGRU) - { - CopyTensor(handle, - sp_desc, - workSpace, - sp_desc, - workSpace, - static_cast(offset) + ri * wei_len + 2 * hy_h, - pretime_shift + 2 * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - - // update hidden status - sp_size[1] = in_n.at(cur_time); - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - if(rnnMode == miopenRNNRELU || rnnMode == miopenRNNTANH) - { - // activation - activDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenLSTM) - { - if(algoMode == miopenRNNdefault) - { - LSTMBackwardHiddenStateUpdate( - handle, - rnn_data_type, - ti == 0, - ti == seqLen - 1, - ri, - in_n.at(0), - in_n.at(cur_time), - in_n.at(use_time), - in_n.at(use_time2), - hy_h, - hy_stride, - wei_len, - wei_stride, - cx, - hx_shift + ri * hy_n * hy_h, - reserveSpace, - offset + static_cast(ri) * wei_len, - offset + hy_h + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - (li * batch_n + cur_batch) * bi * hy_h + ri * hy_h + - nLayers * batch_n * hy_stride, - li * batch_n * hy_stride + pre_batch2 * hy_stride + bi * wei_len + - ri * hy_h, - dcy, - hx_shift + ri * hy_n * hy_h, - workSpace, - offset + static_cast(ri) * wei_len, - offset + hy_h + static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - li * batch_n * hy_stride + pre_batch * hy_stride + bi * wei_len + - ri * hy_h, - offset + dhd_off + static_cast(ri) * hy_h, - li * batch_n * hy_stride + pre_batch * hy_stride + hy_h + - ri * wei_len); - - // Update time - profileRNNkernels(handle, 1, ctime); - continue; - } - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - // update cell state - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + dhd_off + static_cast(ri) * hy_h, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - tanhDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - nLayers * batch_n * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(ti == seqLen - 1) - { - if(dcy != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hx_desc, - dcy, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 0 && dcy != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time); - hx_size[2] = hy_h; - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time); - sp_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - hx_desc, - dcy, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(in_n.at(use_time)) * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - pretime_shift = li * batch_n * hy_stride + pre_batch * hy_stride; - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - - if(in_n.at(cur_time) != in_n.at(use_time)) - { - sp_size[1] = in_n.at(use_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - pretime_shift + hy_h + ri * wei_len + - nLayers * batch_n * hy_stride, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(in_n.at(cur_time) != in_n.at(use_time)) - { - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - } - - // update forget gate - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - if(ti == 0) - { - if(cx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - hx_shift + ri * hy_n * hy_h, - offset + hy_h + static_cast(ri) * wei_len, - true); - } - } - else - { - if(ri == 1 && cx != nullptr && in_n.at(cur_time) > in_n.at(use_time2)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time2); - hx_size[2] = hy_h; - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time2); - sp_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - hx_desc, - cx, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + - static_cast(in_n.at(use_time2)) * hy_stride, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time2) * hy_h, - offset + hy_h + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time2)) * hy_stride, - true); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - if(in_n.at(use_time2) > 0) - { - pretime_shift = li * batch_n * hy_stride + pre_batch2 * hy_stride; - - if(in_n.at(cur_time) != in_n.at(use_time2)) - { - sp_size[1] = in_n.at(use_time2); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - pretime_shift + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + hy_h + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(in_n.at(cur_time) != in_n.at(use_time2)) - { - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - } - } - - // update input gate - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len + - nLayers * batch_n * hy_stride, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update output gate - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + dhd_off + static_cast(ri) * hy_h, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h + nLayers * batch_n * hy_stride, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // update c gate - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + static_cast(bi) * wei_len + - static_cast(ri) * hy_h, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - tanhDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len + - nLayers * batch_n * hy_stride, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 3 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[2] = 3 * hy_h; - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - sigDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - else if(rnnMode == miopenGRU) - { - // c gate - alpha0 = 1; - alpha1 = -1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + dhd_off + static_cast(ri) * hy_h, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + dhd_off + static_cast(ri) * hy_h, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - tanhDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - // r gate - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + dhd_off + static_cast(ri) * hy_h + - nLayers * batch_n * hy_stride, - offset + hy_h + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len, - offset + hy_h + static_cast(ri) * wei_len + - nLayers * batch_n * hy_stride, - offset + dhd_off + static_cast(ri) * hy_h + - nLayers * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - - // z gate - if(ti == 0) - { - if(hx != nullptr) - { - hx_size[1] = in_n.at(cur_time); - hx_size[2] = hy_h; - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - hx_desc, - hx, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h, - offset + dhd_off + static_cast(ri) * hy_h, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time2)) - { - hx_size[1] = in_n.at(cur_time) - in_n.at(use_time2); - hx_size[2] = hy_h; - sp_size[1] = in_n.at(cur_time) - in_n.at(use_time2); - hx_desc = - miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - hx_desc, - hx, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time2) * hy_h, - offset + dhd_off + static_cast(ri) * hy_h + - static_cast(in_n.at(use_time2)) * hy_stride, - offset + static_cast(ri) * wei_len + - static_cast(in_n.at(use_time2)) * hy_stride, - true); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - if(in_n.at(use_time2) > 0) - { - if(in_n.at(use_time2) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(use_time2); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - hid_shift + pre_batch2 * hy_stride + dhd_off + ri * hy_h, - offset + dhd_off + static_cast(ri) * hy_h, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - if(in_n.at(use_time2) != in_n.at(cur_time)) - { - sp_size[1] = in_n.at(cur_time); - sp_desc = - miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - } - } - - alpha0 = -1; - alpha1 = 1; - beta_t = 1; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - reserveSpace, - &alpha1, - sp_desc, - workSpace, - &beta_t, - sp_desc, - workSpace, - offset + 2 * static_cast(hy_h) + - static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + dhd_off + static_cast(ri) * hy_h, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - - sp_size[2] = 2 * hy_h; - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - sigDesc.Backward(handle, - &alpha, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - sp_desc, - reserveSpace, - &beta, - sp_desc, - workSpace, - offset + static_cast(ri) * wei_len + - static_cast(nLayers) * batch_n * hy_stride, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len, - offset + static_cast(ri) * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - - baccbi += in_n.at(seqLen - 1 - ti); - } - - // dcx, dhx - if(dhx != nullptr || (rnnMode == miopenLSTM && dcx != nullptr)) - { - hx_size[2] = hy_h; - sp_size[2] = hy_h; - - bacc = 0; - baccbi = batch_n; - for(int ti = 0; ti < seqLen; ti++) - { - baccbi -= in_n.at(seqLen - 1 - ti); - for(int ri = 0; ri < bi; ri++) - { - cur_time = ri == 0 ? ti : seqLen - 1 - ti; - cur_batch = ri == 0 ? bacc : baccbi; - use_time = 0; - int use_batch = 0; - - if(ti > 0) - { - use_time = ri == 0 ? ti - 1 : seqLen - ti; - use_batch = in_n.at(use_time); - } - - if(in_n.at(cur_time) > use_batch) - { - pretime_shift = li * batch_n * hy_stride + cur_batch * hy_stride; - - if(rnnMode == miopenLSTM || rnnMode == miopenGRU) - { - sp_size[1] = in_n.at(cur_time) - use_batch; - hx_size[1] = in_n.at(cur_time) - use_batch; - hx_desc = miopen::TensorDescriptor(rnn_data_type, hx_size, hx_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_type, sp_size, sp_stride); - } - - if(dhx != nullptr) - { - if(rnnMode == miopenGRU) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - sp_desc, - reserveSpace, - pretime_shift + 2 * hy_h + ri * wei_len + - use_batch * hy_stride, - pretime_shift + hy_h + ri * wei_len + - use_batch * hy_stride + nLayers * batch_n * hy_stride, - pretime_shift + dhd_off + ri * hy_h + - use_batch * hy_stride + nLayers * batch_n * hy_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - false, - false, - (in_n.at(cur_time) - use_batch), - hy_h, - hy_h, - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 0, // beta - rnn_data_type, - false}; - - miopenStatus_t gemm_status = CallGemm( - handle, - gemm_desc, - reserveSpace, - pretime_shift + dhd_off + ri * hy_h + use_batch * hy_stride + - static_cast(nLayers) * batch_n * hy_stride, - w, - weitime_shift + 2 * hy_h * uni_stride + - ri * wei_len * uni_stride, - dhx, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - - beta_t = 1; - - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - hx_desc, - dhx, - pretime_shift + dhd_off + ri * hy_h + - use_batch * hy_stride, - pretime_shift + ri * wei_len + use_batch * hy_stride + - nLayers * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - false, - false, - (in_n.at(cur_time) - use_batch), - hy_h, - wei_len_t, - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_type, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - pretime_shift + ri * wei_len + use_batch * hy_stride, - w, - weitime_shift + ri * wei_len * uni_stride, - dhx, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h, - GemmBackend_t::rocblas); - - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(rnnMode == miopenLSTM && dcx != nullptr) - { - alpha0 = 1; - alpha1 = 1; - beta_t = 1; - if(algoMode == miopenRNNdefault) - { - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - hx_desc, - dcx, - pretime_shift + bi * wei_len + ri * hy_h + - static_cast(use_batch) * hy_stride, - pretime_shift + hy_h + ri * wei_len + - use_batch * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - continue; - } - OpTensor(handle, - miopenTensorOpMul, - &alpha0, - sp_desc, - workSpace, - &alpha1, - sp_desc, - reserveSpace, - &beta_t, - hx_desc, - dcx, - pretime_shift + bi * wei_len + ri * hy_h + - static_cast(use_batch) * hy_stride, - pretime_shift + hy_h + ri * wei_len + use_batch * hy_stride + - nLayers * batch_n * hy_stride, - hx_shift + ri * hy_n * hy_h + use_batch * hy_h); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - } - bacc += in_n.at(ti); - } - } - } - - // dinput - if(inputMode == miopenRNNskip) - { - const std::vector dx_size{1, batch_n, hy_h}; - x_desc = miopen::TensorDescriptor(rnn_data_type, dx_size, x_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_type, dx_size, sp_stride); - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - CopyTensor(handle, sp_desc, workSpace, x_desc, dx, 0, 0, true); - profileRNNkernels(handle, 1, ctime); - for(int gi = 1; gi < nHiddenTensorsPerLayer * bi; gi++) - { - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - x_desc, - dx, - &beta_t, - x_desc, - dx, - static_cast(gi) * hy_h, - 0, - 0); - // Update time - profileRNNkernels(handle, (gi == nHiddenTensorsPerLayer * bi - 1) ? 2 : 1, ctime); - } - } - else - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - false, - false, - batch_n, - in_h, - wei_len * bi, - hy_stride, - in_stride, - in_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 0, // beta - rnn_data_type, - false}; - miopenStatus_t gemm_status = - CallGemm(handle, gemm_desc, workSpace, 0, w, 0, dx, 0, GemmBackend_t::rocblas); - if(gemm_status != miopenStatusSuccess) - { - if(gemm_status == miopenStatusNotImplemented) - { - MIOPEN_LOG_E("GEMM not implemented"); - } - else - { - MIOPEN_LOG_E("GEMM failed"); - } - } - // Update time - profileRNNkernels(handle, 2, ctime); - } - -#else - - (void)handle; - (void)seqLen; - (void)dhy; - (void)dcy; - (void)dyDesc; - (void)dy; - (void)w; - (void)hx; - (void)cx; - (void)dxDesc; - (void)dx; - (void)dhxDesc; - (void)dhx; - (void)dcxDesc; - (void)dcx; - (void)workSpace; - (void)workSpaceSize; - (void)reserveSpace; - (void)reserveSpaceSize; - MIOPEN_THROW("GEMM is not supported"); -#endif -}; - -void RNNDescriptor::RNNBackwardWeights(Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - c_array_view dyDesc, - ConstData_t dy, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const -{ - (void)dy; - -#if MIOPEN_BACKEND_HIP - RnnHipAutoProfiler kernel_profiler{handle}; - - try - { -#endif - - if(paddingMode == miopenRNNIONotPadded) - { - return RNNBackwardWeightsPackedTensors(handle, - seqLen, - xDesc, - x, - hxDesc, - hx, - dyDesc, - dwDesc, - dw, - workSpace, - workSpaceSize, - reserveSpace, - reserveSpaceSize); - } - else - { - Data_t packedXIn = workSpace; - - size_t packedXSize, WA_workSpace_bug; - std::tie(packedXSize, WA_workSpace_bug) = - RNNTensorPaddingConverter::GetTempPackedBuffersSpace(*this, xDesc); - - auto shifted_workSpace = static_cast(reinterpret_cast(workSpace) + - (packedXSize + WA_workSpace_bug)); - auto shifted_workSpace_size = workSpaceSize - (packedXSize + WA_workSpace_bug); - - std::vector in_n(seqLen); - - for(int i = 0; i < seqLen; i++) - { - int batchval, batchvalout; - std::tie(batchval, std::ignore) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, std::ignore) = miopen::tien<2>(dyDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm, - "Input batch length: " + std::to_string(batchval) + - ", Output batch length: " + std::to_string(batchvalout)); - } - in_n[i] = batchval; - } - - RNNTensorPaddingConverter::ConvertTensorData( - handle, xDesc[0], in_n, x, packedXIn, true); - - RNNDescriptor packedRnnDesc(*this); - packedRnnDesc.SetPaddingmode(miopenRNNIONotPadded); - - packedRnnDesc.RNNBackwardWeightsPackedTensors(handle, - seqLen, - xDesc, - packedXIn, - hxDesc, - hx, - dyDesc, - dwDesc, - dw, - shifted_workSpace, - shifted_workSpace_size, - reserveSpace, - reserveSpaceSize); - } - -#if MIOPEN_BACKEND_HIP - } - catch(...) - { - kernel_profiler.abortProfiling(); - throw; - } -#endif -} - -void RNNDescriptor::RNNBackwardWeightsPackedTensors( - Handle& handle, - const int seqLen, - c_array_view xDesc, - ConstData_t x, - const TensorDescriptor& hxDesc, - ConstData_t hx, - c_array_view dyDesc, - const TensorDescriptor& dwDesc, - Data_t dw, - Data_t workSpace, - size_t workSpaceSize, - ConstData_t reserveSpace, - size_t reserveSpaceSize) const -{ - -#if MIOPEN_USE_GEMM - float ctime = 0.; - // reset kernel timer - profileRNNkernels(handle, 0, ctime); - // if projections supported, dcxDesc.GetLengths()[2] should be used for hidden_size, - // dhxDesc.GetLengths()[2] for proj_size. - - if(paddingMode != miopenRNNIONotPadded) - { - MIOPEN_THROW("Padded IO is not supported by this solver"); - } - - if(x == nullptr || dw == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(workSpaceSize < GetWorkspaceSize(handle, seqLen, xDesc)) - { - MIOPEN_THROW("Workspace is required"); - } - if(reserveSpaceSize < GetReserveSize(handle, seqLen, xDesc)) - { - MIOPEN_THROW("Reservespace is required"); - } - - std::string network_config; - std::vector in_n; - int in_h = xDesc[0].GetLengths()[1]; - int hy_d = hxDesc.GetLengths()[0]; - int hy_n = hxDesc.GetLengths()[1]; - int hy_h = hxDesc.GetLengths()[2]; - int out_h = dyDesc[0].GetLengths()[1]; - - miopenDataType_t rnn_data_t = hxDesc.GetType(); - - bool use_dropout = !float_equal(miopen::deref(dropoutDesc).dropout, 0); - - if(in_h <= 0 || hy_h <= 0 || hy_n <= 0 || hy_d <= 0 || out_h <= 0 || seqLen <= 0) - { - MIOPEN_THROW(miopenStatusBadParm); - } - - int batch_n = 0; - for(int i = 0; i < seqLen; i++) - { - int batchval, inputvec, batchvalout, outputvec; - std::tie(batchval, inputvec) = miopen::tien<2>(xDesc[i].GetLengths()); - std::tie(batchvalout, outputvec) = miopen::tien<2>(dyDesc[i].GetLengths()); - if(batchval != batchvalout) - { - MIOPEN_THROW(miopenStatusBadParm); - } - if(i == 0) - { - if(batchval <= 0) - { - MIOPEN_THROW(miopenStatusBadParm, "Input batch is ZERO!"); - } - } - else - { - if(batchval > in_n.back() || batchval < 0) - { - MIOPEN_THROW(miopenStatusBadParm, - "Incorrect input batch size at time " + std::to_string(i) + - "! Batch size must not ascend!"); - } - } - in_n.push_back(batchval); - batch_n += xDesc[i].GetLengths()[0]; - } - - int bi = dirMode != 0u ? 2 : 1; - if(out_h != (bi * hy_h)) - { - MIOPEN_THROW(miopenStatusBadParm, "Output size doesn't match hidden state size!"); - } - - int in_stride = in_h; - int hy_stride = hy_h * bi * static_cast(workspaceScale); - int wei_stride = hy_h * bi * static_cast(nHiddenTensorsPerLayer); - int uni_stride = hy_h; - int bi_stride = hy_h * bi; - - if(inputMode == miopenRNNskip) - { - if(in_h != hy_h) - { - MIOPEN_THROW(miopenStatusBadParm, - "The input tensor size must equal to the hidden " - "state size of the network in SKIP_INPUT mode!"); - } - in_h = 0; - } - - if(dirMode == 0 && rnnMode == miopenLSTM && !use_dropout && algoMode == miopenRNNdefault && - !env::disabled(MIOPEN_RNNWRW_EXP)) - { - SeqTensorDescriptor x_seq = - makeSeqTensorDescriptor(xDesc, seqLen, miopenRNNDataSeqMajorNotPadded); - - SeqTensorDescriptor y_seq = - makeSeqTensorDescriptor(dyDesc, seqLen, miopenRNNDataSeqMajorNotPadded); - - return this->ModularBackwardWeights(handle, - x_seq, - x, - hxDesc, - hx, - y_seq, - dw, - workSpace, - workSpaceSize, - reserveSpace, - reserveSpaceSize); - } - - size_t wei_shift_bias = (in_h + hy_h + (bi * hy_h + hy_h) * (nLayers - 1)) * wei_stride; - - float alpha0, alpha1, beta_t = 0; - - std::vector sp_size(3, 1), sp_stride(3, 1), w_size(3, 1), w_stride(3, 1); - miopen::TensorDescriptor sp_desc, w_desc; - - sp_stride[0] = batch_n * hy_stride; - sp_stride[1] = hy_stride; - - const auto dw_tensor_size = GetParamsSize(xDesc[0].GetLengths()[1]) / GetTypeSize(rnn_data_t); - - w_desc = miopen::TensorDescriptor( - rnn_data_t, {1, 1, dw_tensor_size}, {dw_tensor_size, dw_tensor_size, 1}); - - SetTensor(handle, w_desc, dw, &beta_t); - // Update time - profileRNNkernels(handle, 1, ctime); - w_stride[0] = wei_stride; - w_stride[1] = wei_stride; - w_size[2] = 1; - - int wei_len = 0; - int hid_off = 0; - int use_time = 0; - int pre_batch = 0; - - switch(rnnMode) - { - case miopenRNNRELU: - case miopenRNNTANH: - // printf("run rnn gpu bwd weights \n"); - wei_len = hy_h; - hid_off = static_cast(nLayers) * batch_n * hy_stride; - break; - case miopenLSTM: - // printf("run lstm gpu bwd weights \n"); - wei_len = hy_h * 4; - hid_off = bi * hy_h * 5; - break; - case miopenGRU: - // printf("run gru gpu bwd weights \n"); - wei_len = hy_h * 3; - hid_off = bi * hy_h * 3; - break; - } - - for(int li = 0; li < nLayers; li++) - { - int hid_shift = li * batch_n * hy_stride; - int wei_shift = (in_h + hy_h) * wei_stride + (li - 1) * (bi * hy_h + hy_h) * wei_stride; - - size_t dw_bias_offset = wei_shift_bias + static_cast(li) * 2 * wei_stride; - - // between layers - if(li == 0) - { - if(inputMode == miopenRNNlinear) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - true, - false, - wei_len * bi, - in_h, - batch_n, - hy_stride, - in_stride, - in_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, gemm_desc, workSpace, 0, x, 0, dw, 0, GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - auto prelayer_shift = static_cast( - use_dropout ? (algoMode == miopenRNNdefault && rnnMode == miopenLSTM - ? nLayers * batch_n * hy_stride + nLayers * batch_n * hy_h * bi - : 2 * nLayers * batch_n * hy_stride) + - (static_cast(li) - 1) * batch_n * hy_h * bi - : (li - 1) * batch_n * hy_stride + hid_off); - - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - true, - false, - wei_len * bi, - hy_h * bi, - batch_n, - hy_stride, - use_dropout ? hy_h * bi : hy_stride, - bi_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - workSpace, - hid_shift, - reserveSpace, - prelayer_shift, - dw, - wei_shift, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - if(biasMode != 0u) - { - const std::vector ws_bias_strides{ - static_cast(batch_n) * hy_stride, static_cast(hy_stride), 1}; - const miopen::TensorDescriptor ws_desc{ - rnn_data_t, - {1, static_cast(batch_n), static_cast(wei_stride)}, - ws_bias_strides}; - - const std::vector dw_bias_strides{ - static_cast(wei_stride), static_cast(wei_stride), 1}; - const miopen::TensorDescriptor dw_desc{ - rnn_data_t, {1, 1, static_cast(wei_stride)}, dw_bias_strides}; - - size_t main_ws_size = - GetMainSolWorkspaceSize(batch_n, miopenRNNTraining, miopenRNNDataSeqMajorNotPadded); - size_t reduction_ws_size = workSpaceSize - main_ws_size; - - Data_t reduction_workSpace = static_cast(workSpace) + main_ws_size; - - ReducAddBias(handle, - dw, - workSpace, - dw_desc, - ws_desc, - dw_bias_offset, - hid_shift, - reduction_workSpace, - reduction_ws_size); - - // Update time - profileRNNkernels(handle, 1, ctime); - } - - // between time - // Calculate feedback for c gate in GRU - if(rnnMode == miopenGRU) - { - sp_size[1] = batch_n; - sp_size[2] = hy_h; - sp_desc = miopen::TensorDescriptor(dwDesc.GetType(), sp_size, sp_stride); - - for(int ri = 0; ri < bi; ri++) - { - CopyTensor(handle, - sp_desc, - reserveSpace, - sp_desc, - workSpace, - hid_shift + hid_off + ri * hy_h + - static_cast(nLayers) * batch_n * hy_stride, - hid_shift + 2 * hy_h + ri * wei_len); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - if(biasMode != 0u) - { - wei_shift = static_cast(wei_shift_bias) + li * 2 * wei_stride + wei_stride; - - alpha0 = 1; - alpha1 = 1; - beta_t = 0; - - if(hx != nullptr) - { - if(rnnMode == miopenGRU) - { - sp_size[1] = batch_n; - sp_size[2] = wei_stride; - w_size[1] = 1; - w_size[2] = wei_stride; - w_desc = miopen::TensorDescriptor(dwDesc.GetType(), w_size, w_stride); - sp_desc = miopen::TensorDescriptor(dwDesc.GetType(), sp_size, sp_stride); - - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - w_desc, - dw, - &alpha1, - sp_desc, - workSpace, - &beta_t, - w_desc, - dw, - wei_shift, - hid_shift, - wei_shift, - true); - - // Update time - profileRNNkernels(handle, 1, ctime); - } - else - { - // second dw bias equal to the first, so just copy reduction result - const std::vector dw_bias_strides{wei_stride, wei_stride, 1}; - const miopen::TensorDescriptor dw_desc{ - rnn_data_t, {1, 1, wei_stride}, dw_bias_strides}; - - CopyTensor(handle, - dw_desc, - dw, - dw_desc, - dw, - dw_bias_offset, - dw_bias_offset + wei_stride); - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - else - { - sp_size[1] = 1; - sp_size[2] = wei_len; - w_size[1] = 1; - w_size[2] = wei_len; - w_desc = miopen::TensorDescriptor(rnn_data_t, w_size, w_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_t, sp_size, sp_stride); - - for(int bs = 0; bs < batch_n; bs++) - { - if(!(hx == nullptr && bs < in_n.at(0))) - { - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - dw, - &beta_t, - w_desc, - dw, - hid_shift + bs * hy_stride, - wei_shift, - wei_shift); - - // Update time - profileRNNkernels(handle, 1, ctime); - } - } - - if(dirMode != 0u) - { - sp_size[1] = 1; - sp_size[2] = wei_len; - w_size[1] = 1; - w_size[2] = wei_len; - w_desc = miopen::TensorDescriptor(rnn_data_t, w_size, w_stride); - sp_desc = miopen::TensorDescriptor(rnn_data_t, sp_size, sp_stride); - - int cur_batch = 0; - for(int ti = 0; ti < seqLen - 1; ti++) - { - for(int bs = 0; bs < in_n.at(ti + 1); bs++) - { - OpTensor(handle, - miopenTensorOpAdd, - &alpha0, - sp_desc, - workSpace, - &alpha1, - w_desc, - dw, - &beta_t, - w_desc, - dw, - hid_shift + (cur_batch + bs) * hy_stride + wei_len, - wei_shift + wei_len, - wei_shift + wei_len); - - // Update time - profileRNNkernels(handle, 1, ctime); - } - cur_batch += in_n.at(ti); - } - } - } - } - - int pretime_shift, hx_shift, cur_time; - bool comb_check = true; - if(seqLen > 2) - { - if(in_n.at(0) != in_n.at(seqLen - 2)) - { - comb_check = false; - } - } - - if(comb_check) - { - hx_shift = li * hy_n * bi_stride; - wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - for(int ri = 0; ri < bi; ri++) - { - hid_shift = - ri == 0 ? li * batch_n * hy_stride - : (li * batch_n * hy_stride + in_n.at(0) * (seqLen - 1) * hy_stride); - cur_time = ri == 0 ? 0 : seqLen - 1; - - if(in_n.at(cur_time) > 0 && hx != nullptr) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - in_n.at(cur_time), - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len, - hx, - hx_shift + ri * hy_n * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - - // Update time - if(li == nLayers - 1 && ri == bi - 1 && seqLen == 1) - profileRNNkernels(handle, 2, ctime); - else - profileRNNkernels(handle, 1, ctime); - } - - if(seqLen > 1) - { - if(ri == 1 && hx != nullptr && in_n.at(0) > in_n.at(seqLen - 1)) - { - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - (in_n.at(0) - in_n.at(seqLen - 1)), - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len - - (in_n.at(0) - in_n.at(seqLen - 1)) * hy_stride, - hx, - hx_shift + ri * hy_n * hy_h + in_n.at(seqLen - 1) * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - - // Update time - profileRNNkernels(handle, 1, ctime); - } - - hid_shift = ri == 0 ? (li * batch_n * hy_stride + in_n.at(0) * hy_stride) - : (li * batch_n * hy_stride); - pretime_shift = - ri == 0 ? li * batch_n * hy_stride + hid_off - : li * batch_n * hy_stride + in_n.at(0) * hy_stride + hid_off; - - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - in_n.at(0) * (seqLen - 2) + in_n.at(seqLen - 1), - hy_stride, - hy_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = CallGemm(handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len, - reserveSpace, - pretime_shift + ri * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - - // Update time - if(li == nLayers - 1 && ri == bi - 1) - profileRNNkernels(handle, 2, ctime); - else - profileRNNkernels(handle, 1, ctime); - } - } - } - else - { - int bacc = 0; - int baccbi = batch_n; - for(int ti = 0; ti < seqLen; ti++) - { - baccbi -= in_n.at(seqLen - 1 - ti); - - hx_shift = li * hy_n * bi_stride; - wei_shift = in_h * wei_stride + li * (bi * hy_h + hy_h) * wei_stride; - - for(int ri = 0; ri < bi; ri++) - { - hid_shift = ri == 0 ? (li * batch_n * hy_stride + bacc * hy_stride) - : (li * batch_n * hy_stride + baccbi * hy_stride); - cur_time = ri == 0 ? ti : seqLen - 1 - ti; - if(ti > 0) - { - pre_batch = - ri == 0 ? bacc - in_n.at(ti - 1) : baccbi + in_n.at(seqLen - 1 - ti); - use_time = ri == 0 ? ti : seqLen - ti; - } - - if(in_n.at(cur_time) > 0) - { - if(ti == 0) - { - if(hx != nullptr) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - in_n.at(cur_time), - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len, - hx, - hx_shift + ri * hy_n * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - // Update time - if(li == nLayers - 1 && ti == seqLen - 1 && ri == bi - 1) - profileRNNkernels(handle, 2, ctime); - else - profileRNNkernels(handle, 1, ctime); - } - } - else - { - if(ri == 1 && hx != nullptr && in_n.at(cur_time) > in_n.at(use_time)) - { - miopen::GemmDescriptor gemm_desc = - GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - (in_n.at(cur_time) - in_n.at(use_time)), - hy_stride, - uni_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = CallGemm( - handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len + in_n.at(use_time) * hy_stride, - hx, - hx_shift + ri * hy_n * hy_h + in_n.at(use_time) * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - // Update time - profileRNNkernels(handle, 1, ctime); - } - - pretime_shift = - li * batch_n * hy_stride + pre_batch * hy_stride + hid_off; - - if(in_n.at(use_time) > 0) - { - miopen::GemmDescriptor gemm_desc = GemmDescriptor{false, - true, - false, - wei_len, - hy_h, - in_n.at(use_time), - hy_stride, - hy_stride, - uni_stride, - 1, // batch count - 0, // Stride A - 0, // Stride B - 0, // Stride C - 1, // alpha - 1, // beta - rnn_data_t, - false}; - - miopenStatus_t gemm_status = - CallGemm(handle, - gemm_desc, - workSpace, - hid_shift + ri * wei_len, - reserveSpace, - pretime_shift + ri * hy_h, - dw, - wei_shift + ri * wei_len * uni_stride, - GemmBackend_t::rocblas); - - checkGemmStatusAndLog(gemm_status); - // Update time - if(li == nLayers - 1 && ti == seqLen - 1 && ri == bi - 1) - profileRNNkernels(handle, 2, ctime); - else - profileRNNkernels(handle, 1, ctime); - } - } - } - } - - bacc += in_n.at(ti); - } - } - } - -#else - (void)handle; - (void)seqLen; - (void)xDesc; - (void)x; - (void)hxDesc; - (void)hx; - (void)dyDesc; - (void)dwDesc; - (void)dw; - (void)workSpace; - (void)workSpaceSize; - (void)reserveSpace; - (void)reserveSpaceSize; - MIOPEN_THROW("GEMM is not supported"); -#endif -}; - -} // namespace miopen diff --git a/src/problem.cpp b/src/problem.cpp index 16e199c002..e69de29bb2 100644 --- a/src/problem.cpp +++ b/src/problem.cpp @@ -1,1191 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace miopen::debug { -/// \todo: This should be updated when a separate driver command is implemented -void LogCmdFindConvolution(const miopen::TensorDescriptor& x, - const miopen::TensorDescriptor& w, - const miopen::ConvolutionDescriptor& conv, - const miopen::TensorDescriptor& y, - miopenProblemDirection_t dir, - std::optional solver_id); - -/// \todo: This should be updated when a separate driver command is implemented -void LogCmdActivation(const miopen::TensorDescriptor& x_desc, - const miopen::ActivationDescriptor& activ_desc, - bool fwd); -} // namespace miopen::debug - -namespace miopen { - -namespace detail { - -// Selected only with empty VariantArgs -template class Visitor, class... VariantArgs> -struct VisitTypeImpl -{ - template - void operator()(int, Args...) - { - MIOPEN_THROW(miopenStatusInvalidValue); - } -}; - -template class Visitor, class VariantArg, class... VariantArgs> -struct VisitTypeImpl -{ - template - void operator()(int id, Args... args) - { - if(i == id) - { - Visitor{args...}(); - return; - } - - VisitTypeImpl{}(id, args...); - } -}; - -template