From cbcdb3fcb6f832e19bd251187c8c27a4c3a0078c Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sat, 27 Jul 2024 12:54:31 +0200 Subject: [PATCH 01/46] treat template arguments as dependencies detect indirect dependency cycles --- README.md | 18 +- scripts/run.sh | 2 +- sdk-dummy/source2gen_user_types.hpp | 95 ++++++++++ src/sdk/sdk.cpp | 268 ++++++++++++++++++++-------- 4 files changed, 304 insertions(+), 79 deletions(-) create mode 100644 sdk-dummy/source2gen_user_types.hpp diff --git a/README.md b/README.md index 6e4e576..3f56d74 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +TOOD: remove + +- add missing types as dummies +- add empty `interfaces` sdk +- sort template classes below their template types + # Source2Gen Source2Gen is a tool to generate Source 2 SDKs. \ @@ -15,6 +21,7 @@ can use the registry to find the game path and set `PATH` automatically. ```sh source2gen +copy .\sdk-dummy\source2gen_user_types.hpp .\sdk # view generated sdk dir .\sdk ``` @@ -23,6 +30,7 @@ dir .\sdk ```sh ./scripts/run.sh "$HOME/.steam/steam/steamapps/cs2/" +pc ./sdk-dummy/source2gen_user_types.hpp ./sdk # view generated sdk ls ./sdk ``` @@ -39,6 +47,14 @@ errors, bugs, and wrong output. Please only file issues if you want to work on them. This note will be removed once we have thoroughly tested Source2Gen on Linux. +### Using the generated SDK + +The sdk depends on a file/module called "source2gen_user_types". This file has +to be provided by the user and expose all types listed in +[source2gen_user_types.hpp](sdk-dummy/source2gen_user_types.hpp). If you don't +intend to access any of these types, you can use the dummy +[source2gen_user_types.hpp](sdk-dummy/source2gen_user_types.hpp). + ## Getting Started These instructions will help you set up the project on your local machine for development and testing purposes. @@ -74,7 +90,7 @@ or You can use premake5 options and specify which game you want to dump: ```bash - --game=CS2 + --game=CS2 Choose a particular game for dumping source 2 sdk; one of: ARTIFACT1 Artifact Classic ARTIFACT2 Artifact Foundry diff --git a/scripts/run.sh b/scripts/run.sh index e0c3cae..7f56ae1 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -31,4 +31,4 @@ if [ -z "${BINARY}" ]; then else set -x LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" "${BINARY}" -fi \ No newline at end of file +fi diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp new file mode 100644 index 0000000..c987341 --- /dev/null +++ b/sdk-dummy/source2gen_user_types.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include + +using CAnimVariant = char[0x14]; +using CBufferString = char[0x10]; +using CColorGradient = char[0x18]; +using CEntityHandle = char[0x04]; +using CEntityIndex = char[0x04]; +using CGlobalSymbol = char[0x08]; +using CKV3MemberNameWithStorage = char[0x38]; +using CNetworkedQuantizedFloat = char[0x08]; +using CParticleNamedValueRef = char[0x40]; +using CPiecewiseCurve = char[0x40]; +using CPlayerSlot = char[0x04]; +using CPulseValueFullType = char[0x10]; +using CResourceName = char[0xe0]; +using CSplitScreenSlot = char[0x04]; +using CTransform = char[0x20]; +using CUtlBinaryBlock = char[0x18]; +using CUtlStringTokenWithStorage = char[0x18]; +using CUtlStringToken = char[0x04]; +using CUtlString = char[0x08]; +using CUtlSymbolLarge = char[0x08]; +using CUtlSymbol = char[0x02]; +template +using CAnimGraphParamOptionalRef = char[0x20]; +template +using CAnimGraphParamRef = char[0x20]; +template +using CBitVec = char[(N + 7) / 8]; +template +using CEntityOutputTemplate = char[0x28]; +template +using CHandle = char[0x04]; +template +using C_NetworkUtlVectorBase = char[0x18]; +template +using CUtlLeanVector = char[0x10]; +template +using CUtlOrderedMap = char[0x28]; +template +using CUtlVector = char[0x18]; +template +using C_UtlVectorEmbeddedNetworkVar = char[0x50]; +using CUtlVectorSIMDPaddedVector = char[0x18]; +template +using CWeakHandle = char[0x18]; +using Color = char[0x04]; +using DegreeEuler = char[0x0c]; +using FourVectors = char[0x30]; +using HSCRIPT = char[0x08]; +using KeyValues3 = char[0x10]; +// size doesn't mapper. only used as a pointer +using KeyValues = char[0x01]; +using QAngle = char[0x0c]; +using QuaternionStorage = char[0x10]; +using Quaternion = char[0x10]; +using RadianEuler = char[0x14]; +// using RenderPrimitiveType_t = char[TOOD]; +using RotationVector = char[0x0c]; +using Vector2D = char[0x08]; +using Vector4D = char[0x10]; +using VectorAligned = char[0x10]; +using Vector = char[0x0c]; +using WorldGroupId_t = char[0x04]; +using float32 = char[0x04]; +using fltx4 = char[0x10]; +using matrix3x4_t = char[0x30]; +using matrix3x4a_t = char[0x30]; +using uint256_t = char[0x20]; + +using uint8 = std::uint8_t; // TOOD: happens in CUtlVector. how to fix? + +// intentionally left undefined. if you want to access static fields, add your own sdk. +namespace interfaces { + struct SchemaStaticFieldData_t { + void* m_pInstance{}; + }; + + struct CSchemaClassInfo { + auto GetStaticFields() -> SchemaStaticFieldData_t**; + }; + + struct CSchemaSystemTypeScope { + auto FindDeclaredClass(std::string_view) -> CSchemaClassInfo*; + }; + + struct schema_t { + auto FindTypeScopeForModule(std::string_view) -> CSchemaSystemTypeScope*; + }; + + extern schema_t* g_schema; +} // namespace interfaces diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index c0b8fd4..9c0371a 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -5,14 +5,16 @@ #include "sdk/sdk.h" #include #include +#include #include #include +#include namespace { using namespace std::string_view_literals; constexpr std::string_view kOutDirName = "sdk"sv; - constinit std::array include_paths = {""sv}; + constinit std::array include_paths = {""sv, "\"source2gen_user_types.hpp\""sv}; constexpr uint32_t kMaxReferencesForClassEmbed = 2; constexpr std::size_t kMinFieldCountForClassEmbed = 2; @@ -142,6 +144,68 @@ namespace { } // namespace namespace sdk { + struct class_t { + CSchemaClassInfo* target_{}; + std::set refs_; + std::uint32_t used_count_{}; + std::list> cached_fields_; + + struct cached_datamap_t { + std::string type_; + std::string name_; + std::ptrdiff_t offset_; + }; + std::list cached_datamap_fields_; + + [[nodiscard]] CSchemaClassInfo* GetParent() const { + if (!target_->m_pBaseClassses) + return nullptr; + + return target_->m_pBaseClassses->m_pClass; + } + + void AddRefToClass(CSchemaType* type) { + if (type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { + refs_.insert(reinterpret_cast(type)->m_pClassInfo); + } + + // auto ptr = type->GetRefClass(); + // if (ptr && ptr->m_nTypeCategory == Schema_DeclaredClass) + // { + // refs_.insert(ptr->m_pClassInfo); + // return; + // } + } + + [[nodiscard]] bool IsDependsOn(const class_t& other) const { + // if current class inherit other. + const auto parent = this->GetParent(); + if (parent == other.target_) + return true; + + // if current class contains ref to other. + if (this->refs_.contains(other.target_)) + return true; + + // otherwise, order doesn`t matter. + return false; + } + + [[nodiscard]] SchemaClassFieldData_t* GetFirstField() const { + if (target_->m_nFieldSize) + return &target_->m_pFields[0]; + return nullptr; + } + + // @note: @es3n1n: Returns the struct size without its parent's size + // + [[nodiscard]] std::ptrdiff_t ClassSizeWithoutParent() const { + if (const CSchemaClassInfo* class_parent = this->GetParent(); class_parent) + return this->target_->m_nSizeOf - class_parent->m_nSizeOf; + return this->target_->m_nSizeOf; + } + }; + namespace { void PrintClassInfo(codegen::generator_t::self_ref builder, CSchemaClassBinding* class_info) { builder.comment(std::format("Alignment: {}", class_info->GetAligment())).comment(std::format("Size: {:#x}", class_info->m_nSizeOf)); @@ -334,133 +398,147 @@ namespace sdk { return {maybe_with_module_name(type->m_pszName), {}}; }; - void AssembleClasses(codegen::generator_t::self_ref builder, const std::unordered_set& classes) { - struct class_t { - CSchemaClassInfo* target_{}; - std::set refs_; - uint32_t used_count_{}; - std::list> cached_fields_; - - struct cached_datamap_t { - std::string type_; - std::string name_; - ptrdiff_t offset_; - }; - std::list cached_datamap_fields_; + // TOOD: move up + struct OrderedClasses { + std::list forward_declarations{}; + std::list classes{}; + }; - [[nodiscard]] CSchemaClassInfo* GetParent() const { - if (!target_->m_pBaseClassses) - return nullptr; + /// Turns e.g. "HashMap>" into ["int", "CUtlVector", "float"] + /// @return An empty list if @p type_name is not a template or has no template parameters + std::vector parse_template_arguments_recursive(std::string_view type_name) { + // TODO: use a library for this once we have a package manager + const auto trim = [](std::string_view str) { + if (const auto found = str.find_first_not_of(" "); found != std::string_view::npos) { + str.remove_prefix(found); + } else { + return std::string_view{}; + } - return target_->m_pBaseClassses->m_pClass; + if (const auto found = str.find_last_not_of(" "); found != std::string_view::npos) { + str.remove_suffix(str.size() - (found + 1)); } - void AddRefToClass(CSchemaType* type) { - if (type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - refs_.insert(reinterpret_cast(type)->m_pClassInfo); - } + return str; + }; + + // TODO: remove + assert(trim("A") == "A"); + assert(trim(" A ") == "A"); + assert(trim(" A") == "A"); + assert(trim("A ") == "A"); + assert(trim(" ") == ""); + assert(trim(" ") == ""); + + const auto split_trim = [trim](std::string_view str, std::string_view separators) -> std::vector { + std::vector result{}; + std::string_view remainder = str; - // auto ptr = type->GetRefClass(); - // if (ptr && ptr->m_nTypeCategory == Schema_DeclaredClass) - // { - // refs_.insert(ptr->m_pClassInfo); - // return; - // } + while (true) { + if (const auto found = remainder.find_first_of(separators); found != std::string_view::npos) { + if (const auto part = trim(remainder.substr(0, found)); !part.empty()) { + result.emplace_back(part); + } + remainder.remove_prefix(found + 1); + } else { + if (const auto part = trim(remainder); !part.empty()) { + result.emplace_back(part); + } + break; + } } - [[nodiscard]] bool IsDependsOn(const class_t& other) const { - // if current class inherit other. - const auto parent = this->GetParent(); - if (parent == other.target_) - return true; + return result; + }; - // if current class contains ref to other. - if (this->refs_.contains(other.target_)) - return true; + if (const auto start = type_name.find("<"); start != std::string_view::npos) { + if (const auto end = type_name.rfind(">"); end != std::string_view::npos) { + const auto raw_list = type_name.substr(start + 1, end - start - 1); - // otherwise, order doesn`t matter. - return false; + return split_trim(raw_list, "<,>"); } + } - [[nodiscard]] SchemaClassFieldData_t* GetFirstField() const { - if (target_->m_nFieldSize) - return &target_->m_pFields[0]; - return nullptr; - } + return {}; + } - // @note: @es3n1n: Returns the struct size without its parent's size - // - [[nodiscard]] std::ptrdiff_t ClassSizeWithoutParent() const { - if (const CSchemaClassInfo* class_parent = this->GetParent(); class_parent) - return this->target_->m_nSizeOf - class_parent->m_nSizeOf; - return this->target_->m_nSizeOf; - } - }; + /** + * Orderes classes such that dependencies are declared or defined befor dependees. + */ + OrderedClasses OrderClasses(const std::unordered_set& classes) { + // TOOD: remove tests + assert(parse_template_arguments_recursive("vector") == std::vector{std::string{"int"}}); + assert(parse_template_arguments_recursive("vector") == std::vector{}); + assert((parse_template_arguments_recursive("map") == std::vector{"int", "float"})); + assert((parse_template_arguments_recursive("A< B< C > >") == std::vector{"B", "C"})); - // @note: @soufiw: - // sort all classes based on refs and inherit, and then print it. - // ================== - std::list classes_to_dump; - bool did_forward_decls = false; + std::list forward_declarations{}; + std::list ordered_classes{}; for (const auto* schema_class_binding : classes) { assert(schema_class_binding != nullptr); const auto class_info = schema_class_binding->m_pTypeScope->FindDeclaredClass(schema_class_binding->m_pszName); - auto& class_dump = classes_to_dump.emplace_back(); + auto& class_dump = ordered_classes.emplace_back(); class_dump.target_ = class_info; } - for (auto& class_dump : classes_to_dump) { + for (auto& class_dump : ordered_classes) { const auto class_info = class_dump.target_; - for (auto k = 0; k < class_info->m_nFieldSize; k++) { - const auto field = &class_info->m_pFields[k]; - if (!field) - continue; - + for (const auto& field : std::span{class_info->m_pFields, static_cast(class_info->m_nFieldSize)}) { // forward declare all classes. // @todo: maybe we need to forward declare only pointers to classes? - auto ptr = field->m_pSchemaType->GetRefClass(); + auto ref_class = field.m_pSchemaType->GetRefClass(); - if (auto actual_type = ptr ? ptr : field->m_pSchemaType; actual_type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - builder.forward_declaration(actual_type->m_pszName); - did_forward_decls = true; + if (auto actual_type = ref_class ? ref_class : field.m_pSchemaType; + actual_type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { + forward_declarations.emplace_back(std::string{actual_type->m_pszName}); } - class_dump.AddRefToClass(field->m_pSchemaType); + class_dump.AddRefToClass(field.m_pSchemaType); - auto field_class = std::ranges::find_if(classes_to_dump, [field](const class_t& cls) { - return cls.target_ == reinterpret_cast(field->m_pSchemaType)->m_pClassInfo; + const auto type_name = std::string_view{field.m_pSchemaType->m_pszName}; + + for (const auto& template_argument : parse_template_arguments_recursive(type_name)) { + if (const auto found = std::ranges::find_if(classes, [=](const auto& el) { return el->m_pszName == template_argument; }); + found != classes.end()) { + class_dump.AddRefToClass((*found)->m_pSchemaType); + } + } + + auto field_class = std::ranges::find_if(ordered_classes, [field](const class_t& cls) { + return cls.target_ == reinterpret_cast(field.m_pSchemaType)->m_pClassInfo; }); - if (field_class != classes_to_dump.end()) + if (field_class != ordered_classes.end()) field_class->used_count_++; } } - if (did_forward_decls) - builder.next_line(); + // to detect inderict dependency cycles + std::set hot_classes{}; + std::set previous_hot_classes{}; bool did_change = false; do { did_change = false; // swap until we done. - for (auto first = classes_to_dump.begin(); first != classes_to_dump.end(); ++first) { + for (auto first = ordered_classes.begin(); first != ordered_classes.end(); ++first) { bool second_below_first = false; - for (auto second = classes_to_dump.begin(); second != classes_to_dump.end(); ++second) { + for (auto second = ordered_classes.begin(); second != ordered_classes.end(); ++second) { if (second == first) { second_below_first = true; continue; } // swap if second class below first, and first depends on second. - bool first_depend = first->IsDependsOn(*second); + const bool first_depend = first->IsDependsOn(*second); // swap if first class below second, and second depends on first. - bool second_depend = second->IsDependsOn(*first); + const bool second_depend = second->IsDependsOn(*first); if (first_depend && second_depend) { // classes depends on each other, forward declare them. @@ -469,13 +547,49 @@ namespace sdk { } if (second_below_first ? first_depend : second_depend) { + hot_classes.emplace(first->target_); + hot_classes.emplace(second->target_); + std::iter_swap(first, second); did_change = true; } } } + + if (did_change && (hot_classes == previous_hot_classes)) { + std::cout << "detected dependency cycle in the following classes\n"; + for (const auto* class_ : hot_classes) { + std::cout << std::format("- {}::{}\n", class_->m_pszModule, class_->m_pszName); + } + std::cout << "you need to resolve the dependency cycle by hand in the generated sdk\n"; + break; + } + std::swap(hot_classes, previous_hot_classes); + hot_classes.clear(); } while (did_change); + + return OrderedClasses{ + .forward_declarations = forward_declarations, + .classes = ordered_classes, + }; + } + + void AssembleClasses(codegen::generator_t::self_ref builder, const std::unordered_set& classes) { + + // @note: @soufiw: + // sort all classes based on refs and inherit, and then print it. // ================== + const auto ordered_classes = OrderClasses(classes); + + for (const auto& type_name : ordered_classes.forward_declarations) { + builder.forward_declaration(type_name); + } + + if (!ordered_classes.forward_declarations.empty()) { + builder.next_line(); + } + + auto classes_to_dump = ordered_classes.classes; for (auto& class_dump : classes_to_dump) { // @note: @es3n1n: get class info, assemble it From ff87134eae276c2f74c5c9a04bb48802b28369db Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sat, 27 Jul 2024 21:01:59 +0200 Subject: [PATCH 02/46] include modules required by template arguments --- README.md | 6 + include/sdk/interfaces/common/CUtlVector.h | 2 + include/sdk/interfaces/schemasystem/schema.h | 21 +-- include/sdk/interfaces/tier0/IMemAlloc.h | 1 + include/tools/field_parser.h | 5 +- include/tools/virtual.h | 9 +- src/sdk/sdk.cpp | 139 ++++++++++++------- 7 files changed, 121 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 3f56d74..a54b390 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ TOOD: remove - add missing types as dummies - add empty `interfaces` sdk - sort template classes below their template types +- detect indirect dependency cycles + +TOOD: + +- rename template arguments to use fully qualified names +- add includes for template arguments (add decompose_type() and use that in many places?) # Source2Gen diff --git a/include/sdk/interfaces/common/CUtlVector.h b/include/sdk/interfaces/common/CUtlVector.h index d29172f..b323d23 100644 --- a/include/sdk/interfaces/common/CUtlVector.h +++ b/include/sdk/interfaces/common/CUtlVector.h @@ -1,7 +1,9 @@ // Copyright (C) 2024 neverlosecc // See end of file for extended copyright information. #pragma once + #include +#include template class CUtlVector { diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index a8ca40d..7ffde48 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -382,7 +383,7 @@ class CSchemaType { } // @todo: @og: find out to what class pointer points. - [[nodiscard]] CSchemaType* GetRefClass(); + [[nodiscard]] CSchemaType* GetRefClass() const; [[nodiscard]] ETypeCategory GetTypeCategory() const { #if defined(CS2) || defined(DOTA2) @@ -421,13 +422,13 @@ class CSchemaType_Ptr : public CSchemaType { CSchemaType* m_pObjectType; }; -[[nodiscard]] inline CSchemaType* CSchemaType::GetRefClass() { +[[nodiscard]] inline CSchemaType* CSchemaType::GetRefClass() const { if (GetTypeCategory() != ETypeCategory::Schema_Ptr) return nullptr; - auto ptr = reinterpret_cast(this)->m_pObjectType; + auto ptr = reinterpret_cast(this)->m_pObjectType; while (ptr && ptr->GetTypeCategory() == ETypeCategory::Schema_Ptr) - ptr = reinterpret_cast(ptr)->m_pObjectType; + ptr = reinterpret_cast(ptr)->m_pObjectType; return ptr; } @@ -772,25 +773,25 @@ class CSchemaSystemTypeScope { return Virtual::Get(this, 1)(this, szName.data(), a2); } - [[nodiscard]] CSchemaClassInfo* FindDeclaredClass(const std::string_view szName) { + [[nodiscard]] CSchemaClassInfo* FindDeclaredClass(const std::string_view szName) const { if constexpr (kSchemaSystemVersion == 2) { CSchemaClassInfo* class_info; - Virtual::Get(this, 2)(this, &class_info, szName.data()); + Virtual::Get(this, 2)(this, &class_info, szName.data()); return class_info; } else { - return Virtual::Get(this, 2)(this, szName.data()); + return Virtual::Get(this, 2)(this, szName.data()); } } - [[nodiscard]] CSchemaEnumInfo* FindDeclaredEnum(const std::string_view szName) { + [[nodiscard]] CSchemaEnumInfo* FindDeclaredEnum(const std::string_view szName) const { if constexpr (kSchemaSystemVersion == 2) { CSchemaEnumInfo* enum_info; - Virtual::Get(this, 3)(this, &enum_info, szName.data()); + Virtual::Get(this, 3)(this, &enum_info, szName.data()); return enum_info; } else { - return Virtual::Get(this, 3)(this, szName.data()); + return Virtual::Get(this, 3)(this, szName.data()); } } diff --git a/include/sdk/interfaces/tier0/IMemAlloc.h b/include/sdk/interfaces/tier0/IMemAlloc.h index 306cb36..f58ffea 100644 --- a/include/sdk/interfaces/tier0/IMemAlloc.h +++ b/include/sdk/interfaces/tier0/IMemAlloc.h @@ -4,6 +4,7 @@ #pragma warning(push) #pragma warning(disable:4191) +#include "tools/platform.h" #include "tools/virtual.h" class IMemAlloc { diff --git a/include/tools/field_parser.h b/include/tools/field_parser.h index 9463feb..286c9bd 100644 --- a/include/tools/field_parser.h +++ b/include/tools/field_parser.h @@ -3,15 +3,14 @@ #pragma once #include +#include #include -enum class fieldtype_t : uint8_t; - namespace field_parser { class field_info_t { public: std::string m_type; // var type - fieldtype_t m_field_type = static_cast(24); // var type + fieldtype_t m_field_type = fieldtype_t::FIELD_UNUSED; // var type std::string m_name; // var name // array sizes, for example {13, 37} for multi demensional array "[13][37]" diff --git a/include/tools/virtual.h b/include/tools/virtual.h index 3109960..76e70ae 100644 --- a/include/tools/virtual.h +++ b/include/tools/virtual.h @@ -2,12 +2,17 @@ // See end of file for extended copyright information. #pragma once -#include +#include namespace Virtual { template inline T Get(void* instance, const unsigned int index) { - return (*static_cast(instance))[index]; + return (*static_cast(static_cast(instance)))[index]; + } + + template + inline T Get(const void* instance, const unsigned int index) { + return (*static_cast(static_cast(instance)))[index]; } template diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 9c0371a..f605799 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -5,9 +5,11 @@ #include "sdk/sdk.h" #include #include +#include #include #include #include +#include #include namespace { @@ -344,19 +346,19 @@ namespace sdk { } /// @return {type_name, array_sizes} - auto parse_array(CSchemaType* type) -> std::pair> { - const auto ptr = type->GetRefClass(); - const auto actual_type = ptr ? ptr : type; + auto parse_array(const CSchemaType& type) -> std::pair> { + const auto* ptr = type.GetRefClass(); + const auto& actual_type = ptr ? *ptr : type; std::string base_type; std::vector sizes; - if (actual_type->GetTypeCategory() == ETypeCategory::Schema_FixedArray) { + if (actual_type.GetTypeCategory() == ETypeCategory::Schema_FixedArray) { // dump all sizes. - auto schema = reinterpret_cast(actual_type); + auto* schema = reinterpret_cast(&actual_type); while (true) { sizes.emplace_back(schema->m_nElementCount); - schema = reinterpret_cast(schema->m_pElementType); + schema = reinterpret_cast(schema->m_pElementType); if (schema->GetTypeCategory() != ETypeCategory::Schema_FixedArray) { base_type = schema->m_pszName; @@ -368,23 +370,30 @@ namespace sdk { return {base_type, sizes}; }; - /// @return @ref std::nullopt if the type is not contained in a module, e.g. because it is a built-in type - auto get_module(CSchemaType* type) -> std::optional { - if (type->m_pTypeScope != nullptr) { - if (const auto* class_ = type->m_pTypeScope->FindDeclaredClass(type->m_pszName)) { - return class_->m_pszModule; - } else if (const auto* enum_ = type->m_pTypeScope->FindDeclaredEnum(type->m_pszName)) { - return enum_->m_pszModule; - } + /// @return @ref std::nullopt if the type is not contained in a module visible in @p scope + auto get_module_of_type_in_scope(const CSchemaSystemTypeScope& scope, std::string_view type_name) -> std::optional { + if (const auto* class_ = scope.FindDeclaredClass(type_name)) { + return class_->m_pszModule; + } else if (const auto* enum_ = scope.FindDeclaredEnum(type_name)) { + return enum_->m_pszModule; + } else { + return std::nullopt; } + } - return std::nullopt; + /// @return @ref std::nullopt if the type is not contained in a module visible in its scope, e.g. because it is a built-in type + auto get_module_of_type(const CSchemaType& type) -> std::optional { + if (type.m_pTypeScope != nullptr) { + return get_module_of_type_in_scope(*type.m_pTypeScope, type.m_pszName); + } else { + return std::nullopt; + } }; /// @return {type_name, array_sizes} where type_name is a fully qualified name - auto get_type(CSchemaType* type) -> std::pair> { + auto get_type(const CSchemaType& type) -> std::pair> { const auto maybe_with_module_name = [type](const auto& type_name) { - return get_module(type) + return get_module_of_type(type) .transform([&](const auto module_name) { return std::format("{}::{}", module_name, type_name); }) .value_or(type_name); }; @@ -395,7 +404,7 @@ namespace sdk { if (!type_name.empty() && !array_sizes.empty()) return {maybe_with_module_name(type_name), array_sizes}; - return {maybe_with_module_name(type->m_pszName), {}}; + return {maybe_with_module_name(type.m_pszName), {}}; }; // TOOD: move up @@ -404,9 +413,12 @@ namespace sdk { std::list classes{}; }; - /// Turns e.g. "HashMap>" into ["int", "CUtlVector", "float"] - /// @return An empty list if @p type_name is not a template or has no template parameters - std::vector parse_template_arguments_recursive(std::string_view type_name) { + // TODO: consider moving these functions to field_parser + /// Decomposes a templated type into its componets, keeping template syntax for later reassembly. + /// e.g. "HashMap>" -> ["HashMap", '<', "int", ',', "Vector", '<', "float", '>', '>'] + /// @return std::string for types, char for syntax (',', '<', '>'). Spaces are removed. + [[nodiscard]] + std::vector> decompose_template(std::string_view type_name) { // TODO: use a library for this once we have a package manager const auto trim = [](std::string_view str) { if (const auto found = str.find_first_not_of(" "); found != std::string_view::npos) { @@ -430,19 +442,21 @@ namespace sdk { assert(trim(" ") == ""); assert(trim(" ") == ""); - const auto split_trim = [trim](std::string_view str, std::string_view separators) -> std::vector { - std::vector result{}; + /// Preserves separators in output. Removes space. + const auto split_trim = [trim](std::string_view str, std::string_view separators) -> std::vector> { + std::vector> result{}; std::string_view remainder = str; while (true) { if (const auto found = remainder.find_first_of(separators); found != std::string_view::npos) { if (const auto part = trim(remainder.substr(0, found)); !part.empty()) { - result.emplace_back(part); + result.emplace_back(std::string{part}); } + result.emplace_back(remainder[found]); remainder.remove_prefix(found + 1); } else { if (const auto part = trim(remainder); !part.empty()) { - result.emplace_back(part); + result.emplace_back(std::string{part}); } break; } @@ -451,26 +465,59 @@ namespace sdk { return result; }; - if (const auto start = type_name.find("<"); start != std::string_view::npos) { - if (const auto end = type_name.rfind(">"); end != std::string_view::npos) { - const auto raw_list = type_name.substr(start + 1, end - start - 1); + return split_trim(type_name, "<,>"); + } - return split_trim(raw_list, "<,>"); + /// e.g. "HashMap>" -> ["HashMap", "int", "CUtlVector", "float"] + /// @return An empty list if @p type_name is not a template or has no template parameters + [[nodiscard]] + std::vector parse_template_recursive(std::string_view type_name) { + assert((decompose_template("vector") == std::vector>{"vector", '<', "int", '>'})); + assert((decompose_template("vector>") == + std::vector>{"vector", '<', "hashmap", '<', "int", ',', "float", '>', '>'})); + assert((decompose_template("int") == std::vector>{"int"})); + + std::vector result{}; + + // remove the topmost type and all syntax entries + for (const auto& el : decompose_template(type_name)) { + if (std::holds_alternative(el)) { + result.emplace_back(std::move(std::get(el))); } } - return {}; + return result; + } + + // TOOD: need to sort these fuctions, they're all over the place + + /// @return All modules directly required to declare this type. Returns multiple + /// modules for template types. + [[nodiscard]] + auto get_modules_of_type(const CSchemaType& type) -> std::unordered_set { + if (type.m_pTypeScope != nullptr) { + std::unordered_set result{}; + for (const auto& type_name : parse_template_recursive(type.m_pszName)) { + if (auto module{get_module_of_type_in_scope(*type.m_pTypeScope, type_name)}) { + result.emplace(std::move(module.value())); + } + } + return result; + } else { + return {}; + } } /** * Orderes classes such that dependencies are declared or defined befor dependees. */ + [[nodiscard]] OrderedClasses OrderClasses(const std::unordered_set& classes) { // TOOD: remove tests - assert(parse_template_arguments_recursive("vector") == std::vector{std::string{"int"}}); - assert(parse_template_arguments_recursive("vector") == std::vector{}); - assert((parse_template_arguments_recursive("map") == std::vector{"int", "float"})); - assert((parse_template_arguments_recursive("A< B< C > >") == std::vector{"B", "C"})); + assert((parse_template_recursive("vector") == std::vector{"vector", "int"})); + assert((parse_template_recursive("vector") == std::vector{"vector"})); + assert((parse_template_recursive("map") == std::vector{"map", "int", "float"})); + assert((parse_template_recursive("A< B< C > >") == std::vector{"A", "B", "C"})); std::list forward_declarations{}; std::list ordered_classes{}; @@ -501,7 +548,7 @@ namespace sdk { const auto type_name = std::string_view{field.m_pSchemaType->m_pszName}; - for (const auto& template_argument : parse_template_arguments_recursive(type_name)) { + for (const auto& template_argument : parse_template_recursive(type_name)) { if (const auto found = std::ranges::find_if(classes, [=](const auto& el) { return el->m_pszName == template_argument; }); found != classes.end()) { class_dump.AddRefToClass((*found)->m_pSchemaType); @@ -671,7 +718,7 @@ namespace sdk { // @note: @es3n1n: parsing type // - const auto [type_name, array_sizes] = get_type(field.m_pSchemaType); + const auto [type_name, array_sizes] = get_type(*field.m_pSchemaType); const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); // @note: @es3n1n: insert padding if needed @@ -802,7 +849,7 @@ namespace sdk { for (auto s = 0; s < class_info->m_nStaticFieldsSize; s++) { auto static_field = &class_info->m_pStaticFields[s]; - auto [type, mod] = get_type(static_field->m_pSchemaType); + auto [type, mod] = get_type(*static_field->m_pSchemaType); const auto var_info = field_parser::parse(type, static_field->m_pszName, mod); builder.static_field_getter(var_info.m_type, var_info.m_name, scope_name, class_info->m_pszName, s); } @@ -859,25 +906,23 @@ namespace sdk { /// @param exclude_module_name will not be contained in the return value /// @return All modules that are required by properties of @p classes std::set find_required_modules_of_class(std::string_view exclude_module_name, const CSchemaClassBinding& class_) { - std::set modules{}; + std::set result{}; for (const auto& field : std::span{class_.m_pFields, static_cast(class_.m_nFieldSize)}) { - if (const auto module = get_module(field.m_pSchemaType)) { - modules.emplace(module.value()); - } + const auto modules = get_modules_of_type(*field.m_pSchemaType); + result.insert(modules.begin(), modules.end()); } for (const auto& field : std::span{class_.m_pStaticFields, static_cast(class_.m_nStaticFieldsSize)}) { - if (const auto module = get_module(field.m_pSchemaType)) { - modules.emplace(module.value()); - } + const auto modules = get_modules_of_type(*field.m_pSchemaType); + result.insert(modules.begin(), modules.end()); } - if (const auto found = modules.find(std::string{exclude_module_name}); found != modules.end()) { - modules.erase(found); + if (const auto found = result.find(std::string{exclude_module_name}); found != result.end()) { + result.erase(found); } - return modules; + return result; } /// @param exclude_module_name will not be contained in the return value From 16d529c9612edba6d200b539f0f1722284945293 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 28 Jul 2024 18:02:15 +0200 Subject: [PATCH 03/46] resolve built-in types in template arguments --- README.md | 1 + include/tools/field_parser.h | 7 ++++ src/sdk/sdk.cpp | 78 +++++++++++++++++++++++++----------- src/tools/field_parser.cpp | 29 +++++++++----- 4 files changed, 81 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a54b390..042f2a5 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ TOOD: remove - add empty `interfaces` sdk - sort template classes below their template types - detect indirect dependency cycles +- add includes to modules of template arguments TOOD: diff --git a/include/tools/field_parser.h b/include/tools/field_parser.h index 286c9bd..7c2c6e8 100644 --- a/include/tools/field_parser.h +++ b/include/tools/field_parser.h @@ -2,9 +2,12 @@ // See end of file for extended copyright information. #pragma once +#include #include +#include #include #include +#include namespace field_parser { class field_info_t { @@ -64,6 +67,10 @@ namespace field_parser { } }; + /// @return @ref std::nullopt if type_name is not a built-in type + [[nodiscard]] + std::optional type_name_to_cpp(std::string_view type_name); + field_info_t parse(const std::string& type_name, const std::string& name, const std::vector& array_sizes); field_info_t parse(const fieldtype_t& type_name, const std::string& name, const std::size_t& array_sizes = 1); } // namespace field_parser diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index f605799..67927a4 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -390,29 +390,6 @@ namespace sdk { } }; - /// @return {type_name, array_sizes} where type_name is a fully qualified name - auto get_type(const CSchemaType& type) -> std::pair> { - const auto maybe_with_module_name = [type](const auto& type_name) { - return get_module_of_type(type) - .transform([&](const auto module_name) { return std::format("{}::{}", module_name, type_name); }) - .value_or(type_name); - }; - const auto [type_name, array_sizes] = parse_array(type); - - assert(type_name.empty() == array_sizes.empty()); - - if (!type_name.empty() && !array_sizes.empty()) - return {maybe_with_module_name(type_name), array_sizes}; - - return {maybe_with_module_name(type.m_pszName), {}}; - }; - - // TOOD: move up - struct OrderedClasses { - std::list forward_declarations{}; - std::list classes{}; - }; - // TODO: consider moving these functions to field_parser /// Decomposes a templated type into its componets, keeping template syntax for later reassembly. /// e.g. "HashMap>" -> ["HashMap", '<', "int", ',', "Vector", '<', "float", '>', '>'] @@ -468,6 +445,59 @@ namespace sdk { return split_trim(type_name, "<,>"); } + /// Adds module qualifiers and resolves built-in types + auto reassemble_retyped_template(const CSchemaSystemTypeScope& scope, + const std::vector>& decomposed) -> std::string { + const auto maybe_with_module_name = [&scope](const std::string_view type_name) { + return get_module_of_type_in_scope(scope, type_name) + .transform([&](const auto module_name) { return std::format("{}::{}", module_name, type_name); }) + .value_or(std::string{type_name}); + }; + + std::string result{}; + + for (const auto& el : decomposed) { + std::visit( + [&](const auto& e) { + if constexpr (std::is_same_v, char>) { + result += e; + } else { + if (const auto built_in = field_parser::type_name_to_cpp(e)) { + result += built_in.value(); + } else { + result += maybe_with_module_name(e); + } + } + }, + el); + } + + // std::cout << "-> " << result << '\n'; + + return result; + } + + /// @return {type_name, array_sizes} where type_name is a fully qualified name + auto get_type(const CSchemaType& type) -> std::pair> { + const auto [type_name, array_sizes] = parse_array(type); + + assert(type_name.empty() == array_sizes.empty()); + + const auto type_name_with_modules = + reassemble_retyped_template(*type.m_pTypeScope, decompose_template(type_name.empty() ? type.m_pszName : type_name)); + + if (!type_name.empty() && !array_sizes.empty()) + return {type_name_with_modules, array_sizes}; + + return {type_name_with_modules, {}}; + }; + + // TOOD: move up + struct OrderedClasses { + std::list forward_declarations{}; + std::list classes{}; + }; + /// e.g. "HashMap>" -> ["HashMap", "int", "CUtlVector", "float"] /// @return An empty list if @p type_name is not a template or has no template parameters [[nodiscard]] @@ -781,7 +811,7 @@ namespace sdk { } // if this prop is a non-pointer class, check if its worth directly embedding the accumulated offset of it into the metadata - auto prop_class = + const auto prop_class = std::ranges::find_if(classes_to_dump, [type_name](const class_t& cls) { return cls.target_->GetName().compare(type_name) == 0; }); if (prop_class != classes_to_dump.end()) { // verify for min/max fields count, we don't want to bloat the dump by embeding too much stuff diff --git a/src/tools/field_parser.cpp b/src/tools/field_parser.cpp index 2b89a79..fb54b08 100644 --- a/src/tools/field_parser.cpp +++ b/src/tools/field_parser.cpp @@ -12,17 +12,17 @@ namespace field_parser { // clang-format off constexpr auto kTypeNameToCpp = std::to_array>({ - {"float32"sv, "float"sv}, + {"float32"sv, "float"sv}, {"float64"sv, "double"sv}, - - {"int8"sv, "int8_t"sv}, - {"int16"sv, "int16_t"sv}, - {"int32"sv, "int32_t"sv}, + + {"int8"sv, "int8_t"sv}, + {"int16"sv, "int16_t"sv}, + {"int32"sv, "int32_t"sv}, {"int64"sv, "int64_t"sv}, - - {"uint8"sv, "uint8_t"sv}, - {"uint16"sv, "uint16_t"sv}, - {"uint32"sv, "uint32_t"sv}, + + {"uint8"sv, "uint8_t"sv}, + {"uint16"sv, "uint16_t"sv}, + {"uint32"sv, "uint32_t"sv}, {"uint64"sv, "uint64_t"sv} }); @@ -123,7 +123,7 @@ namespace field_parser { if (result.m_field_type == fieldtype_t::FIELD_UNUSED) result.m_field_type = type_name; - // @note: @es3n1n: applying kTypeNameToCpp rules + // @note: @es3n1n: applying kDatamapToCpp rules for (auto& rule : kDatamapToCpp) { if (result.m_field_type != rule.first) continue; @@ -134,6 +134,15 @@ namespace field_parser { } } // namespace detail + std::optional type_name_to_cpp(std::string_view type_name) { + if (const auto found = std::ranges::find(detail::kTypeNameToCpp, type_name, &decltype(detail::kTypeNameToCpp)::value_type::first); + found != detail::kTypeNameToCpp.end()) { + return found->second; + } else { + return std::nullopt; + } + } + field_info_t parse(const std::string& type_name, const std::string& name, const std::vector& array_sizes) { field_info_t result = {}; result.m_name = name; From 8e0d25a52f1e71ad32b314fbd2b99014805411dc Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 29 Jul 2024 18:34:09 +0200 Subject: [PATCH 04/46] MPulseCellOutflowHookInfo is string metadata --- README.md | 5 ++- scripts/run.sh | 2 +- sdk-dummy/CMakeLists.txt | 28 +++++++++++++ src/sdk/sdk.cpp | 85 ++++++++++++++++++++-------------------- 4 files changed, 75 insertions(+), 45 deletions(-) create mode 100644 sdk-dummy/CMakeLists.txt diff --git a/README.md b/README.md index 042f2a5..97725d9 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ TOOD: remove - sort template classes below their template types - detect indirect dependency cycles - add includes to modules of template arguments +- rename template arguments to use fully qualified names +- add includes for template arguments (add decompose_type() and use that in many places?) TOOD: -- rename template arguments to use fully qualified names -- add includes for template arguments (add decompose_type() and use that in many places?) +- se why the sdk doesn't compile # Source2Gen diff --git a/scripts/run.sh b/scripts/run.sh index 7f56ae1..2b77a46 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -30,5 +30,5 @@ if [ -z "${BINARY}" ]; then exit 1 else set -x - LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" "${BINARY}" + LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" lldb -o r -- "${BINARY}" fi diff --git a/sdk-dummy/CMakeLists.txt b/sdk-dummy/CMakeLists.txt new file mode 100644 index 0000000..a230e65 --- /dev/null +++ b/sdk-dummy/CMakeLists.txt @@ -0,0 +1,28 @@ +# TOOD: file should not be here + +cmake_minimum_required(VERSION 3.30) + +project(source2sdk + LANGUAGES CXX +) + +file(GLOB_RECURSE source2sdk_headers "./**.hpp") + +foreach(el ${source2sdk_headers}) + string(APPEND generated_cpp_contents "#include \"${el}\"\n") +endforeach() + +set(generated_cpp_file "${CMAKE_BINARY_DIR}/all_headers.cpp") + +file(WRITE ${generated_cpp_file} ${generated_cpp_contents}) + +add_library(${PROJECT_NAME} ${generated_cpp_file}) + +target_compile_options(${PROJECT_NAME} PRIVATE + "-Wfatal-errors" + "-pedantic-errors" +) + +target_precompile_headers(${PROJECT_NAME} PRIVATE ${source2sdk_sources}) + +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 67927a4..5e5da47 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -23,49 +23,50 @@ namespace { constexpr std::size_t kMaxFieldCountForClassEmbed = 12; constinit std::array string_metadata_entries = { - FNV32("MNetworkChangeCallback"), - FNV32("MPropertyFriendlyName"), - FNV32("MPropertyDescription"), - FNV32("MPropertyAttributeRange"), - FNV32("MPropertyStartGroup"), - FNV32("MPropertyAttributeChoiceName"), - FNV32("MPropertyGroupName"), - FNV32("MNetworkUserGroup"), - FNV32("MNetworkAlias"), - FNV32("MNetworkTypeAlias"), - FNV32("MNetworkSerializer"), - FNV32("MPropertyAttributeEditor"), - FNV32("MPropertySuppressExpr"), - FNV32("MKV3TransferName"), + FNV32("MCellForDomain"), + FNV32("MCustomFGDMetadata"), FNV32("MFieldVerificationName"), - FNV32("MVectorIsSometimesCoordinate"), + FNV32("MKV3TransferName"), + FNV32("MNetworkAlias"), + FNV32("MNetworkChangeCallback"), FNV32("MNetworkEncoder"), - FNV32("MPropertyCustomFGDType"), - FNV32("MPropertyCustomEditor"), - FNV32("MVDataUniqueMonotonicInt"), - FNV32("MScriptDescription"), - FNV32("MPropertyAttributeSuggestionName"), - FNV32("MPropertyIconName"), - FNV32("MVDataOutlinerIcon"), - FNV32("MPropertyExtendedEditor"), - FNV32("MParticleReplacementOp"), - FNV32("MCustomFGDMetadata"), - FNV32("MCellForDomain"), - FNV32("MSrc1ImportDmElementType"), - FNV32("MSrc1ImportAttributeName"), - FNV32("MResourceBlockType"), - FNV32("MVDataOutlinerIconExpr"), - FNV32("MPropertyArrayElementNameKey"), - FNV32("MPropertyFriendlyName"), - FNV32("MPropertyDescription"), FNV32("MNetworkExcludeByName"), FNV32("MNetworkExcludeByUserGroup"), FNV32("MNetworkIncludeByName"), FNV32("MNetworkIncludeByUserGroup"), - FNV32("MNetworkUserGroupProxy"), FNV32("MNetworkReplayCompatField"), - FNV32("MPulseProvideFeatureTag"), + FNV32("MNetworkSerializer"), + FNV32("MNetworkTypeAlias"), + FNV32("MNetworkUserGroup"), + FNV32("MNetworkUserGroupProxy"), + FNV32("MParticleReplacementOp"), + FNV32("MPropertyArrayElementNameKey"), + FNV32("MPropertyAttributeChoiceName"), + FNV32("MPropertyAttributeEditor"), + FNV32("MPropertyAttributeRange"), + FNV32("MPropertyAttributeSuggestionName"), + FNV32("MPropertyCustomEditor"), + FNV32("MPropertyCustomFGDType"), + FNV32("MPropertyDescription"), + FNV32("MPropertyDescription"), + FNV32("MPropertyExtendedEditor"), + FNV32("MPropertyFriendlyName"), + FNV32("MPropertyFriendlyName"), + FNV32("MPropertyGroupName"), + FNV32("MPropertyIconName"), + FNV32("MPropertyStartGroup"), + FNV32("MPropertySuppressExpr"), + FNV32("MPulseCellOutflowHookInfo"), FNV32("MPulseEditorHeaderIcon"), + FNV32("MPulseProvideFeatureTag"), + FNV32("MResourceBlockType"), + FNV32("MScriptDescription"), + FNV32("MSrc1ImportAttributeName"), + FNV32("MSrc1ImportDmElementType"), + FNV32("MVDataOutlinerIcon"), + FNV32("MVDataOutlinerIconExpr"), + FNV32("MVDataUniqueMonotonicInt"), + FNV32("MVectorIsSometimesCoordinate"), }; constinit std::array string_class_metadata_entries = { @@ -75,7 +76,7 @@ namespace { constinit std::array var_name_string_class_metadata_entries = { FNV32("MNetworkVarNames"), FNV32("MNetworkOverride"), FNV32("MNetworkVarTypeOverride"), - FNV32("MPulseCellOutflowHookInfo"), FNV32("MScriptDescription"), FNV32("MParticleDomainTag"), + FNV32("MScriptDescription"), FNV32("MParticleDomainTag"), }; constinit std::array integer_metadata_entries = { @@ -101,7 +102,7 @@ namespace { // @note: @es3n1n: some more utils // - std::string GetMetadataValue(const SchemaMetadataEntryData_t metadata_entry) { + std::string GetMetadataValue(const SchemaMetadataEntryData_t& metadata_entry) { std::string value; const auto value_hash_name = fnv32::hash_runtime(metadata_entry.m_szName); @@ -247,9 +248,9 @@ namespace sdk { for (const auto& metadata : class_info->GetStaticMetadata()) { if (const auto value = GetMetadataValue(metadata); !value.empty()) - builder.comment(std::format("{} \"{}\"", metadata.m_szName, value)); + builder.comment(std::format("static metadata: {} \"{}\"", metadata.m_szName, value)); else - builder.comment(metadata.m_szName); + builder.comment(std::format("static metadata: {}", metadata.m_szName)); } } @@ -262,7 +263,7 @@ namespace sdk { builder.comment(""); for (const auto& metadata : enum_binding->GetStaticMetadata()) { - builder.comment(metadata.m_szName); + builder.comment(std::format("metadata: {}", metadata.m_szName)); } } @@ -805,9 +806,9 @@ namespace sdk { auto field_metadata = field.m_pMetadata[j]; if (auto data = GetMetadataValue(field_metadata); data.empty()) - builder.comment(field_metadata.m_szName); + builder.comment(std::format("metadata: {}", field_metadata.m_szName)); else - builder.comment(std::format("{} \"{}\"", field_metadata.m_szName, data)); + builder.comment(std::format("metadata: {} \"{}\"", field_metadata.m_szName, data)); } // if this prop is a non-pointer class, check if its worth directly embedding the accumulated offset of it into the metadata From 73eca2330988ea45bf50a9fe0d7d247e7cfdb5ce Mon Sep 17 00:00:00 2001 From: Cre3per Date: Tue, 30 Jul 2024 11:07:35 +0200 Subject: [PATCH 05/46] escape and decay - escape class names - decay type names for lookup --- README.md | 7 +- include/sdk/interfaces/schemasystem/schema.h | 15 +++- include/tools/codegen.h | 22 +++--- sdk-dummy/source2gen_user_types.hpp | 28 +++++++- src/sdk/sdk.cpp | 75 +++++++++++++++----- 5 files changed, 112 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 97725d9..62b8bc4 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,19 @@ TOOD: remove - add missing types as dummies - add empty `interfaces` sdk - sort template classes below their template types +- decay arrays and pointers when sorting classes by dependencies - detect indirect dependency cycles - add includes to modules of template arguments - rename template arguments to use fully qualified names - add includes for template arguments (add decompose_type() and use that in many places?) +- escape class names TOOD: -- se why the sdk doesn't compile +- rename nested types to use underscores +- see why the sdk doesn't compile +- generate static assertions +- remove all double underscores # Source2Gen diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 7ffde48..4630992 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -639,13 +640,13 @@ static_assert(offsetof(SchemaClassInfoData_t, m_pFn) == 0x68); class CSchemaClassInfo : public SchemaClassInfoData_t { public: - [[nodiscard]] std::string_view GetName() { + [[nodiscard]] std::string_view GetName() const { if (m_pszName) return {m_pszName}; return {}; } - [[nodiscard]] std::string_view GetModule() { + [[nodiscard]] std::string_view GetModule() const { if (m_pszModule) return {m_pszModule}; return {}; @@ -766,14 +767,20 @@ static_assert(sizeof(CSchemaPtrMap) == platform_specific{.windows = 0x class CSchemaSystemTypeScope { public: void* InsertNewClassBinding(const std::string_view szName, void* a2) { + assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); + return Virtual::Get(this, 0)(this, szName.data(), a2); } void* InsertNewEnumBinding(const std::string_view szName, void* a2) { + assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); + return Virtual::Get(this, 1)(this, szName.data(), a2); } [[nodiscard]] CSchemaClassInfo* FindDeclaredClass(const std::string_view szName) const { + assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); + if constexpr (kSchemaSystemVersion == 2) { CSchemaClassInfo* class_info; @@ -785,6 +792,8 @@ class CSchemaSystemTypeScope { } [[nodiscard]] CSchemaEnumInfo* FindDeclaredEnum(const std::string_view szName) const { + assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); + if constexpr (kSchemaSystemVersion == 2) { CSchemaEnumInfo* enum_info; @@ -796,6 +805,8 @@ class CSchemaSystemTypeScope { } [[nodiscard]] CSchemaType* FindSchemaTypeByName(const std::string_view szName) { + assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); + if constexpr (kSchemaSystemVersion == 2) { CSchemaType* schema_type; diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 5800532..4c2a8ba 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -132,14 +132,14 @@ namespace codegen { } self_ref begin_class(const std::string& class_name, const std::string& access_modifier = "public") { - return begin_block(std::format("class {}", class_name), access_modifier); + return begin_block(std::format("class {}", escape_name(class_name)), access_modifier); } self_ref begin_class_with_base_type(const std::string& class_name, const std::string& base_type, const std::string& access_modifier = "public") { if (base_type.empty()) - return begin_class(std::cref(class_name), access_modifier); - - return begin_block(std::format("class {} : public {}", class_name, base_type), access_modifier); + return begin_class(class_name, access_modifier); + else + return begin_block(std::format("class {} : public {}", escape_name(class_name), escape_name(base_type)), access_modifier); } self_ref end_class() { @@ -183,8 +183,8 @@ namespace codegen { } // @todo: @es3n1n: add func params - self_ref begin_function(const std::string& prefix, const std::string& type_name, const std::string& func_name, const bool increment_tabs_count = true, - const bool move_cursor_to_next_line = true) { + self_ref begin_function(const std::string& prefix, const std::string& type_name, const std::string& func_name, + const bool increment_tabs_count = true, const bool move_cursor_to_next_line = true) { return begin_block(std::format("{}{} {}()", prefix, type_name, escape_name(func_name)), "", increment_tabs_count, move_cursor_to_next_line); } @@ -246,7 +246,7 @@ namespace codegen { if (is_private_field) type_name = "[[maybe_unused]] " + type_name; - auto pad_name = pad_offset.has_value() ? std::format("__pad{:04x}", pad_offset.value()) : std::format("__pad{:d}", _pads_count++); + auto pad_name = pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value()) : std::format("pad_{:d}", _pads_count++); if (!bitfield_size) pad_name = pad_name + std::format("[{:#x}]", padding_size); @@ -265,12 +265,11 @@ namespace codegen { } self_ref begin_bitfield_block() { - return begin_struct("", ""); + return comment("start of bitfield block"); } self_ref end_bitfield_block(const bool move_cursor_to_next_line = true) { - dec_tabs_count(1); - return push_line(move_cursor_to_next_line ? "};" : "}; ", move_cursor_to_next_line); + return comment(std::format("end of bitfield block{}", move_cursor_to_next_line ? "" : " "), move_cursor_to_next_line); } public: @@ -293,8 +292,7 @@ namespace codegen { result.resize(name.size()); for (std::size_t i = 0; i < name.size(); i++) - result[i] = - std::ranges::find(kBlacklistedCharacters, name[i]) == std::end(kBlacklistedCharacters) ? name[i] : '_'; + result[i] = std::ranges::find(kBlacklistedCharacters, name[i]) == std::end(kBlacklistedCharacters) ? name[i] : '_'; return result; } diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index c987341..1347f96 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -3,9 +3,14 @@ #include #include +template +using CAnimValue = char[0x08]; using CAnimVariant = char[0x14]; using CBufferString = char[0x10]; using CColorGradient = char[0x18]; +// size doesn't mapper. only used as a pointer +template +using CCompressor = char[0x01]; using CEntityHandle = char[0x04]; using CEntityIndex = char[0x04]; using CGlobalSymbol = char[0x08]; @@ -19,6 +24,8 @@ using CResourceName = char[0xe0]; using CSplitScreenSlot = char[0x04]; using CTransform = char[0x20]; using CUtlBinaryBlock = char[0x18]; +template +using CUtlHashtable = char[0x20]; using CUtlStringTokenWithStorage = char[0x18]; using CUtlStringToken = char[0x04]; using CUtlString = char[0x08]; @@ -40,12 +47,28 @@ template using CUtlLeanVector = char[0x10]; template using CUtlOrderedMap = char[0x28]; +// size doesn't mapper. only used as a pointer +template +using CUtlPair = char[0x01]; template using CUtlVector = char[0x18]; +// size is a guess that fits both occurences of this type in CS2 +template +using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; template using C_UtlVectorEmbeddedNetworkVar = char[0x50]; using CUtlVectorSIMDPaddedVector = char[0x18]; template +using CSmartPtr = char[0x08]; +template +using CResourceNameTyped = char[0xe0]; +template +using CStrongHandle = char[0x08]; +template +using CStrongHandleCopyable = char[0x08]; +// size doesn't mapper. only used as a pointer +using CStrongHandleVoid = char[0x08]; +template using CWeakHandle = char[0x18]; using Color = char[0x04]; using DegreeEuler = char[0x0c]; @@ -58,8 +81,11 @@ using QAngle = char[0x0c]; using QuaternionStorage = char[0x10]; using Quaternion = char[0x10]; using RadianEuler = char[0x14]; -// using RenderPrimitiveType_t = char[TOOD]; +// we don't have a field size for this type. uses the fallback of 1. +using RenderPrimitiveType_t = char[0x01]; using RotationVector = char[0x0c]; +template +using SphereBase_t = char[0x10]; using Vector2D = char[0x08]; using Vector4D = char[0x10]; using VectorAligned = char[0x10]; diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 5e5da47..3dd6ad5 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -75,8 +75,7 @@ namespace { }; constinit std::array var_name_string_class_metadata_entries = { - FNV32("MNetworkVarNames"), FNV32("MNetworkOverride"), FNV32("MNetworkVarTypeOverride"), - FNV32("MScriptDescription"), FNV32("MParticleDomainTag"), + FNV32("MNetworkVarNames"), FNV32("MNetworkOverride"), FNV32("MNetworkVarTypeOverride"), FNV32("MScriptDescription"), FNV32("MParticleDomainTag"), }; constinit std::array integer_metadata_entries = { @@ -373,9 +372,14 @@ namespace sdk { /// @return @ref std::nullopt if the type is not contained in a module visible in @p scope auto get_module_of_type_in_scope(const CSchemaSystemTypeScope& scope, std::string_view type_name) -> std::optional { - if (const auto* class_ = scope.FindDeclaredClass(type_name)) { + // resolve pointers to their actual type + while (type_name.ends_with('*')) { + type_name.remove_suffix(1); + } + + if (const auto* class_ = scope.FindDeclaredClass(std::string{type_name})) { return class_->m_pszModule; - } else if (const auto* enum_ = scope.FindDeclaredEnum(type_name)) { + } else if (const auto* enum_ = scope.FindDeclaredEnum(std::string{type_name})) { return enum_->m_pszModule; } else { return std::nullopt; @@ -446,13 +450,31 @@ namespace sdk { return split_trim(type_name, "<,>"); } + // TOOD: use a library + [[nodiscard]] + auto string_replace(std::string str, std::string_view search, std::string_view replace) -> std::string { + const std::size_t pos = str.find(search); + if (pos != std::string::npos) { + str.replace(pos, search.length(), replace); + } + return str; + } + /// Adds module qualifiers and resolves built-in types auto reassemble_retyped_template(const CSchemaSystemTypeScope& scope, const std::vector>& decomposed) -> std::string { const auto maybe_with_module_name = [&scope](const std::string_view type_name) { + // This is a hack to support nested types. + // When we define nested types, they're not actually nested, but contain their outer class' name in their name, + // e.g. "struct Player { struct Hand {}; };" is emitted as + // "struct Player {}; struct Player__Hand{};". + // But when used as a property, types expect `Hand` in `Player`, i.e. `Player::Hand m_hand;` + // Instead of doing this hackery, we should probably declare nested classes as nested classes. + const auto escaped_type_name = string_replace(std::string{type_name}, "::", "__"); + return get_module_of_type_in_scope(scope, type_name) - .transform([&](const auto module_name) { return std::format("{}::{}", module_name, type_name); }) - .value_or(std::string{type_name}); + .transform([&](const auto module_name) { return std::format("{}::{}", module_name, escaped_type_name); }) + .value_or(std::string{escaped_type_name}); }; std::string result{}; @@ -550,6 +572,20 @@ namespace sdk { assert((parse_template_recursive("map") == std::vector{"map", "int", "float"})); assert((parse_template_recursive("A< B< C > >") == std::vector{"A", "B", "C"})); + /// @return Lifetime is bound to string viewed by @p type_name + const auto decay_type_name = [](std::string_view type_name) -> std::string_view { + if (const auto found = type_name.find('['); found != std::string_view::npos) { + // "array[123]" -> "array" + type_name = type_name.substr(0, found); + } + if (const auto found = type_name.find('*'); found != std::string_view::npos) { + // "pointer***" -> "pointer" + type_name = type_name.substr(0, found); + } + + return type_name; + }; + std::list forward_declarations{}; std::list ordered_classes{}; @@ -563,24 +599,24 @@ namespace sdk { } for (auto& class_dump : ordered_classes) { - const auto class_info = class_dump.target_; + const auto& class_info = *class_dump.target_; - for (const auto& field : std::span{class_info->m_pFields, static_cast(class_info->m_nFieldSize)}) { - // forward declare all classes. - // @todo: maybe we need to forward declare only pointers to classes? - auto ref_class = field.m_pSchemaType->GetRefClass(); + // forward declare all classes that are not nested. nested classes cannot be forward declared. + // @todo: maybe we need to forward declare only pointers to classes? + if (!class_info.GetName().contains("::")) { + forward_declarations.emplace_back(class_info.GetName()); + } - if (auto actual_type = ref_class ? ref_class : field.m_pSchemaType; - actual_type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - forward_declarations.emplace_back(std::string{actual_type->m_pszName}); - } + for (const auto& field : std::span{class_info.m_pFields, static_cast(class_info.m_nFieldSize)}) { + auto ref_class = field.m_pSchemaType->GetRefClass(); class_dump.AddRefToClass(field.m_pSchemaType); - const auto type_name = std::string_view{field.m_pSchemaType->m_pszName}; + const auto full_type_name = decay_type_name(std::string_view{field.m_pSchemaType->m_pszName}); - for (const auto& template_argument : parse_template_recursive(type_name)) { - if (const auto found = std::ranges::find_if(classes, [=](const auto& el) { return el->m_pszName == template_argument; }); + // if `full_type_name` is `Map`, this loop runs [`Map`, `int`, 'float`] + for (const auto& type_name : parse_template_recursive(full_type_name)) { + if (const auto found = std::ranges::find_if(classes, [=](const auto& el) { return el->m_pszName == type_name; }); found != classes.end()) { class_dump.AddRefToClass((*found)->m_pSchemaType); } @@ -744,8 +780,9 @@ namespace sdk { } // @note: @es3n1n: obtaining size + // fall back to 1 because there are no 0-sized types // - const int field_size = field.m_pSchemaType->GetSize().value_or(0); + const int field_size = field.m_pSchemaType->GetSize().value_or(1); // @note: @es3n1n: parsing type // From 5b5592455680b303fa8aaf21778e5af0375ccd1f Mon Sep 17 00:00:00 2001 From: Cre3per Date: Wed, 31 Jul 2024 20:12:52 +0200 Subject: [PATCH 06/46] dump one type per file - not done: generate includes --- include/sdk/interfaces/schemasystem/schema.h | 7 +- scripts/run.sh | 7 +- sdk-dummy/source2gen_user_types.hpp | 4 + src/sdk/sdk.cpp | 731 ++++++++++--------- 4 files changed, 407 insertions(+), 342 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 4630992..214d6f7 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -658,15 +658,16 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { return std::nullopt; } - [[nodiscard]] std::vector GetFields() { + // TODO: when we have AddressSanitizer, try returning std::span instead of std::vector. Repeats for other functions. + [[nodiscard]] std::vector GetFields() const { return {m_pFields, m_pFields + m_nFieldSize}; } - [[nodiscard]] std::vector GetStaticFields() { + [[nodiscard]] std::vector GetStaticFields() const { return {m_pStaticFields, m_pStaticFields + m_nStaticFieldsSize}; } - [[nodiscard]] std::vector GetStaticMetadata() { + [[nodiscard]] std::vector GetStaticMetadata() const { return {m_pStaticMetadata, m_pStaticMetadata + m_nStaticMetadataSize}; } diff --git a/scripts/run.sh b/scripts/run.sh index 2b77a46..a0f7ea9 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -29,6 +29,11 @@ if [ -z "${BINARY}" ]; then echo "source2gen binary not found. set LD_PRELOAD_PATH and run source2gen by hand." exit 1 else + export LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" set -x - LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" lldb -o r -- "${BINARY}" + if [ -z "${DEBUGGER}" ]; then + "${BINARY}" + else + "${DEBUGGER}" -- "${BINARY}" + fi fi diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index 1347f96..8ed7f3f 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -55,6 +55,8 @@ using CUtlVector = char[0x18]; // size is a guess that fits both occurences of this type in CS2 template using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; +template +using CUtlLeanVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; template using C_UtlVectorEmbeddedNetworkVar = char[0x50]; using CUtlVectorSIMDPaddedVector = char[0x18]; @@ -69,6 +71,8 @@ using CStrongHandleCopyable = char[0x08]; // size doesn't mapper. only used as a pointer using CStrongHandleVoid = char[0x08]; template +using CVariantBase = char[0x10]; +template using CWeakHandle = char[0x18]; using Color = char[0x04]; using DegreeEuler = char[0x0c]; diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 3dd6ad5..1453982 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -146,18 +146,16 @@ namespace { } // namespace namespace sdk { + struct cached_datamap_t { + std::string type_; + std::string name_; + std::ptrdiff_t offset_; + }; + struct class_t { CSchemaClassInfo* target_{}; std::set refs_; std::uint32_t used_count_{}; - std::list> cached_fields_; - - struct cached_datamap_t { - std::string type_; - std::string name_; - std::ptrdiff_t offset_; - }; - std::list cached_datamap_fields_; [[nodiscard]] CSchemaClassInfo* GetParent() const { if (!target_->m_pBaseClassses) @@ -209,43 +207,43 @@ namespace sdk { }; namespace { - void PrintClassInfo(codegen::generator_t::self_ref builder, CSchemaClassBinding* class_info) { - builder.comment(std::format("Alignment: {}", class_info->GetAligment())).comment(std::format("Size: {:#x}", class_info->m_nSizeOf)); + void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_info) { + builder.comment(std::format("Alignment: {}", class_info.GetAligment())).comment(std::format("Size: {:#x}", class_info.m_nSizeOf)); - if ((class_info->m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable + if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable builder.comment("Has VTable"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_IS_ABSTRACT) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_IS_ABSTRACT) != 0) builder.comment("Is Abstract"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_CONSTRUCTOR) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_CONSTRUCTOR) != 0) builder.comment("Has Trivial Constructor"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_DESTRUCTOR) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_DESTRUCTOR) != 0) builder.comment("Has Trivial Destructor"); #if defined(CS2) || defined(DOTA2) - if ((class_info->m_nClassFlags & SCHEMA_CF1_CONSTRUCT_ALLOWED) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_ALLOWED) != 0) builder.comment("Construct allowed"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_CONSTRUCT_DISALLOWED) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_DISALLOWED) != 0) builder.comment("Construct disallowed"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MConstructibleClassBase) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MConstructibleClassBase) != 0) builder.comment("MConstructibleClassBase"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasCustomAlignedNewDelete) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasCustomAlignedNewDelete) != 0) builder.comment("MClassHasCustomAlignedNewDelete"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasEntityLimitedDataDesc) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasEntityLimitedDataDesc) != 0) builder.comment("MClassHasEntityLimitedDataDesc"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MDisableDataDescValidation) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MDisableDataDescValidation) != 0) builder.comment("MDisableDataDescValidation"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MIgnoreTypeScopeMetaChecks) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MIgnoreTypeScopeMetaChecks) != 0) builder.comment("MIgnoreTypeScopeMetaChecks"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkNoBase) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkNoBase) != 0) builder.comment("MNetworkNoBase"); - if ((class_info->m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkAssumeNotNetworkable) != 0) + if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkAssumeNotNetworkable) != 0) builder.comment("MNetworkAssumeNotNetworkable"); #endif - if (class_info->m_nStaticMetadataSize > 0) + if (class_info.m_nStaticMetadataSize > 0) builder.comment(""); - for (const auto& metadata : class_info->GetStaticMetadata()) { + for (const auto& metadata : class_info.GetStaticMetadata()) { if (const auto value = GetMetadataValue(metadata); !value.empty()) builder.comment(std::format("static metadata: {} \"{}\"", metadata.m_szName, value)); else @@ -253,95 +251,99 @@ namespace sdk { } } - void PrintEnumInfo(codegen::generator_t::self_ref builder, const CSchemaEnumBinding* enum_binding) { - builder.comment(std::format("Enumerator count: {}", enum_binding->m_nEnumeratorCount)) - .comment(std::format("Alignment: {}", enum_binding->m_unAlignOf)) - .comment(std::format("Size: {:#x}", enum_binding->m_unSizeOf)); + void PrintEnumInfo(codegen::generator_t::self_ref builder, const CSchemaEnumBinding& enum_binding) { + builder.comment(std::format("Enumerator count: {}", enum_binding.m_nEnumeratorCount)) + .comment(std::format("Alignment: {}", enum_binding.m_unAlignOf)) + .comment(std::format("Size: {:#x}", enum_binding.m_unSizeOf)); - if (enum_binding->m_nStaticMetadataSize > 0) + if (enum_binding.m_nStaticMetadataSize > 0) builder.comment(""); - for (const auto& metadata : enum_binding->GetStaticMetadata()) { + for (const auto& metadata : enum_binding.GetStaticMetadata()) { builder.comment(std::format("metadata: {}", metadata.m_szName)); } } - void AssembleEnums(codegen::generator_t::self_ref builder, const std::unordered_set& enums) { - for (auto schema_enum_binding : enums) { - // @note: @es3n1n: get type name by align size - // - const auto get_type_name = [schema_enum_binding]() -> std::string { - std::string type_storage; + void AssembleEnum(codegen::generator_t::self_ref builder, const CSchemaEnumBinding& schema_enum_binding) { + // @note: @es3n1n: get type name by align size + // + const auto get_type_name = [&schema_enum_binding]() -> std::string { + std::string type_storage; - switch (schema_enum_binding->m_unAlignOf) { - case 1: - type_storage = "std::uint8_t"; - break; - case 2: - type_storage = "std::uint16_t"; - break; - case 4: - type_storage = "std::uint32_t"; - break; - case 8: - type_storage = "std::uint64_t"; - break; - default: - type_storage = "INVALID_TYPE"; - } + switch (schema_enum_binding.m_unAlignOf) { + case 1: + type_storage = "std::uint8_t"; + break; + case 2: + type_storage = "std::uint16_t"; + break; + case 4: + type_storage = "std::uint32_t"; + break; + case 8: + type_storage = "std::uint64_t"; + break; + default: + type_storage = "INVALID_TYPE"; + } - return type_storage; - }; + return type_storage; + }; - // @note: @es3n1n: print meta info - // - PrintEnumInfo(builder, schema_enum_binding); + // @note: @es3n1n: print meta info + // + PrintEnumInfo(builder, schema_enum_binding); - // @note: @es3n1n: begin enum class - // - builder.begin_enum_class(schema_enum_binding->m_pszName, get_type_name()); + // @note: @es3n1n: begin enum class + // + builder.begin_enum_class(schema_enum_binding.m_pszName, get_type_name()); - // @note: @og: build max based on numeric_limits of unAlignOf - // - const auto print_enum_item = [schema_enum_binding, &builder](const SchemaEnumeratorInfoData_t& field) { - switch (schema_enum_binding->m_unAlignOf) { - case 1: - builder.enum_item(field.m_szName, field.m_uint8); - break; - case 2: - builder.enum_item(field.m_szName, field.m_uint16); - break; - case 4: - builder.enum_item(field.m_szName, field.m_uint32); - break; - case 8: - builder.enum_item(field.m_szName, field.m_uint64); - break; - default: - builder.enum_item(field.m_szName, field.m_uint64); - } - }; + // @note: @og: build max based on numeric_limits of unAlignOf + // + const auto print_enum_item = [schema_enum_binding, &builder](const SchemaEnumeratorInfoData_t& field) { + switch (schema_enum_binding.m_unAlignOf) { + case 1: + builder.enum_item(field.m_szName, field.m_uint8); + break; + case 2: + builder.enum_item(field.m_szName, field.m_uint16); + break; + case 4: + builder.enum_item(field.m_szName, field.m_uint32); + break; + case 8: + builder.enum_item(field.m_szName, field.m_uint64); + break; + default: + builder.enum_item(field.m_szName, field.m_uint64); + } + }; - // @note: @es3n1n: assemble enum items + // @note: @es3n1n: assemble enum items + // + for (const auto& field : schema_enum_binding.GetEnumeratorValues()) { + // @note: @og: dump enum metadata // - for (const auto& field : schema_enum_binding->GetEnumeratorValues()) { - // @note: @og: dump enum metadata - // - for (auto j = 0; j < field.m_nMetadataSize; j++) { - auto field_metadata = field.m_pMetadata[j]; - - if (auto data = GetMetadataValue(field_metadata); data.empty()) - builder.comment(field_metadata.m_szName); - else - builder.comment(std::format("{} \"{}\"", field_metadata.m_szName, data)); - } + for (auto j = 0; j < field.m_nMetadataSize; j++) { + auto field_metadata = field.m_pMetadata[j]; - print_enum_item(field); + if (auto data = GetMetadataValue(field_metadata); data.empty()) + builder.comment(field_metadata.m_szName); + else + builder.comment(std::format("{} \"{}\"", field_metadata.m_szName, data)); } - // @note: @es3n1n: we are done with this enum - // - builder.end_enum_class(); + print_enum_item(field); + } + + // @note: @es3n1n: we are done with this enum + // + builder.end_enum_class(); + } + + void AssembleEnums(codegen::generator_t::self_ref builder, const std::unordered_set& enums) { + for (const auto* schema_enum_binding : enums) { + AssembleEnum(builder, *schema_enum_binding); } } @@ -688,284 +690,289 @@ namespace sdk { }; } - void AssembleClasses(codegen::generator_t::self_ref builder, const std::unordered_set& classes) { - - // @note: @soufiw: - // sort all classes based on refs and inherit, and then print it. - // ================== - const auto ordered_classes = OrderClasses(classes); - - for (const auto& type_name : ordered_classes.forward_declarations) { - builder.forward_declaration(type_name); - } - - if (!ordered_classes.forward_declarations.empty()) { - builder.next_line(); + // TOOD: see if we can get rid of, or simplify, classes_to_dump + void AssembleClass(const std::list& classes_to_dump, codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + // @note: @es3n1n: get class info, assemble it + // + const auto* class_parent = class_.m_pBaseClassses ? class_.m_pBaseClassses->m_pClass : nullptr; + const auto& class_info = class_; + const auto is_struct = std::string_view{class_info.m_pszName}.ends_with("_t"); + + PrintClassInfo(builder, class_info); + + // @note: @es3n1n: get parent name + // + const std::string parent_class_name = (class_parent != nullptr) ? class_parent->m_pszName : ""; + + // @note: @es3n1n: start class + // + if (is_struct) + builder.begin_struct_with_base_type(class_info.m_pszName, parent_class_name, ""); + else + builder.begin_class_with_base_type(class_info.m_pszName, parent_class_name, ""); + + // @note: @es3n1n: field assembling state + // + struct { + std::size_t last_field_size = 0ull; + std::size_t last_field_offset = 0ull; + bool assembling_bitfield = false; + std::size_t total_bits_count_in_union = 0ull; + + std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var + } state; + + std::list> cached_fields{}; + std::list cached_datamap_fields{}; + + // @note: @es3n1n: if we need to pad first field or if there's no fields in this class + // and we need to properly pad it to make sure its size is the same as we expect it + // + std::optional first_field_offset = std::nullopt; + if (const auto* first_field = (class_.m_pFields == nullptr) ? nullptr : &class_.m_pFields[0]; first_field) + first_field_offset = first_field->m_nSingleInheritanceOffset; + + const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; + const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size; + + std::ptrdiff_t expected_pad_size = first_field_offset.value_or(class_size_without_parent); + if (expected_pad_size) // @note: @es3n1n: if there's a pad size we should account the parent class size + expected_pad_size -= parent_class_size; + + // @note: @es3n1n: and finally insert a pad + // + if (expected_pad_size > 0) // @fixme: @es3n1n: this is wrong, i probably should check for collisions instead + builder.access_modifier("private") + .struct_padding(parent_class_size, expected_pad_size, false, true) + .reset_tabs_count() + .comment(std::format("{:#x}", parent_class_size)) + .restore_tabs_count(); + + // @todo: @es3n1n: if for some mysterious reason this class describes fields + // of the base class we should handle it too. + if (class_parent && first_field_offset.has_value() && first_field_offset.value() < class_parent->m_nSizeOf) { + builder.comment( + std::format("Collision detected({:#x}->{:#x}), output may be wrong.", first_field_offset.value_or(0), class_parent->m_nSizeOf)); + state.collision_end_offset = class_parent->m_nSizeOf; } - auto classes_to_dump = ordered_classes.classes; - - for (auto& class_dump : classes_to_dump) { - // @note: @es3n1n: get class info, assemble it - // - const auto class_parent = class_dump.GetParent(); - const auto class_info = class_dump.target_; - const auto is_struct = std::string_view{class_info->m_pszName}.ends_with("_t"); - PrintClassInfo(builder, class_info); + // @note: @es3n1n: begin public members + // + builder.access_modifier("public"); - // @note: @es3n1n: get parent name - // - std::string parent_cls_name; - if (auto parent = class_info->m_pBaseClassses ? class_info->m_pBaseClassses->m_pClass : nullptr; parent) - parent_cls_name = parent->m_pszName; + for (const auto& field : class_info.GetFields()) { + // @fixme: @es3n1n: todo proper collision fix and remove this block + if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { + builder.comment( + std::format("Skipped field \"{}\" @ {:#x} because of the struct collision", field.m_pszName, field.m_nSingleInheritanceOffset)); + continue; + } - // @note: @es3n1n: start class + // @note: @es3n1n: obtaining size + // fall back to 1 because there are no 0-sized types // - if (is_struct) - builder.begin_struct_with_base_type(class_info->m_pszName, parent_cls_name, ""); - else - builder.begin_class_with_base_type(class_info->m_pszName, parent_cls_name, ""); + const int field_size = field.m_pSchemaType->GetSize().value_or(1); - // @note: @es3n1n: field assembling state + // @note: @es3n1n: parsing type // - struct { - std::size_t last_field_size = 0ull; - std::size_t last_field_offset = 0ull; - bool assembling_bitfield = false; - std::size_t total_bits_count_in_union = 0ull; - - std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var - } state; + const auto [type_name, array_sizes] = get_type(*field.m_pSchemaType); + const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); - // @note: @es3n1n: if we need to pad first field or if there's no fields in this class - // and we need to properly pad it to make sure its size is the same as we expect it + // @note: @es3n1n: insert padding if needed // - std::optional first_field_offset = std::nullopt; - if (const auto first_field = class_dump.GetFirstField(); first_field) - first_field_offset = first_field->m_nSingleInheritanceOffset; + const auto expected_offset = state.last_field_offset + state.last_field_size; + if (state.last_field_offset && state.last_field_size && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && + !state.assembling_bitfield) { - const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; - - std::ptrdiff_t expected_pad_size = first_field_offset.value_or(class_dump.ClassSizeWithoutParent()); - if (expected_pad_size) // @note: @es3n1n: if there's a pad size we should account the parent class size - expected_pad_size -= parent_class_size; - - // @note: @es3n1n: and finally insert a pad - // - if (expected_pad_size > 0) // @fixme: @es3n1n: this is wrong, i probably should check for collisions instead builder.access_modifier("private") - .struct_padding(parent_class_size, expected_pad_size, false, true) + .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) .reset_tabs_count() - .comment(std::format("{:#x}", parent_class_size)) - .restore_tabs_count(); - - // @todo: @es3n1n: if for some mysterious reason this class describes fields - // of the base class we should handle it too. - if (class_parent && first_field_offset.has_value() && first_field_offset.value() < class_parent->m_nSizeOf) { - builder.comment( - std::format("Collision detected({:#x}->{:#x}), output may be wrong.", first_field_offset.value_or(0), class_parent->m_nSizeOf)); - state.collision_end_offset = class_parent->m_nSizeOf; + .comment(std::format("{:#x}", expected_offset)) + .restore_tabs_count() + .access_modifier("public"); } - // @note: @es3n1n: begin public members + // @note: @es3n1n: begin union if we're assembling bitfields // - builder.access_modifier("public"); - - for (const auto& field : class_info->GetFields()) { - // @fixme: @es3n1n: todo proper collision fix and remove this block - if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { - builder.comment(std::format("Skipped field \"{}\" @ {:#x} because of the struct collision", field.m_pszName, - field.m_nSingleInheritanceOffset)); - continue; - } - - // @note: @es3n1n: obtaining size - // fall back to 1 because there are no 0-sized types - // - const int field_size = field.m_pSchemaType->GetSize().value_or(1); - - // @note: @es3n1n: parsing type - // - const auto [type_name, array_sizes] = get_type(*field.m_pSchemaType); - const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); - - // @note: @es3n1n: insert padding if needed - // - const auto expected_offset = state.last_field_offset + state.last_field_size; - if (state.last_field_offset && state.last_field_size && - expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { - - builder.access_modifier("private") - .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) - .reset_tabs_count() - .comment(std::format("{:#x}", expected_offset)) - .restore_tabs_count() - .access_modifier("public"); - } - - // @note: @es3n1n: begin union if we're assembling bitfields - // - if (!state.assembling_bitfield && var_info.is_bitfield()) { - builder.begin_bitfield_block(); - state.assembling_bitfield = true; - } + if (!state.assembling_bitfield && var_info.is_bitfield()) { + builder.begin_bitfield_block(); + state.assembling_bitfield = true; + } - // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union - // - if (state.assembling_bitfield && !var_info.is_bitfield()) { - const auto expected_union_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset; - const auto expected_union_size_bits = expected_union_size_bytes * 8; + // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union + // + if (state.assembling_bitfield && !var_info.is_bitfield()) { + const auto expected_union_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset; + const auto expected_union_size_bits = expected_union_size_bytes * 8; - const auto actual_union_size_bits = state.total_bits_count_in_union; + const auto actual_union_size_bits = state.total_bits_count_in_union; - if (expected_union_size_bits < state.total_bits_count_in_union) - throw std::runtime_error( - std::format("Unexpected union size: {}. Expected: {}", state.total_bits_count_in_union, expected_union_size_bits)); + if (expected_union_size_bits < state.total_bits_count_in_union) + throw std::runtime_error( + std::format("Unexpected union size: {}. Expected: {}", state.total_bits_count_in_union, expected_union_size_bits)); - if (expected_union_size_bits > state.total_bits_count_in_union) - builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); + if (expected_union_size_bits > state.total_bits_count_in_union) + builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); - state.last_field_offset += expected_union_size_bytes; - state.last_field_size = expected_union_size_bytes; + state.last_field_offset += expected_union_size_bytes; + state.last_field_size = expected_union_size_bytes; - builder.end_bitfield_block(false) - .reset_tabs_count() - .comment(std::format("{:d} bits", expected_union_size_bits)) - .restore_tabs_count(); + builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); - state.total_bits_count_in_union = 0ull; - state.assembling_bitfield = false; - } + state.total_bits_count_in_union = 0ull; + state.assembling_bitfield = false; + } - // @note: @es3n1n: dump metadata - // - for (auto j = 0; j < field.m_nMetadataSize; j++) { - auto field_metadata = field.m_pMetadata[j]; + // @note: @es3n1n: dump metadata + // + for (auto j = 0; j < field.m_nMetadataSize; j++) { + auto field_metadata = field.m_pMetadata[j]; - if (auto data = GetMetadataValue(field_metadata); data.empty()) - builder.comment(std::format("metadata: {}", field_metadata.m_szName)); - else - builder.comment(std::format("metadata: {} \"{}\"", field_metadata.m_szName, data)); - } + if (auto data = GetMetadataValue(field_metadata); data.empty()) + builder.comment(std::format("metadata: {}", field_metadata.m_szName)); + else + builder.comment(std::format("metadata: {} \"{}\"", field_metadata.m_szName, data)); + } - // if this prop is a non-pointer class, check if its worth directly embedding the accumulated offset of it into the metadata - const auto prop_class = - std::ranges::find_if(classes_to_dump, [type_name](const class_t& cls) { return cls.target_->GetName().compare(type_name) == 0; }); - if (prop_class != classes_to_dump.end()) { - // verify for min/max fields count, we don't want to bloat the dump by embeding too much stuff - if (prop_class->cached_fields_.size() >= kMinFieldCountForClassEmbed && - prop_class->cached_fields_.size() <= kMaxFieldCountForClassEmbed) { - // if a class is used in too many classes its likely not very useful, so ignore it - if (prop_class->used_count_ <= kMaxReferencesForClassEmbed) { - for (const auto& [cached_field_name, cached_field_offset] : prop_class->cached_fields_) { - const auto accumulated_offset = cached_field_offset + field.m_nSingleInheritanceOffset; - builder.comment(std::format("-> {} - {:#x}", cached_field_name, accumulated_offset)); - } + // if this prop is a non-pointer class, check if its worth directly embedding the accumulated offset of it into the metadata + const auto prop_class = + std::ranges::find_if(classes_to_dump, [type_name](const class_t& cls) { return cls.target_->GetName().compare(type_name) == 0; }); + if (prop_class != classes_to_dump.end()) { + // verify for min/max fields count, we don't want to bloat the dump by embeding too much stuff + if (cached_fields.size() >= kMinFieldCountForClassEmbed && cached_fields.size() <= kMaxFieldCountForClassEmbed) { + // if a class is used in too many classes its likely not very useful, so ignore it + if (prop_class->used_count_ <= kMaxReferencesForClassEmbed) { + for (const auto& [cached_field_name, cached_field_offset] : cached_fields) { + const auto accumulated_offset = cached_field_offset + field.m_nSingleInheritanceOffset; + builder.comment(std::format("-> {} - {:#x}", cached_field_name, accumulated_offset)); } } } - - // @note: @es3n1n: update state - // - if (field.m_nSingleInheritanceOffset && field_size) { - state.last_field_offset = field.m_nSingleInheritanceOffset; - state.last_field_size = static_cast(field_size); - } - if (var_info.is_bitfield()) - state.total_bits_count_in_union += var_info.m_bitfield_size; - - // @note: @es3n1n: push prop - // - builder.prop(var_info.m_type, var_info.formatted_name(), false); - if (!var_info.is_bitfield()) { - builder.reset_tabs_count().comment(std::format("{:#x}", field.m_nSingleInheritanceOffset), false).restore_tabs_count(); - class_dump.cached_fields_.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); - } - builder.next_line(); } - // @note: @es3n1n: if struct ends with union we should end union before ending the class + // @note: @es3n1n: update state // - if (state.assembling_bitfield) { - const auto actual_union_size_bits = state.total_bits_count_in_union; - - // @note: @es3n1n: apply 8 bytes align - // - const auto expected_union_size_bits = actual_union_size_bits + (actual_union_size_bits % 8); - - if (expected_union_size_bits > actual_union_size_bits) - builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); + if (field.m_nSingleInheritanceOffset && field_size) { + state.last_field_offset = field.m_nSingleInheritanceOffset; + state.last_field_size = static_cast(field_size); + } + if (var_info.is_bitfield()) + state.total_bits_count_in_union += var_info.m_bitfield_size; - builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); + // @note: @es3n1n: push prop + // + builder.prop(var_info.m_type, var_info.formatted_name(), false); - state.total_bits_count_in_union = 0; - state.assembling_bitfield = false; + if (!var_info.is_bitfield()) { + builder.reset_tabs_count().comment(std::format("{:#x}", field.m_nSingleInheritanceOffset), false).restore_tabs_count(); + cached_fields.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); } + builder.next_line(); + } + + // @note: @es3n1n: if struct ends with union we should end union before ending the class + // + if (state.assembling_bitfield) { + const auto actual_union_size_bits = state.total_bits_count_in_union; - // @note: @es3n1n: dump static fields + // @note: @es3n1n: apply 8 bytes align // - if (class_info->m_nStaticFieldsSize) { - if (class_info->m_nFieldSize) - builder.next_line(); - builder.comment("Static fields:"); - } + const auto expected_union_size_bits = actual_union_size_bits + (actual_union_size_bits % 8); - // The current class may be defined in multiple scopes. It doesn't matter which one we use, as all definitions are the same.. - // TODO: verify the above statement. Are static fields really shared between scopes? - const std::string scope_name{class_info->m_pTypeScope->BGetScopeName()}; + if (expected_union_size_bits > actual_union_size_bits) + builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); - for (auto s = 0; s < class_info->m_nStaticFieldsSize; s++) { - auto static_field = &class_info->m_pStaticFields[s]; + builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); - auto [type, mod] = get_type(*static_field->m_pSchemaType); - const auto var_info = field_parser::parse(type, static_field->m_pszName, mod); - builder.static_field_getter(var_info.m_type, var_info.m_name, scope_name, class_info->m_pszName, s); - } + state.total_bits_count_in_union = 0; + state.assembling_bitfield = false; + } + + // @note: @es3n1n: dump static fields + // + if (class_info.m_nStaticFieldsSize) { + if (class_info.m_nFieldSize) + builder.next_line(); + builder.comment("Static fields:"); + } - if (class_info->m_pFieldMetadataOverrides && class_info->m_pFieldMetadataOverrides->m_iTypeDescriptionCount > 1) { - const auto& dm = class_info->m_pFieldMetadataOverrides; + // The current class may be defined in multiple scopes. It doesn't matter which one we use, as all definitions are the same.. + // TODO: verify the above statement. Are static fields really shared between scopes? + const std::string scope_name{class_info.m_pTypeScope->BGetScopeName()}; - for (std::uint64_t s = 0; s < dm->m_iTypeDescriptionCount; s++) { - auto* t = &dm->m_pTypeDescription[s]; - if (!t) - continue; + for (auto s = 0; s < class_info.m_nStaticFieldsSize; s++) { + auto static_field = &class_info.m_pStaticFields[s]; - if (t->GetFieldName().empty()) - continue; + auto [type, mod] = get_type(*static_field->m_pSchemaType); + const auto var_info = field_parser::parse(type, static_field->m_pszName, mod); + builder.static_field_getter(var_info.m_type, var_info.m_name, scope_name, class_info.m_pszName, s); + } - const auto var_info = field_parser::parse(t->m_iFieldType, t->GetFieldName().data(), t->m_nFieldSize); + if (class_info.m_pFieldMetadataOverrides && class_info.m_pFieldMetadataOverrides->m_iTypeDescriptionCount > 1) { + const auto& dm = class_info.m_pFieldMetadataOverrides; - std::string field_type = var_info.m_type; - if (t->m_iFieldType == fieldtype_t::FIELD_EMBEDDED) { - field_type = t->m_pDataMap->m_pszClassName; - } + for (std::uint64_t s = 0; s < dm->m_iTypeDescriptionCount; s++) { + auto* t = &dm->m_pTypeDescription[s]; + if (!t) + continue; - std::string field_name = var_info.formatted_name(); + if (t->GetFieldName().empty()) + continue; - // @note: @og: if schema dump already has this field, then just skip it - if (const auto it = std::ranges::find_if( - class_dump.cached_fields_, [t, field_name](const auto& f) { return f.first == field_name && f.second == t->m_iOffset; }); - it != class_dump.cached_fields_.end()) - continue; + const auto var_info = field_parser::parse(t->m_iFieldType, t->GetFieldName().data(), t->m_nFieldSize); - class_dump.cached_datamap_fields_.emplace_back(field_type, field_name, t->m_iOffset); + std::string field_type = var_info.m_type; + if (t->m_iFieldType == fieldtype_t::FIELD_EMBEDDED) { + field_type = t->m_pDataMap->m_pszClassName; } - if (!class_dump.cached_datamap_fields_.empty()) { - if (class_info->m_nFieldSize) - builder.next_line(); + std::string field_name = var_info.formatted_name(); - builder.comment("Datamap fields:"); - for (auto& [field_type, field_name, field_offset] : class_dump.cached_datamap_fields_) { - builder.comment(std::format("{} {}; // {:#x}", field_type, field_name, field_offset)); - } + // @note: @og: if schema dump already has this field, then just skip it + if (const auto it = std::ranges::find_if(cached_fields, + [t, field_name](const auto& f) { return f.first == field_name && f.second == t->m_iOffset; }); + it != cached_fields.end()) + continue; + + cached_datamap_fields.emplace_back(field_type, field_name, t->m_iOffset); + } + + if (!cached_datamap_fields.empty()) { + if (class_info.m_nFieldSize) + builder.next_line(); + + builder.comment("Datamap fields:"); + for (auto& [field_type, field_name, field_offset] : cached_datamap_fields) { + builder.comment(std::format("{} {}; // {:#x}", field_type, field_name, field_offset)); } } + } + + if (!class_info.m_nFieldSize && !class_info.m_nStaticMetadataSize) + builder.comment("No schema binary for binding"); + + builder.end_block(); + } + + void AssembleClasses(codegen::generator_t::self_ref builder, const std::unordered_set& classes) { - if (!class_info->m_nFieldSize && !class_info->m_nStaticMetadataSize) - builder.comment("No schema binary for binding"); + // @note: @soufiw: + // sort all classes based on refs and inherit, and then print it. + // ================== + const auto ordered_classes = OrderClasses(classes); - builder.end_block(); + for (const auto& type_name : ordered_classes.forward_declarations) { + builder.forward_declaration(type_name); + } + + if (!ordered_classes.forward_declarations.empty()) { + builder.next_line(); + } + + auto classes_to_dump = ordered_classes.classes; + + for (const auto* class_ : classes) { + AssembleClass(ordered_classes.classes, builder, *class_); } } @@ -1008,29 +1015,62 @@ namespace sdk { return modules; } - void GenerateTypeScopeSdk(std::string_view module_name, const std::unordered_set& enums, - const std::unordered_set& classes) { - // @note: @es3n1n: print debug info + void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { + const std::string out_file_path = std::format("{}/{}/{}.hpp", kOutDirName, module_name, enum_.m_pszName); + + // @note: @es3n1n: init codegen // - std::cout << std::format("{}: Assembling {}", __FUNCTION__, module_name) << std::endl; + auto builder = codegen::get(); + builder.pragma("once"); + + builder.include(""); - // @note: @es3n1n: build file path + // @note: @es3n1n: print banner // - if (!std::filesystem::exists(kOutDirName)) - std::filesystem::create_directories(kOutDirName); - const std::string out_file_path = std::format("{}/{}.hpp", kOutDirName, module_name); + builder.next_line() + .comment("/////////////////////////////////////////////////////////////") + .comment(std::format("Module: {}", module_name)) + .comment(std::string{kCreatedBySource2genMessage}) + .comment("/////////////////////////////////////////////////////////////") + .next_line(); + + builder.begin_namespace(std::format("source2sdk::{}", module_name)); + + // @note: @es3n1n: assemble props + // + AssembleEnum(builder, enum_); + + builder.end_namespace(); + + // @note: @es3n1n: write generated data to output file + // + std::ofstream f(out_file_path, std::ios::out); + f << builder.str(); + if (f.bad()) { + std::cerr << std::format("Could not write to {}: {}", out_file_path, std::strerror(errno)) << std::endl; + // This std::exit() is bad. Instead, we could return the dumped + // header name and content to the caller in a std::expected. Let the + // caller write the file. That would also allow the caller to choose + // the output directory and handle errors. + std::exit(1); + } + } + + // TOOD: get rid of classes_to_dump + void GenerateClassSdk(const std::list& classes_to_dump, std::string_view module_name, const CSchemaClassBinding& class_) { + const std::filesystem::path out_file_path = std::format("{}/{}/{}.hpp", kOutDirName, module_name, class_.m_pszName); // @note: @es3n1n: init codegen // auto builder = codegen::get(); builder.pragma("once"); - // @note: @es3n1n: include files - // - for (const auto& required_module : find_required_modules(module_name, classes)) { - builder.include(std::format("\"{}.hpp\"", required_module)); + for (const auto& required_module : find_required_modules_of_class(module_name, class_)) { + // TOOD: include specific files + builder.include(std::format("\"{}/*.hpp\"", required_module)); } + // TOOD: make include consistent with GenerateEnumSdk for (auto&& include_path : include_paths) builder.include(include_path.data()); @@ -1039,8 +1079,6 @@ namespace sdk { builder.next_line() .comment("/////////////////////////////////////////////////////////////") .comment(std::format("Module: {}", module_name)) - .comment(std::format("Classes count: {}", classes.size())) - .comment(std::format("Enums count: {}", enums.size())) .comment(std::string{kCreatedBySource2genMessage}) .comment("/////////////////////////////////////////////////////////////") .next_line(); @@ -1049,8 +1087,7 @@ namespace sdk { // @note: @es3n1n: assemble props // - AssembleEnums(builder, enums); - AssembleClasses(builder, classes); + AssembleClass(classes_to_dump, builder, class_); builder.end_namespace(); @@ -1059,7 +1096,7 @@ namespace sdk { std::ofstream f(out_file_path, std::ios::out); f << builder.str(); if (f.bad()) { - std::cerr << std::format("Could not write to {}: {}", out_file_path, std::strerror(errno)) << std::endl; + std::cerr << std::format("Could not write to {}: {}", out_file_path.string(), std::strerror(errno)) << std::endl; // This std::exit() is bad. Instead, we could return the dumped // header name and content to the caller in a std::expected. Let the // caller write the file. That would also allow the caller to choose @@ -1067,6 +1104,24 @@ namespace sdk { std::exit(1); } } + + void GenerateTypeScopeSdk(std::string_view module_name, const std::unordered_set& enums, + const std::unordered_set& classes) { + // @note: @es3n1n: print debug info + // + std::cout << std::format("{}: Assembling module {} with {} enum(s) and {} class(es)", __FUNCTION__, module_name, enums.size(), classes.size()) + << std::endl; + + const std::filesystem::path out_directory_path = std::format("{}/{}", kOutDirName, module_name); + + if (!std::filesystem::exists(out_directory_path)) + std::filesystem::create_directories(out_directory_path); + + const auto ordered_classes = OrderClasses(classes); + + std::ranges::for_each(enums, [=](const auto* el) { GenerateEnumSdk(module_name, *el); }); + std::ranges::for_each(classes, [=](const auto* el) { GenerateClassSdk(ordered_classes.classes, module_name, *el); }); + } } // namespace sdk // source2gen - Source2 games SDK generator From 51527a17fdff29d06613093edc67f3be93b5abaa Mon Sep 17 00:00:00 2001 From: Cre3per Date: Wed, 31 Jul 2024 20:15:13 +0200 Subject: [PATCH 07/46] remove type embedding code never activated (in CS2) --- src/sdk/sdk.cpp | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 1453982..c31d78a 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -690,8 +690,7 @@ namespace sdk { }; } - // TOOD: see if we can get rid of, or simplify, classes_to_dump - void AssembleClass(const std::list& classes_to_dump, codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { // @note: @es3n1n: get class info, assemble it // const auto* class_parent = class_.m_pBaseClassses ? class_.m_pBaseClassses->m_pClass : nullptr; @@ -834,22 +833,6 @@ namespace sdk { builder.comment(std::format("metadata: {} \"{}\"", field_metadata.m_szName, data)); } - // if this prop is a non-pointer class, check if its worth directly embedding the accumulated offset of it into the metadata - const auto prop_class = - std::ranges::find_if(classes_to_dump, [type_name](const class_t& cls) { return cls.target_->GetName().compare(type_name) == 0; }); - if (prop_class != classes_to_dump.end()) { - // verify for min/max fields count, we don't want to bloat the dump by embeding too much stuff - if (cached_fields.size() >= kMinFieldCountForClassEmbed && cached_fields.size() <= kMaxFieldCountForClassEmbed) { - // if a class is used in too many classes its likely not very useful, so ignore it - if (prop_class->used_count_ <= kMaxReferencesForClassEmbed) { - for (const auto& [cached_field_name, cached_field_offset] : cached_fields) { - const auto accumulated_offset = cached_field_offset + field.m_nSingleInheritanceOffset; - builder.comment(std::format("-> {} - {:#x}", cached_field_name, accumulated_offset)); - } - } - } - } - // @note: @es3n1n: update state // if (field.m_nSingleInheritanceOffset && field_size) { @@ -969,10 +952,8 @@ namespace sdk { builder.next_line(); } - auto classes_to_dump = ordered_classes.classes; - for (const auto* class_ : classes) { - AssembleClass(ordered_classes.classes, builder, *class_); + AssembleClass(builder, *class_); } } @@ -1056,8 +1037,7 @@ namespace sdk { } } - // TOOD: get rid of classes_to_dump - void GenerateClassSdk(const std::list& classes_to_dump, std::string_view module_name, const CSchemaClassBinding& class_) { + void GenerateClassSdk(std::string_view module_name, const CSchemaClassBinding& class_) { const std::filesystem::path out_file_path = std::format("{}/{}/{}.hpp", kOutDirName, module_name, class_.m_pszName); // @note: @es3n1n: init codegen @@ -1087,7 +1067,7 @@ namespace sdk { // @note: @es3n1n: assemble props // - AssembleClass(classes_to_dump, builder, class_); + AssembleClass(builder, class_); builder.end_namespace(); @@ -1117,10 +1097,8 @@ namespace sdk { if (!std::filesystem::exists(out_directory_path)) std::filesystem::create_directories(out_directory_path); - const auto ordered_classes = OrderClasses(classes); - std::ranges::for_each(enums, [=](const auto* el) { GenerateEnumSdk(module_name, *el); }); - std::ranges::for_each(classes, [=](const auto* el) { GenerateClassSdk(ordered_classes.classes, module_name, *el); }); + std::ranges::for_each(classes, [=](const auto* el) { GenerateClassSdk(module_name, *el); }); } } // namespace sdk From 2e3b55a5f81cd5a36bf54a6b6d97ffc2878d3ab9 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Thu, 1 Aug 2024 17:22:53 +0200 Subject: [PATCH 08/46] forward-declare non-odr-uses of types --- include/sdk/interfaces/schemasystem/schema.h | 18 +- include/tools/codegen.h | 10 +- scripts/run.sh | 2 +- sdk-dummy/CMakeLists.txt | 4 + src/sdk/sdk.cpp | 372 ++++++------------- 5 files changed, 127 insertions(+), 279 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 214d6f7..72fc5e5 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -619,7 +619,7 @@ struct SchemaClassInfoData_t { SchemaClassFieldData_t* m_pFields; // 0x0028 SchemaStaticFieldData_t* m_pStaticFields; // 0x0030 - SchemaBaseClassInfoData_t* m_pBaseClassses; // 0x0038 + SchemaBaseClassInfoData_t* m_pBaseClasses; // 0x0038 SchemaFieldMetadataOverrideSetData_t* m_pFieldMetadataOverrides; // 0x0040 SchemaMetadataEntryData_t* m_pStaticMetadata; // 0x0048 CSchemaSystemTypeScope* m_pTypeScope; // 0x0050 @@ -653,8 +653,8 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { } [[nodiscard]] std::optional GetBaseClass() const { - if (m_nBaseClassSize && m_pBaseClassses) - return m_pBaseClassses->m_pClass; + if (m_nBaseClassSize && m_pBaseClasses) + return m_pBaseClasses->m_pClass; return std::nullopt; } @@ -672,9 +672,9 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { } [[nodiscard]] std::string_view GetPrevClassName() const { - if (!m_pBaseClassses || !m_pBaseClassses->m_pClass) + if (!m_pBaseClasses || !m_pBaseClasses->m_pClass) return {}; - return m_pBaseClassses->m_pClass->GetName(); + return m_pBaseClasses->m_pClass->GetName(); } [[nodiscard]] bool IsA(CSchemaType* pInheritance) const { @@ -689,19 +689,19 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { } [[nodiscard]] bool RecursiveHasVirtualTable() const { - return HasVirtualTable() || (m_pBaseClassses && m_pBaseClassses->m_pClass && m_pBaseClassses->m_pClass->HasVirtualTable()); + return HasVirtualTable() || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->HasVirtualTable()); } [[nodiscard]] bool IsInherits(const std::string_view from) const { - if (!m_nBaseClassSize || !m_pBaseClassses || !m_pBaseClassses->m_pClass) + if (!m_nBaseClassSize || !m_pBaseClasses || !m_pBaseClasses->m_pClass) return false; - if (m_pBaseClassses->m_pClass->GetName() == from) + if (m_pBaseClasses->m_pClass->GetName() == from) return true; return false; } [[nodiscard]] bool IsRecursiveInherits(const std::string_view from) const { - return IsInherits(from) || (m_pBaseClassses && m_pBaseClassses->m_pClass && m_pBaseClassses->m_pClass->IsRecursiveInherits(from)); + return IsInherits(from) || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->IsRecursiveInherits(from)); } [[nodiscard]] int GetSize() const { diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 4c2a8ba..aa14028 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -139,14 +139,14 @@ namespace codegen { if (base_type.empty()) return begin_class(class_name, access_modifier); else - return begin_block(std::format("class {} : public {}", escape_name(class_name), escape_name(base_type)), access_modifier); + return begin_block(std::format("class {} : public {}", escape_name(class_name), base_type), access_modifier); } self_ref end_class() { return end_block(); } - self_ref begin_namespace(const std::string& namespace_name) { + self_ref begin_namespace(const std::string_view namespace_name) { return begin_block(std::format("namespace {}", namespace_name)); } @@ -226,9 +226,9 @@ namespace codegen { return push_line(line, move_cursor_to_next_line); } - self_ref forward_declaration(const std::string& text) { + self_ref forward_declaration(const std::string& type_name) { // @note: @es3n1n: forward decl only once - const auto fwd_decl_hash = fnv32::hash_runtime(text.data()); + const auto fwd_decl_hash = fnv32::hash_runtime(type_name.data()); if (_forward_decls.contains(fwd_decl_hash)) return *this; @@ -236,7 +236,7 @@ namespace codegen { // @fixme: split method to class_forward_declaration & struct_forward_declaration // one for `struct uwu_t` and the other one for `class c_uwu` - return push_line(std::format("struct {};", text)); + return push_line(std::format("struct {};", type_name)); } self_ref struct_padding(const std::optional pad_offset, const std::size_t padding_size, const bool move_cursor_to_next_line = true, diff --git a/scripts/run.sh b/scripts/run.sh index a0f7ea9..2bd436e 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -31,7 +31,7 @@ if [ -z "${BINARY}" ]; then else export LD_LIBRARY_PATH="${GAME_DIRECTORY}/game/bin/linuxsteamrt64/:${GAME_DIRECTORY}/game/csgo/bin/linuxsteamrt64/:${LD_LIBRARY_PATH:-}" set -x - if [ -z "${DEBUGGER}" ]; then + if [ -z "${DEBUGGER:-}" ]; then "${BINARY}" else "${DEBUGGER}" -- "${BINARY}" diff --git a/sdk-dummy/CMakeLists.txt b/sdk-dummy/CMakeLists.txt index a230e65..6a230b2 100644 --- a/sdk-dummy/CMakeLists.txt +++ b/sdk-dummy/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.30) +set(CMAKE_EXPORT_COMPILE_COMMANDS On) + project(source2sdk LANGUAGES CXX ) @@ -18,6 +20,8 @@ file(WRITE ${generated_cpp_file} ${generated_cpp_contents}) add_library(${PROJECT_NAME} ${generated_cpp_file}) +target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_options(${PROJECT_NAME} PRIVATE "-Wfatal-errors" "-pedantic-errors" diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index c31d78a..7186302 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -4,6 +4,7 @@ // ReSharper disable CppClangTidyClangDiagnosticLanguageExtensionToken #include "sdk/sdk.h" #include +#include #include #include #include @@ -152,60 +153,6 @@ namespace sdk { std::ptrdiff_t offset_; }; - struct class_t { - CSchemaClassInfo* target_{}; - std::set refs_; - std::uint32_t used_count_{}; - - [[nodiscard]] CSchemaClassInfo* GetParent() const { - if (!target_->m_pBaseClassses) - return nullptr; - - return target_->m_pBaseClassses->m_pClass; - } - - void AddRefToClass(CSchemaType* type) { - if (type->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - refs_.insert(reinterpret_cast(type)->m_pClassInfo); - } - - // auto ptr = type->GetRefClass(); - // if (ptr && ptr->m_nTypeCategory == Schema_DeclaredClass) - // { - // refs_.insert(ptr->m_pClassInfo); - // return; - // } - } - - [[nodiscard]] bool IsDependsOn(const class_t& other) const { - // if current class inherit other. - const auto parent = this->GetParent(); - if (parent == other.target_) - return true; - - // if current class contains ref to other. - if (this->refs_.contains(other.target_)) - return true; - - // otherwise, order doesn`t matter. - return false; - } - - [[nodiscard]] SchemaClassFieldData_t* GetFirstField() const { - if (target_->m_nFieldSize) - return &target_->m_pFields[0]; - return nullptr; - } - - // @note: @es3n1n: Returns the struct size without its parent's size - // - [[nodiscard]] std::ptrdiff_t ClassSizeWithoutParent() const { - if (const CSchemaClassInfo* class_parent = this->GetParent(); class_parent) - return this->target_->m_nSizeOf - class_parent->m_nSizeOf; - return this->target_->m_nSizeOf; - } - }; - namespace { void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_info) { builder.comment(std::format("Alignment: {}", class_info.GetAligment())).comment(std::format("Size: {:#x}", class_info.m_nSizeOf)); @@ -341,12 +288,6 @@ namespace sdk { builder.end_enum_class(); } - void AssembleEnums(codegen::generator_t::self_ref builder, const std::unordered_set& enums) { - for (const auto* schema_enum_binding : enums) { - AssembleEnum(builder, *schema_enum_binding); - } - } - /// @return {type_name, array_sizes} auto parse_array(const CSchemaType& type) -> std::pair> { const auto* ptr = type.GetRefClass(); @@ -372,12 +313,27 @@ namespace sdk { return {base_type, sizes}; }; + // TOOD: I've seen a similar function somewhere. make sure it's not duplicate + /// @return Lifetime is bound to string viewed by @p type_name + [[nodiscard]] + auto decay_type_name(std::string_view type_name) -> std::string_view { + if (const auto found = type_name.find('['); found != std::string_view::npos) { + // "array[123]" -> "array" + type_name = type_name.substr(0, found); + } + if (const auto found = type_name.find('*'); found != std::string_view::npos) { + // "pointer***" -> "pointer" + type_name = type_name.substr(0, found); + } + + return type_name; + }; + /// @return @ref std::nullopt if the type is not contained in a module visible in @p scope auto get_module_of_type_in_scope(const CSchemaSystemTypeScope& scope, std::string_view type_name) -> std::optional { - // resolve pointers to their actual type - while (type_name.ends_with('*')) { - type_name.remove_suffix(1); - } + assert((decay_type_name(type_name) == type_name) && + "you need to decay your type names before using them for lookups. you probably need to decay them anyway if you intend to you them for " + "anything really, so do it before calling this function."); if (const auto* class_ = scope.FindDeclaredClass(std::string{type_name})) { return class_->m_pszModule; @@ -391,7 +347,7 @@ namespace sdk { /// @return @ref std::nullopt if the type is not contained in a module visible in its scope, e.g. because it is a built-in type auto get_module_of_type(const CSchemaType& type) -> std::optional { if (type.m_pTypeScope != nullptr) { - return get_module_of_type_in_scope(*type.m_pTypeScope, type.m_pszName); + return get_module_of_type_in_scope(*type.m_pTypeScope, decay_type_name(type.m_pszName)); } else { return std::nullopt; } @@ -462,23 +418,24 @@ namespace sdk { return str; } + /// Adds the module specifier to @p type_name, if @p type_name is declared in @p scope. Otherwise returns @p type_name unmodified. + auto maybe_with_module_name(const CSchemaSystemTypeScope& scope, const std::string_view type_name) -> std::string { + // This is a hack to support nested types. + // When we define nested types, they're not actually nested, but contain their outer class' name in their name, + // e.g. "struct Player { struct Hand {}; };" is emitted as + // "struct Player {}; struct Player__Hand{};". + // But when used as a property, types expect `Hand` in `Player`, i.e. `Player::Hand m_hand;` + // Instead of doing this hackery, we should probably declare nested classes as nested classes. + const auto escaped_type_name = string_replace(std::string{type_name}, "::", "__"); + + return get_module_of_type_in_scope(scope, type_name) + .transform([&](const auto module_name) { return std::format("{}::{}", module_name, escaped_type_name); }) + .value_or(std::string{escaped_type_name}); + }; + /// Adds module qualifiers and resolves built-in types auto reassemble_retyped_template(const CSchemaSystemTypeScope& scope, const std::vector>& decomposed) -> std::string { - const auto maybe_with_module_name = [&scope](const std::string_view type_name) { - // This is a hack to support nested types. - // When we define nested types, they're not actually nested, but contain their outer class' name in their name, - // e.g. "struct Player { struct Hand {}; };" is emitted as - // "struct Player {}; struct Player__Hand{};". - // But when used as a property, types expect `Hand` in `Player`, i.e. `Player::Hand m_hand;` - // Instead of doing this hackery, we should probably declare nested classes as nested classes. - const auto escaped_type_name = string_replace(std::string{type_name}, "::", "__"); - - return get_module_of_type_in_scope(scope, type_name) - .transform([&](const auto module_name) { return std::format("{}::{}", module_name, escaped_type_name); }) - .value_or(std::string{escaped_type_name}); - }; - std::string result{}; for (const auto& el : decomposed) { @@ -490,7 +447,12 @@ namespace sdk { if (const auto built_in = field_parser::type_name_to_cpp(e)) { result += built_in.value(); } else { - result += maybe_with_module_name(e); + // e is a dirty name, e.g. "CPlayer*[10]". We need to add the module, but keep it dirty. + const auto type_name = decay_type_name(e); + const auto type_name_with_module = maybe_with_module_name(scope, type_name); + const auto dirty_type_name_with_module = + std::string{e}.replace(e.find(type_name), type_name.length(), type_name_with_module); + result += dirty_type_name_with_module; } } }, @@ -517,12 +479,6 @@ namespace sdk { return {type_name_with_modules, {}}; }; - // TOOD: move up - struct OrderedClasses { - std::list forward_declarations{}; - std::list classes{}; - }; - /// e.g. "HashMap>" -> ["HashMap", "int", "CUtlVector", "float"] /// @return An empty list if @p type_name is not a template or has no template parameters [[nodiscard]] @@ -546,154 +502,39 @@ namespace sdk { // TOOD: need to sort these fuctions, they're all over the place - /// @return All modules directly required to declare this type. Returns multiple + // We assume that everything that is not a pointer odr-used. + // This assumption not correct, e.g. template classes that internally store pointers are + // not always odr-users of a type. It's good enough for what we do though. + [[nodiscard]] + constexpr auto is_odr_use(std::string_view type_name) -> bool { + return !type_name.contains('*'); + } + + /// @return {module,type} All includes to types odr-used by @p type. Returns multiple /// modules for template types. [[nodiscard]] - auto get_modules_of_type(const CSchemaType& type) -> std::unordered_set { + auto get_required_includes_for_type(const CSchemaType& type) -> std::set> { + // m_pTypeScope can be nullptr for built-in types if (type.m_pTypeScope != nullptr) { - std::unordered_set result{}; - for (const auto& type_name : parse_template_recursive(type.m_pszName)) { + std::set> result{}; + + for (const auto& type_name : + parse_template_recursive(type.m_pszName) | std::views::filter(is_odr_use) | std::views::transform(decay_type_name)) { if (auto module{get_module_of_type_in_scope(*type.m_pTypeScope, type_name)}) { - result.emplace(std::move(module.value())); + result.emplace(std::move(module.value()), type_name); } } + return result; } else { return {}; } } - /** - * Orderes classes such that dependencies are declared or defined befor dependees. - */ - [[nodiscard]] - OrderedClasses OrderClasses(const std::unordered_set& classes) { - // TOOD: remove tests - assert((parse_template_recursive("vector") == std::vector{"vector", "int"})); - assert((parse_template_recursive("vector") == std::vector{"vector"})); - assert((parse_template_recursive("map") == std::vector{"map", "int", "float"})); - assert((parse_template_recursive("A< B< C > >") == std::vector{"A", "B", "C"})); - - /// @return Lifetime is bound to string viewed by @p type_name - const auto decay_type_name = [](std::string_view type_name) -> std::string_view { - if (const auto found = type_name.find('['); found != std::string_view::npos) { - // "array[123]" -> "array" - type_name = type_name.substr(0, found); - } - if (const auto found = type_name.find('*'); found != std::string_view::npos) { - // "pointer***" -> "pointer" - type_name = type_name.substr(0, found); - } - - return type_name; - }; - - std::list forward_declarations{}; - std::list ordered_classes{}; - - for (const auto* schema_class_binding : classes) { - assert(schema_class_binding != nullptr); - - const auto class_info = schema_class_binding->m_pTypeScope->FindDeclaredClass(schema_class_binding->m_pszName); - - auto& class_dump = ordered_classes.emplace_back(); - class_dump.target_ = class_info; - } - - for (auto& class_dump : ordered_classes) { - const auto& class_info = *class_dump.target_; - - // forward declare all classes that are not nested. nested classes cannot be forward declared. - // @todo: maybe we need to forward declare only pointers to classes? - if (!class_info.GetName().contains("::")) { - forward_declarations.emplace_back(class_info.GetName()); - } - - for (const auto& field : std::span{class_info.m_pFields, static_cast(class_info.m_nFieldSize)}) { - auto ref_class = field.m_pSchemaType->GetRefClass(); - - class_dump.AddRefToClass(field.m_pSchemaType); - - const auto full_type_name = decay_type_name(std::string_view{field.m_pSchemaType->m_pszName}); - - // if `full_type_name` is `Map`, this loop runs [`Map`, `int`, 'float`] - for (const auto& type_name : parse_template_recursive(full_type_name)) { - if (const auto found = std::ranges::find_if(classes, [=](const auto& el) { return el->m_pszName == type_name; }); - found != classes.end()) { - class_dump.AddRefToClass((*found)->m_pSchemaType); - } - } - - auto field_class = std::ranges::find_if(ordered_classes, [field](const class_t& cls) { - return cls.target_ == reinterpret_cast(field.m_pSchemaType)->m_pClassInfo; - }); - if (field_class != ordered_classes.end()) - field_class->used_count_++; - } - } - - // to detect inderict dependency cycles - std::set hot_classes{}; - std::set previous_hot_classes{}; - - bool did_change = false; - do { - did_change = false; - - // swap until we done. - for (auto first = ordered_classes.begin(); first != ordered_classes.end(); ++first) { - bool second_below_first = false; - - for (auto second = ordered_classes.begin(); second != ordered_classes.end(); ++second) { - if (second == first) { - second_below_first = true; - continue; - } - - // swap if second class below first, and first depends on second. - const bool first_depend = first->IsDependsOn(*second); - - // swap if first class below second, and second depends on first. - const bool second_depend = second->IsDependsOn(*first); - - if (first_depend && second_depend) { - // classes depends on each other, forward declare them. - // @todo: verify that cyclic dependencies is a pointers. - continue; - } - - if (second_below_first ? first_depend : second_depend) { - hot_classes.emplace(first->target_); - hot_classes.emplace(second->target_); - - std::iter_swap(first, second); - did_change = true; - } - } - } - - if (did_change && (hot_classes == previous_hot_classes)) { - std::cout << "detected dependency cycle in the following classes\n"; - for (const auto* class_ : hot_classes) { - std::cout << std::format("- {}::{}\n", class_->m_pszModule, class_->m_pszName); - } - std::cout << "you need to resolve the dependency cycle by hand in the generated sdk\n"; - break; - } - std::swap(hot_classes, previous_hot_classes); - hot_classes.clear(); - } while (did_change); - - return OrderedClasses{ - .forward_declarations = forward_declarations, - .classes = ordered_classes, - }; - } - void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { // @note: @es3n1n: get class info, assemble it // - const auto* class_parent = class_.m_pBaseClassses ? class_.m_pBaseClassses->m_pClass : nullptr; + const auto* class_parent = class_.m_pBaseClasses ? class_.m_pBaseClasses->m_pClass : nullptr; const auto& class_info = class_; const auto is_struct = std::string_view{class_info.m_pszName}.ends_with("_t"); @@ -701,7 +542,8 @@ namespace sdk { // @note: @es3n1n: get parent name // - const std::string parent_class_name = (class_parent != nullptr) ? class_parent->m_pszName : ""; + const std::string parent_class_name = + (class_parent != nullptr) ? maybe_with_module_name(*class_parent->m_pTypeScope, class_parent->m_pszName) : ""; // @note: @es3n1n: start class // @@ -936,64 +778,61 @@ namespace sdk { builder.end_block(); } - - void AssembleClasses(codegen::generator_t::self_ref builder, const std::unordered_set& classes) { - - // @note: @soufiw: - // sort all classes based on refs and inherit, and then print it. - // ================== - const auto ordered_classes = OrderClasses(classes); - - for (const auto& type_name : ordered_classes.forward_declarations) { - builder.forward_declaration(type_name); - } - - if (!ordered_classes.forward_declarations.empty()) { - builder.next_line(); - } - - for (const auto* class_ : classes) { - AssembleClass(builder, *class_); - } - } - } // namespace - /// @param exclude_module_name will not be contained in the return value - /// @return All modules that are required by properties of @p classes - std::set find_required_modules_of_class(std::string_view exclude_module_name, const CSchemaClassBinding& class_) { - std::set result{}; + /// Together with @ref find_required_forward_declarations_for_class(), returns all dependencies of a class. + /// Does not return types that are used as pointers. See @ref find_required_forward_declarations_for_class() for that. + /// @return {module,type} All modules+types that are required to define @p classes + std::set> find_required_includes_for_class(const CSchemaClassBinding& class_) { + std::set> result{}; for (const auto& field : std::span{class_.m_pFields, static_cast(class_.m_nFieldSize)}) { - const auto modules = get_modules_of_type(*field.m_pSchemaType); - result.insert(modules.begin(), modules.end()); + const auto includes = get_required_includes_for_type(*field.m_pSchemaType); + result.insert(includes.begin(), includes.end()); } for (const auto& field : std::span{class_.m_pStaticFields, static_cast(class_.m_nStaticFieldsSize)}) { - const auto modules = get_modules_of_type(*field.m_pSchemaType); - result.insert(modules.begin(), modules.end()); + const auto includes = get_required_includes_for_type(*field.m_pSchemaType); + result.insert(includes.begin(), includes.end()); } - if (const auto found = result.find(std::string{exclude_module_name}); found != result.end()) { - result.erase(found); + if (const auto* base_classes = class_.m_pBaseClasses; base_classes != nullptr) { + assert(base_classes->m_pClass->m_pSchemaType != nullptr && "didn't think this could happen, feel free to touch"); + // source2gen doesn't support multiple inheritance, only check class[0] + const auto includes = get_required_includes_for_type(*base_classes[0].m_pClass->m_pSchemaType); + result.insert(includes.begin(), includes.end()); } return result; } - /// @param exclude_module_name will not be contained in the return value - /// @return All modules that are required by properties of @p classes - std::set find_required_modules(std::string_view exclude_module_name, const std::unordered_set& classes) { - std::set modules{}; + /// Together with @ref find_required_includes_for_class(), returns all dependencies of a class. + /// @return {module,type} All modules+types that need to be forward declared to define @p classes + std::set> find_required_forward_declarations_for_class(const CSchemaClassBinding& class_) { + std::set> result{}; + + for (const auto& field : std::span{class_.m_pFields, static_cast(class_.m_nFieldSize)} | + std::views::filter([](auto e) { return e.m_pSchemaType != nullptr; })) { + const std::string_view name = field.m_pszName; + const std::string_view type_name = field.m_pSchemaType->m_pszName; + if (name == "m_poseCacheHandles") { + std::cout << "heter"; + } - for (const auto& class_ : classes) { - assert(class_ != nullptr); + if (!is_odr_use(type_name)) { + if (const auto module = get_module_of_type(*field.m_pSchemaType)) { + result.emplace(module.value(), decay_type_name(type_name)); + } + } + } - const auto partial = find_required_modules_of_class(exclude_module_name, *class_); - modules.insert(partial.begin(), partial.end()); + // don't forward-declare self. happens for self-referencing types, e.g. entity2::CEntityComponentHelper + if (const auto found = result.find(std::pair{get_module_of_type(*class_.m_pSchemaType).value(), class_.GetName()}); + found != result.end()) { + result.erase(found); } - return modules; + return result; } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { @@ -1045,15 +884,20 @@ namespace sdk { auto builder = codegen::get(); builder.pragma("once"); - for (const auto& required_module : find_required_modules_of_class(module_name, class_)) { - // TOOD: include specific files - builder.include(std::format("\"{}/*.hpp\"", required_module)); + for (const auto& required_include : find_required_includes_for_class(class_)) { + builder.include(std::format("\"{}/{}.hpp\"", required_include.first, required_include.second)); } // TOOD: make include consistent with GenerateEnumSdk - for (auto&& include_path : include_paths) + for (const auto& include_path : include_paths) builder.include(include_path.data()); + for (const auto& forward_declaration : find_required_forward_declarations_for_class(class_)) { + builder.begin_namespace(std::format("source2sdk::{}", forward_declaration.first)); + builder.forward_declaration(forward_declaration.second); + builder.end_namespace(); + } + // @note: @es3n1n: print banner // builder.next_line() From ddcd099d741a232a43025b2a1ea5e56bfe57a7fd Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 4 Aug 2024 15:16:41 +0200 Subject: [PATCH 09/46] add includes for static fields --- sdk-dummy/source2gen_user_types.hpp | 4 +++- src/sdk/sdk.cpp | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index 8ed7f3f..8f8535a 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -43,9 +43,11 @@ template using CHandle = char[0x04]; template using C_NetworkUtlVectorBase = char[0x18]; +// size unknown. only used in dynamic containers. +using CSoundEventName = char[0x01]; template using CUtlLeanVector = char[0x10]; -template +template using CUtlOrderedMap = char[0x28]; // size doesn't mapper. only used as a pointer template diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 7186302..70a3faa 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -815,9 +815,18 @@ namespace sdk { std::views::filter([](auto e) { return e.m_pSchemaType != nullptr; })) { const std::string_view name = field.m_pszName; const std::string_view type_name = field.m_pSchemaType->m_pszName; - if (name == "m_poseCacheHandles") { - std::cout << "heter"; + + if (!is_odr_use(type_name)) { + if (const auto module = get_module_of_type(*field.m_pSchemaType)) { + result.emplace(module.value(), decay_type_name(type_name)); + } } + } + + for (const auto& field : std::span{class_.m_pStaticFields, static_cast(class_.m_nStaticFieldsSize)} | + std::views::filter([](auto e) { return e.m_pSchemaType != nullptr; })) { + const std::string_view name = field.m_pszName; + const std::string_view type_name = field.m_pSchemaType->m_pszName; if (!is_odr_use(type_name)) { if (const auto module = get_module_of_type(*field.m_pSchemaType)) { From 49cad6dea266d4c954d38c9cdcd357269b7f6424 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 4 Aug 2024 15:58:23 +0200 Subject: [PATCH 10/46] merge dependency resolves to prevent the case where both resolvers don't resolve a dependency --- sdk-dummy/source2gen_user_types.hpp | 3 + src/sdk/sdk.cpp | 143 ++++++++++++++-------------- 2 files changed, 77 insertions(+), 69 deletions(-) diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index 8f8535a..36e8e4a 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -6,6 +6,9 @@ template using CAnimValue = char[0x08]; using CAnimVariant = char[0x14]; +// size is a guess +template +using CAnimScriptParam = char[0x08]; using CBufferString = char[0x10]; using CColorGradient = char[0x18]; // size doesn't mapper. only used as a pointer diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 70a3faa..0703ad8 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -500,7 +500,8 @@ namespace sdk { return result; } - // TOOD: need to sort these fuctions, they're all over the place + // TOOD: need to sort these fuctions, they're all over the place. + // TOOD: move local functions into an anoymous namespace // We assume that everything that is not a pointer odr-used. // This assumption not correct, e.g. template classes that internally store pointers are @@ -510,27 +511,6 @@ namespace sdk { return !type_name.contains('*'); } - /// @return {module,type} All includes to types odr-used by @p type. Returns multiple - /// modules for template types. - [[nodiscard]] - auto get_required_includes_for_type(const CSchemaType& type) -> std::set> { - // m_pTypeScope can be nullptr for built-in types - if (type.m_pTypeScope != nullptr) { - std::set> result{}; - - for (const auto& type_name : - parse_template_recursive(type.m_pszName) | std::views::filter(is_odr_use) | std::views::transform(decay_type_name)) { - if (auto module{get_module_of_type_in_scope(*type.m_pTypeScope, type_name)}) { - result.emplace(std::move(module.value()), type_name); - } - } - - return result; - } else { - return {}; - } - } - void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { // @note: @es3n1n: get class info, assemble it // @@ -780,64 +760,86 @@ namespace sdk { } } // namespace - /// Together with @ref find_required_forward_declarations_for_class(), returns all dependencies of a class. - /// Does not return types that are used as pointers. See @ref find_required_forward_declarations_for_class() for that. - /// @return {module,type} All modules+types that are required to define @p classes - std::set> find_required_includes_for_class(const CSchemaClassBinding& class_) { - std::set> result{}; + enum class TypeSource { + include, + forward_declaration, + }; + + // TOOD: "dependency" is a misleading term here. repeats for functions. "includes" and "forward declarations" were good terms. + struct TypeDependency { + std::string module{}; + + /// Decayed + std::string type_name{}; + + TypeSource source{}; + + auto operator<=>(const TypeDependency&) const = default; + }; + + /// @return All dependencies to types used by @p type. Returns multiple + /// modules for template types. + [[nodiscard]] + auto get_required_dependencies_for_type(const CSchemaType& type) -> std::set { + // m_pTypeScope can be nullptr for built-in types + if (type.m_pTypeScope != nullptr) { + std::set result{}; + + const auto destructured = parse_template_recursive(type.m_pszName); + + // This is a slight hack to break dependency cycles. Some template types don't odr-use + // their template arguments during their declaration, think std::unique_ptr. + // There's no foolproof way to detect those template types, so we're hardcoding the ones that are + // known to cause circular dependencies. + static std::unordered_set non_odr_containers{"CHandle"}; + bool is_used_in_non_odr_container = false; + + for (const auto& dirty_type_name : destructured) { + const auto type_name = decay_type_name(dirty_type_name); + + if (auto module{get_module_of_type_in_scope(*type.m_pTypeScope, type_name)}) { + const auto source = + (!is_used_in_non_odr_container && is_odr_use(dirty_type_name)) ? TypeSource::include : TypeSource::forward_declaration; + + result.emplace(TypeDependency{.module = std::move(module.value()), .type_name = std::string{type_name}, .source = source}); + } + + is_used_in_non_odr_container = non_odr_containers.contains(type_name); + } + + return result; + } else { + return {}; + } + } + + /// @return All modules+types that are required to define @p classes + std::set find_required_dependencies_for_class(const CSchemaClassBinding& class_) { + std::set result{}; for (const auto& field : std::span{class_.m_pFields, static_cast(class_.m_nFieldSize)}) { - const auto includes = get_required_includes_for_type(*field.m_pSchemaType); + const auto includes = get_required_dependencies_for_type(*field.m_pSchemaType); result.insert(includes.begin(), includes.end()); } for (const auto& field : std::span{class_.m_pStaticFields, static_cast(class_.m_nStaticFieldsSize)}) { - const auto includes = get_required_includes_for_type(*field.m_pSchemaType); + const auto includes = get_required_dependencies_for_type(*field.m_pSchemaType); result.insert(includes.begin(), includes.end()); } if (const auto* base_classes = class_.m_pBaseClasses; base_classes != nullptr) { assert(base_classes->m_pClass->m_pSchemaType != nullptr && "didn't think this could happen, feel free to touch"); // source2gen doesn't support multiple inheritance, only check class[0] - const auto includes = get_required_includes_for_type(*base_classes[0].m_pClass->m_pSchemaType); + const auto includes = get_required_dependencies_for_type(*base_classes[0].m_pClass->m_pSchemaType); result.insert(includes.begin(), includes.end()); } - return result; - } - - /// Together with @ref find_required_includes_for_class(), returns all dependencies of a class. - /// @return {module,type} All modules+types that need to be forward declared to define @p classes - std::set> find_required_forward_declarations_for_class(const CSchemaClassBinding& class_) { - std::set> result{}; - - for (const auto& field : std::span{class_.m_pFields, static_cast(class_.m_nFieldSize)} | - std::views::filter([](auto e) { return e.m_pSchemaType != nullptr; })) { - const std::string_view name = field.m_pszName; - const std::string_view type_name = field.m_pSchemaType->m_pszName; - - if (!is_odr_use(type_name)) { - if (const auto module = get_module_of_type(*field.m_pSchemaType)) { - result.emplace(module.value(), decay_type_name(type_name)); - } - } - } - - for (const auto& field : std::span{class_.m_pStaticFields, static_cast(class_.m_nStaticFieldsSize)} | - std::views::filter([](auto e) { return e.m_pSchemaType != nullptr; })) { - const std::string_view name = field.m_pszName; - const std::string_view type_name = field.m_pSchemaType->m_pszName; - - if (!is_odr_use(type_name)) { - if (const auto module = get_module_of_type(*field.m_pSchemaType)) { - result.emplace(module.value(), decay_type_name(type_name)); - } - } - } + const auto is_self = [self_module{get_module_of_type(*class_.m_pSchemaType).value()}, self_type_name{class_.GetName()}](const auto& that) { + return (that.module == self_module) && (that.type_name == self_type_name); + }; - // don't forward-declare self. happens for self-referencing types, e.g. entity2::CEntityComponentHelper - if (const auto found = result.find(std::pair{get_module_of_type(*class_.m_pSchemaType).value(), class_.GetName()}); - found != result.end()) { + // don't forward-declare or include self. happens for self-referencing types, e.g. entity2::CEntityComponentHelper + if (const auto found = std::ranges::find_if(result, is_self); found != result.end()) { result.erase(found); } @@ -893,17 +895,20 @@ namespace sdk { auto builder = codegen::get(); builder.pragma("once"); - for (const auto& required_include : find_required_includes_for_class(class_)) { - builder.include(std::format("\"{}/{}.hpp\"", required_include.first, required_include.second)); + const auto dependencies = find_required_dependencies_for_class(class_); + + for (const auto& include : dependencies | std::views::filter([](const auto& el) { return el.source == TypeSource::include; })) { + builder.include(std::format("\"{}/{}.hpp\"", include.module, include.type_name)); } // TOOD: make include consistent with GenerateEnumSdk for (const auto& include_path : include_paths) builder.include(include_path.data()); - for (const auto& forward_declaration : find_required_forward_declarations_for_class(class_)) { - builder.begin_namespace(std::format("source2sdk::{}", forward_declaration.first)); - builder.forward_declaration(forward_declaration.second); + for (const auto& forward_declaration : + dependencies | std::views::filter([](const auto& el) { return el.source == TypeSource::forward_declaration; })) { + builder.begin_namespace(std::format("source2sdk::{}", forward_declaration.module)); + builder.forward_declaration(forward_declaration.type_name); builder.end_namespace(); } From 980d2b49c0a3d79ee36cee6db7cb999b466cf285 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 4 Aug 2024 16:11:27 +0200 Subject: [PATCH 11/46] remove bitfields larger than 8 bits --- include/tools/codegen.h | 48 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/include/tools/codegen.h b/include/tools/codegen.h index aa14028..184a199 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -14,33 +14,6 @@ namespace codegen { constexpr std::size_t kTabsPerBlock = 1; // @note: @es3n1n: how many \t characters shall we place per each block constexpr std::array kBlacklistedCharacters = {':', ';', '\\', '/'}; - // @note: @es3n1n: a list of possible integral types for bitfields (would be used in `guess_bitfield_type`) - // - // clang-format off - constexpr auto kBitfieldIntegralTypes = std::to_array>({ - {8, "uint8_t"}, - {16, "uint16_t"}, - {32, "uint32_t"}, - {64, "uint64_t"}, - - // @todo: @es3n1n: define uint128_t/uint256_t/... as custom structs in the very beginning of the file - {128, "uint128_t"}, - {256, "uint256_t"}, - {512, "uint512_t"}, - }); - // clang-format on - - inline std::string guess_bitfield_type(const std::size_t bits_count) { - for (auto p : kBitfieldIntegralTypes) { - if (bits_count > p.first) - continue; - - return p.second.data(); - } - - throw std::runtime_error(std::format("{} : Unable to guess bitfield type with size {}", __FUNCTION__, bits_count)); - } - struct generator_t { using self_ref = std::add_lvalue_reference_t; @@ -241,16 +214,29 @@ namespace codegen { self_ref struct_padding(const std::optional pad_offset, const std::size_t padding_size, const bool move_cursor_to_next_line = true, const bool is_private_field = false, const std::size_t bitfield_size = 0ull) { + const auto bytes = (bitfield_size == 0) ? padding_size : bitfield_size / 8; + const auto remaining_bits = bitfield_size % 8; + // @note: @es3n1n: mark private fields as maybe_unused to silence -Wunused-private-field - std::string type_name = bitfield_size ? guess_bitfield_type(bitfield_size) : "uint8_t"; + std::string type_name = "std::uint8_t"; if (is_private_field) type_name = "[[maybe_unused]] " + type_name; auto pad_name = pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value()) : std::format("pad_{:d}", _pads_count++); - if (!bitfield_size) - pad_name = pad_name + std::format("[{:#x}]", padding_size); - return prop(type_name, bitfield_size ? std::format("{}: {}", pad_name, bitfield_size) : pad_name, move_cursor_to_next_line); + if (bytes != 0) { + pad_name = pad_name + std::format("[{:#x}]", bytes); + } + + prop(type_name, pad_name, move_cursor_to_next_line); + + if (remaining_bits != 0) { + auto remainder_pad_name = + pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value() + bytes) : std::format("pad_{:d}", _pads_count++); + prop(type_name, std::format("{}: {}", remainder_pad_name, remaining_bits), move_cursor_to_next_line); + } + + return *this; } self_ref begin_union(std::string name = "") { From c9cf98615b56b1f1eb46223c165ccba5215b2d29 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 4 Aug 2024 16:15:04 +0200 Subject: [PATCH 12/46] move guess_bitfield_type to field_parser it is the last user --- sdk-dummy/source2gen_user_types.hpp | 5 ++--- src/tools/field_parser.cpp | 29 ++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index 36e8e4a..8a84533 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -68,6 +68,8 @@ using CUtlVectorSIMDPaddedVector = char[0x18]; template using CSmartPtr = char[0x08]; template +using CResourceArray = char[0x08]; +template using CResourceNameTyped = char[0xe0]; template using CStrongHandle = char[0x08]; @@ -104,9 +106,6 @@ using float32 = char[0x04]; using fltx4 = char[0x10]; using matrix3x4_t = char[0x30]; using matrix3x4a_t = char[0x30]; -using uint256_t = char[0x20]; - -using uint8 = std::uint8_t; // TOOD: happens in CUtlVector. how to fix? // intentionally left undefined. if you want to access static fields, add your own sdk. namespace interfaces { diff --git a/src/tools/field_parser.cpp b/src/tools/field_parser.cpp index fb54b08..da55b0b 100644 --- a/src/tools/field_parser.cpp +++ b/src/tools/field_parser.cpp @@ -10,6 +10,33 @@ namespace field_parser { constexpr std::string_view kBitfieldTypePrefix = "bitfield:"sv; + // @note: @es3n1n: a list of possible integral types for bitfields (would be used in `guess_bitfield_type`) + // + // clang-format off + constexpr auto kBitfieldIntegralTypes = std::to_array>({ + {8, "uint8_t"}, + {16, "uint16_t"}, + {32, "uint32_t"}, + {64, "uint64_t"}, + + // @todo: @es3n1n: define uint128_t/uint256_t/... as custom structs in the very beginning of the file + {128, "uint128_t"}, + {256, "uint256_t"}, + {512, "uint512_t"}, + }); + // clang-format on + + inline std::string guess_bitfield_type(const std::size_t bits_count) { + for (auto p : kBitfieldIntegralTypes) { + if (bits_count > p.first) + continue; + + return p.second.data(); + } + + throw std::runtime_error(std::format("{} : Unable to guess bitfield type with size {}", __FUNCTION__, bits_count)); + } + // clang-format off constexpr auto kTypeNameToCpp = std::to_array>({ {"float32"sv, "float"sv}, @@ -97,7 +124,7 @@ namespace field_parser { // @note: @es3n1n: saving parsed value result.m_bitfield_size = bitfield_size; - result.m_type = codegen::guess_bitfield_type(bitfield_size); + result.m_type = guess_bitfield_type(bitfield_size); } // @note: @es3n1n: we are assuming that this function would be executed right after From 5891d98767505731578c3954339cb2e9471fa90a Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 4 Aug 2024 16:26:21 +0200 Subject: [PATCH 13/46] add more dummy types sdk compiles! --- sdk-dummy/source2gen_user_types.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-dummy/source2gen_user_types.hpp index 8a84533..ab7ec21 100644 --- a/sdk-dummy/source2gen_user_types.hpp +++ b/sdk-dummy/source2gen_user_types.hpp @@ -46,6 +46,8 @@ template using CHandle = char[0x04]; template using C_NetworkUtlVectorBase = char[0x18]; +template +using CNetworkUtlVectorBase = char[0x18]; // size unknown. only used in dynamic containers. using CSoundEventName = char[0x01]; template @@ -64,11 +66,17 @@ template using CUtlLeanVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; template using C_UtlVectorEmbeddedNetworkVar = char[0x50]; +template +using CUtlVectorEmbeddedNetworkVar = char[0x50]; using CUtlVectorSIMDPaddedVector = char[0x18]; template using CSmartPtr = char[0x08]; template using CResourceArray = char[0x08]; +// size unknown +using CResourceString = char[0x08]; +template +using CResourcePointer = char[0x08]; template using CResourceNameTyped = char[0xe0]; template @@ -92,6 +100,7 @@ using QAngle = char[0x0c]; using QuaternionStorage = char[0x10]; using Quaternion = char[0x10]; using RadianEuler = char[0x14]; +using RenderInputLayoutField_t = char[0x04]; // we don't have a field size for this type. uses the fallback of 1. using RenderPrimitiveType_t = char[0x01]; using RotationVector = char[0x0c]; From 9e6734b3d113b2bc0a285a562a13c9e06976aa2d Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 5 Aug 2024 15:47:43 +0200 Subject: [PATCH 14/46] remove TODOs --- README.md | 29 ++++--------------- {sdk-dummy => sdk-static}/CMakeLists.txt | 26 +++++++++++------ .../source2sdk}/source2gen_user_types.hpp | 0 src/sdk/sdk.cpp | 19 +++++++++--- 4 files changed, 37 insertions(+), 37 deletions(-) rename {sdk-dummy => sdk-static}/CMakeLists.txt (50%) rename {sdk-dummy => sdk-static/source2sdk}/source2gen_user_types.hpp (100%) diff --git a/README.md b/README.md index 62b8bc4..d8bd54d 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,3 @@ -TOOD: remove - -- add missing types as dummies -- add empty `interfaces` sdk -- sort template classes below their template types -- decay arrays and pointers when sorting classes by dependencies -- detect indirect dependency cycles -- add includes to modules of template arguments -- rename template arguments to use fully qualified names -- add includes for template arguments (add decompose_type() and use that in many places?) -- escape class names - -TOOD: - -- rename nested types to use underscores -- see why the sdk doesn't compile -- generate static assertions -- remove all double underscores - # Source2Gen Source2Gen is a tool to generate Source 2 SDKs. \ @@ -34,7 +15,7 @@ can use the registry to find the game path and set `PATH` automatically. ```sh source2gen -copy .\sdk-dummy\source2gen_user_types.hpp .\sdk +copy .\sdk-static\source2gen_user_types.hpp .\sdk # view generated sdk dir .\sdk ``` @@ -43,7 +24,7 @@ dir .\sdk ```sh ./scripts/run.sh "$HOME/.steam/steam/steamapps/cs2/" -pc ./sdk-dummy/source2gen_user_types.hpp ./sdk +cp -r ./sdk-static/* ./sdk # view generated sdk ls ./sdk ``` @@ -64,9 +45,9 @@ Linux. The sdk depends on a file/module called "source2gen_user_types". This file has to be provided by the user and expose all types listed in -[source2gen_user_types.hpp](sdk-dummy/source2gen_user_types.hpp). If you don't -intend to access any of these types, you can use the dummy -[source2gen_user_types.hpp](sdk-dummy/source2gen_user_types.hpp). +[source2gen_user_types.hpp](sdk-static/source2gen_user_types.hpp). If you don't +intend to access any of these types, you can use the dummy file +[source2gen_user_types.hpp](sdk-static/source2gen_user_types.hpp). ## Getting Started diff --git a/sdk-dummy/CMakeLists.txt b/sdk-static/CMakeLists.txt similarity index 50% rename from sdk-dummy/CMakeLists.txt rename to sdk-static/CMakeLists.txt index 6a230b2..13e99e6 100644 --- a/sdk-dummy/CMakeLists.txt +++ b/sdk-static/CMakeLists.txt @@ -1,4 +1,4 @@ -# TOOD: file should not be here +# TOOD: use meson cmake_minimum_required(VERSION 3.30) @@ -10,6 +10,18 @@ project(source2sdk file(GLOB_RECURSE source2sdk_headers "./**.hpp") +add_library(${PROJECT_NAME} ${source2sdk_headers}) +target_include_directories(${PROJECT_NAME} PUBLIC source2sdk) + +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + +target_compile_options(${PROJECT_NAME} PRIVATE + "-Wfatal-errors" + "-pedantic-errors" +) + +# Add a target that includes all headers of the generated library to check for compile-time errors + foreach(el ${source2sdk_headers}) string(APPEND generated_cpp_contents "#include \"${el}\"\n") endforeach() @@ -18,15 +30,11 @@ set(generated_cpp_file "${CMAKE_BINARY_DIR}/all_headers.cpp") file(WRITE ${generated_cpp_file} ${generated_cpp_contents}) -add_library(${PROJECT_NAME} ${generated_cpp_file}) +add_library(${PROJECT_NAME}-compile-test ${generated_cpp_file}) +set_target_properties(${PROJECT_NAME}-compile-test PROPERTIES LINKER_LANGUAGE CXX) +target_link_libraries(${PROJECT_NAME}-compile-test ${PROJECT_NAME}) -target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - -target_compile_options(${PROJECT_NAME} PRIVATE +target_compile_options(${PROJECT_NAME}-compile-test PRIVATE "-Wfatal-errors" "-pedantic-errors" ) - -target_precompile_headers(${PROJECT_NAME} PRIVATE ${source2sdk_sources}) - -set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) diff --git a/sdk-dummy/source2gen_user_types.hpp b/sdk-static/source2sdk/source2gen_user_types.hpp similarity index 100% rename from sdk-dummy/source2gen_user_types.hpp rename to sdk-static/source2sdk/source2gen_user_types.hpp diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 0485fca..bdecc03 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -33,7 +33,18 @@ namespace { using namespace std::string_view_literals; + /** + * Project structure is + * + * - CMakeLists.txt + * - ... + * - + * - some_module + * - some_header.hpp + */ constexpr std::string_view kOutDirName = "sdk"sv; + /// Include directory + constexpr std::string_view kSdkDirName = "source2sdk"sv; constexpr uint32_t kMaxReferencesForClassEmbed = 2; constexpr std::size_t kMinFieldCountForClassEmbed = 2; @@ -820,7 +831,7 @@ namespace { } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { - const std::string out_file_path = std::format("{}/{}/{}.hpp", kOutDirName, module_name, enum_.m_pszName); + const std::string out_file_path = std::format("{}/{}/{}/{}.hpp", kOutDirName, kSdkDirName, module_name, enum_.m_pszName); // @note: @es3n1n: init codegen // @@ -871,10 +882,10 @@ namespace { const auto names = GetRequiredNamesForClass(class_); for (const auto& include : names | std::views::filter([](const auto& el) { return el.source == NameSource::include; })) { - builder.include(std::format("\"{}/{}.hpp\"", include.module, include.type_name)); + builder.include(std::format("\"{}/{}/{}.hpp\"", kSdkDirName, include.module, include.type_name)); } - builder.include("\"source2gen_user_types.hpp\""); + builder.include(std::format("\"{}/source2gen_user_types.hpp\"", kSdkDirName)); builder.include(""); for (const auto& forward_declaration : names | std::views::filter([](const auto& el) { return el.source == NameSource::forward_declaration; })) { @@ -923,7 +934,7 @@ namespace sdk { std::cout << std::format("{}: Assembling module {} with {} enum(s) and {} class(es)", __FUNCTION__, module_name, enums.size(), classes.size()) << std::endl; - const std::filesystem::path out_directory_path = std::format("{}/{}", kOutDirName, module_name); + const std::filesystem::path out_directory_path = std::format("{}/{}/{}", kOutDirName, kSdkDirName, module_name); if (!std::filesystem::exists(out_directory_path)) std::filesystem::create_directories(out_directory_path); From ed4f4bb83e2052bf989180983b45cefe7db02ec6 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 5 Aug 2024 16:06:36 +0200 Subject: [PATCH 15/46] generate static assertions --- include/tools/codegen.h | 8 ++- premake5.lua | 148 -------------------------------------- sdk-static/CMakeLists.txt | 2 +- src/sdk/sdk.cpp | 10 +-- 4 files changed, 14 insertions(+), 154 deletions(-) delete mode 100644 premake5.lua diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 184a199..096e53e 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -1,7 +1,7 @@ // Copyright (C) 2024 neverlosecc // See end of file for extended copyright information. #pragma once -#include +#include #include #include #include @@ -190,6 +190,12 @@ namespace codegen { return *this; } + self_ref static_assert_size(std::string_view type_name, int expected_size, const bool move_cursor_to_next_line = true) { + assert(expected_size > 0); + + return push_line(std::format("static_assert(sizeof({}) == 0x{:x});", type_name, expected_size)); + } + self_ref comment(const std::string& text, const bool move_cursor_to_next_line = true) { return push_line(std::format("// {}", text), move_cursor_to_next_line); } diff --git a/premake5.lua b/premake5.lua deleted file mode 100644 index a9b0751..0000000 --- a/premake5.lua +++ /dev/null @@ -1,148 +0,0 @@ -newoption { - trigger = "game", - value = "CS2", - description = "Choose a particular game for dumping source 2 sdk", - allowed = { - { "CS2", "Counter-Strike 2" }, - { "DOTA2","Dota 2" }, - { "SBOX","S&BOX" }, - { "ARTIFACT2", "Artifact Foundry" }, - { "ARTIFACT1", "Artifact Classic" }, - { "UNDERLORDS", "Dota Underlords" }, - { "DESKJOB", "Aperture Desk Job" }, - { "HL_ALYX", "Half-Life: Alyx" }, - { "THE_LAB_ROBOT_REPAIR", "Portal: Aperture Robot Repair" } - }, - default = "CS2" -} - -workspace "source2gen" - architecture "x64" - startproject "source2gen" - - configurations - { - "Debug", - "Release", - "Dist" - } - - outputdir = "%{cfg.buildcfg}" - - IncludeDir = {} - - CppVersion = "C++latest" - MsvcToolset = "v143" - WindowsSdkVersion = "10.0" - - function DeclareMSVCOptions() - filter "system:windows" - staticruntime "Off" - systemversion (WindowsSdkVersion) - toolset (MsvcToolset) - cppdialect (CppVersion) - - defines - { - "_CRT_SECURE_NO_WARNINGS", - "NOMINMAX", - "WIN32_LEAN_AND_MEAN", - "_WIN32_WINNT=0x601" -- Support Windows 7 - } - - disablewarnings - { - "4100", -- C4100: unreferenced formal parameter - "4201", -- C4201: nameless struct/union - "4307", -- C4307: integral constant overflow - "4311", -- C4311: 'variable' : pointer truncation from 'type' to 'type' - "4302", -- C4302: 'conversion' : truncation from 'type 1' to 'type 2' - "4267", -- C4267: 'var' : conversion from 'size_t' to 'type', possible loss of data - "4244" -- C4244: 'conversion' conversion from 'type1' to 'type2', possible loss of data - } - end - - function DeclareDebugOptions() - filter "configurations:Debug" - defines { "_DEBUG" } - symbols "On" - flags { "MultiProcessorCompile" } - filter "not configurations:Debug" - defines { "NDEBUG" } - end - - project "source2gen" - location "source2gen" - kind "ConsoleApp" - language "C++" - buildoptions { "/bigobj" } - - targetdir ("bin/" .. outputdir) - objdir ("bin/int/" .. outputdir .. "/%{prj.name}") - - defines - { - } - - files - { - ".clang-format", - "include/**.h", - "include/**.hpp", - "src/**.cpp", - } - - includedirs - { - "include", - "src" - } - - libdirs - { - "bin/lib" - } - - DeclareMSVCOptions() - DeclareDebugOptions() - - flags { "NoImportLib", "Maps", "MultiProcessorCompile" } - - filter "configurations:Debug" - defines { "source2gen_DEBUG" } - - filter "configurations:Release" - defines { "source2ge_RELEASE" } - optimize "speed" - - filter "configurations:Dist" - flags { "LinkTimeOptimization", "FatalCompileWarnings" } - defines { "source2ge_DIST" } - optimize "speed" - - filter { "options:game=CS2" } - defines { "CS2" } - - filter { "options:game=DOTA2" } - defines { "DOTA2" } - - filter { "options:game=SBOX" } - defines { "SBOX" } - - filter { "options:game=ARTIFACT2" } - defines { "ARTIFACT2" } - - filter { "options:game=ARTIFACT1" } - defines { "ARTIFACT1" } - - filter { "options:game=UNDERLORDS" } - defines { "UNDERLORDS" } - - filter { "options:game=DESKJOB" } - defines { "DESKJOB" } - - filter { "options:game=HL_ALYX" } - defines { "HL_ALYX" } - - filter { "options:game=THE_LAB_ROBOT_REPAIR" } - defines { "THE_LAB_ROBOT_REPAIR" } \ No newline at end of file diff --git a/sdk-static/CMakeLists.txt b/sdk-static/CMakeLists.txt index 13e99e6..b800026 100644 --- a/sdk-static/CMakeLists.txt +++ b/sdk-static/CMakeLists.txt @@ -11,7 +11,7 @@ project(source2sdk file(GLOB_RECURSE source2sdk_headers "./**.hpp") add_library(${PROJECT_NAME} ${source2sdk_headers}) -target_include_directories(${PROJECT_NAME} PUBLIC source2sdk) +target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index bdecc03..ab89c4a 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -828,6 +828,8 @@ namespace { builder.comment("No schema binary for binding"); builder.end_block(); + + builder.static_assert_size(class_info.m_pszName, class_info.GetSize()); } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { @@ -861,7 +863,7 @@ namespace { // std::ofstream f(out_file_path, std::ios::out); f << builder.str(); - if (f.bad()) { + if (!f.good()) { std::cerr << std::format("Could not write to {}: {}", out_file_path, std::strerror(errno)) << std::endl; // This std::exit() is bad. Instead, we could return the dumped // header name and content to the caller in a std::expected. Let the @@ -872,7 +874,7 @@ namespace { } void GenerateClassSdk(std::string_view module_name, const CSchemaClassBinding& class_) { - const std::filesystem::path out_file_path = std::format("{}/{}/{}.hpp", kOutDirName, module_name, class_.m_pszName); + const std::string out_file_path = std::format("{}/{}/{}/{}.hpp", kOutDirName, kSdkDirName, module_name, class_.m_pszName); // @note: @es3n1n: init codegen // @@ -915,8 +917,8 @@ namespace { // std::ofstream f(out_file_path, std::ios::out); f << builder.str(); - if (f.bad()) { - std::cerr << std::format("Could not write to {}: {}", out_file_path.string(), std::strerror(errno)) << std::endl; + if (!f.good()) { + std::cerr << std::format("Could not write to {}: {}", out_file_path, std::strerror(errno)) << std::endl; // This std::exit() is bad. Instead, we could return the dumped // header name and content to the caller in a std::expected. Let the // caller write the file. That would also allow the caller to choose From f3ae29f034d503cd97775c7497832466d58bed93 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 5 Aug 2024 16:29:12 +0200 Subject: [PATCH 16/46] generate static_assert() for offsetof() --- include/tools/codegen.h | 9 ++++++++- src/sdk/sdk.cpp | 13 ++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 096e53e..79e76f1 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -193,7 +193,14 @@ namespace codegen { self_ref static_assert_size(std::string_view type_name, int expected_size, const bool move_cursor_to_next_line = true) { assert(expected_size > 0); - return push_line(std::format("static_assert(sizeof({}) == 0x{:x});", type_name, expected_size)); + return push_line(std::format("static_assert(sizeof({}) == {:#x});", type_name, expected_size)); + } + + self_ref static_assert_offset(std::string_view class_name, std::string_view prop_name, int expected_offset, + const bool move_cursor_to_next_line = true) { + assert(expected_offset >= 0); + + return push_line(std::format("static_assert(offsetof({}, {}) == {:#x});", class_name, prop_name, expected_offset)); } self_ref comment(const std::string& text, const bool move_cursor_to_next_line = true) { diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index ab89c4a..ac35470 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -673,9 +673,7 @@ namespace { // @note: @es3n1n: insert padding if needed // const auto expected_offset = state.last_field_offset + state.last_field_size; - if (state.last_field_offset && state.last_field_size && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && - !state.assembling_bitfield) { - + if (state.last_field_size && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { builder.access_modifier("private") .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) .reset_tabs_count() @@ -728,7 +726,7 @@ namespace { // @note: @es3n1n: update state // - if (field.m_nSingleInheritanceOffset && field_size) { + if (field_size != 0) { state.last_field_offset = field.m_nSingleInheritanceOffset; state.last_field_size = static_cast(field_size); } @@ -740,7 +738,7 @@ namespace { builder.prop(var_info.m_type, var_info.formatted_name(), false); if (!var_info.is_bitfield()) { - builder.reset_tabs_count().comment(std::format("{:#x}", field.m_nSingleInheritanceOffset), false).restore_tabs_count(); + builder.reset_tabs_count().comment(std::format("{:#x} ({:#x})", field.m_nSingleInheritanceOffset, field_size), false).restore_tabs_count(); cached_fields.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); } builder.next_line(); @@ -829,6 +827,10 @@ namespace { builder.end_block(); + // TODO: when we have a CLI parse: make static_assert() generation optional because users might know not care about errors in file they don't use + for (const auto& field : class_info.GetFields()) { + builder.static_assert_offset(class_info.m_pszName, field.m_pszName, field.m_nSingleInheritanceOffset); + } builder.static_assert_size(class_info.m_pszName, class_info.GetSize()); } @@ -888,6 +890,7 @@ namespace { } builder.include(std::format("\"{}/source2gen_user_types.hpp\"", kSdkDirName)); + builder.include(""); // for offsetof() builder.include(""); for (const auto& forward_declaration : names | std::views::filter([](const auto& el) { return el.source == NameSource::forward_declaration; })) { From 276f9bf3b3e428baa40f048739bdc7bf7f8efa0a Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 5 Aug 2024 16:41:26 +0200 Subject: [PATCH 17/46] pad class ends --- src/sdk/sdk.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index ac35470..27dc086 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -634,7 +634,8 @@ namespace { // @note: @es3n1n: and finally insert a pad // if (expected_pad_size > 0) // @fixme: @es3n1n: this is wrong, i probably should check for collisions instead - builder.access_modifier("private") + // TOOD: remove access_modifier() altogether? + builder.access_modifier("public") .struct_padding(parent_class_size, expected_pad_size, false, true) .reset_tabs_count() .comment(std::format("{:#x}", parent_class_size)) @@ -674,7 +675,7 @@ namespace { // const auto expected_offset = state.last_field_offset + state.last_field_size; if (state.last_field_size && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { - builder.access_modifier("private") + builder.access_modifier("public") .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) .reset_tabs_count() .comment(std::format("{:#x}", expected_offset)) @@ -762,6 +763,16 @@ namespace { state.assembling_bitfield = false; } + // pad the class end + const auto last_field_end = state.last_field_offset + state.last_field_size; + const auto end_pad = class_.GetSize() - last_field_end; + if (end_pad != 0) { + builder.struct_padding(last_field_end, end_pad, true, true); + } else if (end_pad < 0) [[unlikely]] { + throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), + -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; + } + // @note: @es3n1n: dump static fields // if (class_info.m_nStaticFieldsSize) { From 5275f172a1daf59780dd7bebe9cfc2a6154920f6 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Tue, 6 Aug 2024 16:54:48 +0200 Subject: [PATCH 18/46] pad class ends and bitfields --- include/tools/codegen.h | 8 +++---- src/sdk/sdk.cpp | 51 +++++++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 79e76f1..a8b63a2 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -226,7 +226,9 @@ namespace codegen { } self_ref struct_padding(const std::optional pad_offset, const std::size_t padding_size, const bool move_cursor_to_next_line = true, - const bool is_private_field = false, const std::size_t bitfield_size = 0ull) { + const bool is_private_field = false, const int bitfield_size = 0) { + assert(bitfield_size >= 0); + const auto bytes = (bitfield_size == 0) ? padding_size : bitfield_size / 8; const auto remaining_bits = bitfield_size % 8; @@ -238,11 +240,9 @@ namespace codegen { auto pad_name = pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value()) : std::format("pad_{:d}", _pads_count++); if (bytes != 0) { - pad_name = pad_name + std::format("[{:#x}]", bytes); + prop(type_name, std::format("{}[{:#x}]", pad_name, bytes), move_cursor_to_next_line); } - prop(type_name, pad_name, move_cursor_to_next_line); - if (remaining_bits != 0) { auto remainder_pad_name = pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value() + bytes) : std::format("pad_{:d}", _pads_count++); diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 27dc086..6fa6617 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -627,9 +627,7 @@ namespace { const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size; - std::ptrdiff_t expected_pad_size = first_field_offset.value_or(class_size_without_parent); - if (expected_pad_size) // @note: @es3n1n: if there's a pad size we should account the parent class size - expected_pad_size -= parent_class_size; + const auto expected_pad_size = first_field_offset.transform([&](auto e) { return e - parent_class_size; }).value_or(class_size_without_parent); // @note: @es3n1n: and finally insert a pad // @@ -644,6 +642,7 @@ namespace { // @todo: @es3n1n: if for some mysterious reason this class describes fields // of the base class we should handle it too. if (class_parent && first_field_offset.has_value() && first_field_offset.value() < class_parent->m_nSizeOf) { + // TOOD: there' a `throw` for offset overflows down below. make error reporting consistent. builder.comment( std::format("Collision detected({:#x}->{:#x}), output may be wrong.", first_field_offset.value_or(0), class_parent->m_nSizeOf)); state.collision_end_offset = class_parent->m_nSizeOf; @@ -693,6 +692,7 @@ namespace { // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union // if (state.assembling_bitfield && !var_info.is_bitfield()) { + // TOOD: this code is duplicate const auto expected_union_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset; const auto expected_union_size_bits = expected_union_size_bytes * 8; @@ -750,12 +750,13 @@ namespace { if (state.assembling_bitfield) { const auto actual_union_size_bits = state.total_bits_count_in_union; - // @note: @es3n1n: apply 8 bytes align + // @note: @es3n1n: apply 1 byte align // - const auto expected_union_size_bits = actual_union_size_bits + (actual_union_size_bits % 8); + const auto expected_union_size_bits = actual_union_size_bits + (8 - (actual_union_size_bits % 8)); - if (expected_union_size_bits > actual_union_size_bits) + if (expected_union_size_bits > actual_union_size_bits) { builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); + } builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); @@ -763,14 +764,17 @@ namespace { state.assembling_bitfield = false; } - // pad the class end - const auto last_field_end = state.last_field_offset + state.last_field_size; - const auto end_pad = class_.GetSize() - last_field_end; - if (end_pad != 0) { - builder.struct_padding(last_field_end, end_pad, true, true); - } else if (end_pad < 0) [[unlikely]] { - throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), - -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; + // pad the class end. if this is an empty class, padding will already have been added by the code above + if (class_.m_nFieldSize != 0) { + const auto last_field_end = state.last_field_offset + state.last_field_size; + const auto end_pad = class_.GetSize() - last_field_end; + + if (end_pad != 0) { + builder.struct_padding(last_field_end, end_pad, true, true); + } else if (end_pad < 0) [[unlikely]] { + throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), + -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; + } } // @note: @es3n1n: dump static fields @@ -838,10 +842,23 @@ namespace { builder.end_block(); - // TODO: when we have a CLI parse: make static_assert() generation optional because users might know not care about errors in file they don't use - for (const auto& field : class_info.GetFields()) { - builder.static_assert_offset(class_info.m_pszName, field.m_pszName, field.m_nSingleInheritanceOffset); + // TODO: when we have a CLI parse: make static_assert() generation optional because users might not care about errors in file they don't use + if (class_info.m_nBaseClassSize == 0) { + for (const auto& field : class_info.GetFields()) { + if (field.m_pSchemaType->m_unTypeCategory == ETypeCategory::Schema_Bitfield) { + builder.comment(std::format("Cannot assert offset of bitfield {}::{}", class_info.m_pszName, field.m_pszName)); + } else { + builder.static_assert_offset(class_info.m_pszName, field.m_pszName, field.m_nSingleInheritanceOffset); + } + } + } else { + if (class_info.m_nFieldSize != 0) { + builder.comment(std::format( + "Cannot assert offsets of fields in {}. It is not a standard-layout class because it has a base class with non-static data members.", + class_info.m_pszName)); + } } + builder.next_line(); builder.static_assert_size(class_info.m_pszName, class_info.GetSize()); } From 7b447f7f95e0e69fead2608e67d412652604d871 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Wed, 7 Aug 2024 19:48:48 +0200 Subject: [PATCH 19/46] reduce bitfield padding --- include/tools/codegen.h | 4 ++-- src/sdk/sdk.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/tools/codegen.h b/include/tools/codegen.h index a8b63a2..55cd8f5 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -263,8 +263,8 @@ namespace codegen { return push_line(move_cursor_to_next_line ? "};" : "}; ", move_cursor_to_next_line); } - self_ref begin_bitfield_block() { - return comment("start of bitfield block"); + self_ref begin_bitfield_block(std::ptrdiff_t offset) { + return comment(std::format("start of bitfield block at {:#x}", offset)); } self_ref end_bitfield_block(const bool move_cursor_to_next_line = true) { diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 6fa6617..64d0cdc 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -685,8 +685,9 @@ namespace { // @note: @es3n1n: begin union if we're assembling bitfields // if (!state.assembling_bitfield && var_info.is_bitfield()) { - builder.begin_bitfield_block(); + builder.begin_bitfield_block(expected_offset); state.assembling_bitfield = true; + state.last_field_offset = expected_offset; } // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union @@ -703,7 +704,7 @@ namespace { std::format("Unexpected union size: {}. Expected: {}", state.total_bits_count_in_union, expected_union_size_bits)); if (expected_union_size_bits > state.total_bits_count_in_union) - builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); + builder.struct_padding(expected_offset, 0, true, false, expected_union_size_bits - actual_union_size_bits); state.last_field_offset += expected_union_size_bytes; state.last_field_size = expected_union_size_bytes; @@ -750,12 +751,11 @@ namespace { if (state.assembling_bitfield) { const auto actual_union_size_bits = state.total_bits_count_in_union; - // @note: @es3n1n: apply 1 byte align - // - const auto expected_union_size_bits = actual_union_size_bits + (8 - (actual_union_size_bits % 8)); + // apply 1 byte align + const auto expected_union_size_bits = actual_union_size_bits + ((8 - (actual_union_size_bits % 8)) % 8); if (expected_union_size_bits > actual_union_size_bits) { - builder.struct_padding(std::nullopt, 0, true, false, expected_union_size_bits - actual_union_size_bits); + builder.struct_padding(state.last_field_offset + state.last_field_size, 0, true, false, expected_union_size_bits - actual_union_size_bits); } builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); From 337b9a31c201e63d99f90cade49cb4398e32e627 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 11 Aug 2024 17:17:15 +0200 Subject: [PATCH 20/46] fix pad sizes in special cases --- include/sdk/interfaces/schemasystem/schema.h | 7 ++ include/tools/codegen.h | 6 +- .../source2sdk/source2gen_user_types.hpp | 4 +- src/sdk/sdk.cpp | 79 ++++++++++--------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 35f35b4..6930192 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -454,6 +454,9 @@ class CSchemaType_DeclaredClass : public CSchemaType { bool m_bGlobalPromotionRequired; }; +static_assert(offsetof(CSchemaType_DeclaredClass, m_pClassInfo) == 0x20); +static_assert(sizeof(CSchemaType_DeclaredClass) == 0x30); + class CSchemaType_DeclaredEnum : public CSchemaType { public: CSchemaEnumBinding* m_pClassInfo; @@ -501,6 +504,8 @@ class CSchemaType_Atomic_CollectionOfT : public CSchemaType_Atomic_T { std::uint16_t m_unElementSize; }; +static_assert(offsetof(CSchemaType_Atomic_CollectionOfT, m_pFn) == 0x38); + class CSchemaType_Atomic_TF : public CSchemaType_Atomic_T { public: int m_nFuncPtrSize; @@ -572,6 +577,8 @@ struct SchemaClassFieldData_t { SchemaMetadataEntryData_t* m_pMetadata; // 0x0018 }; +static_assert(sizeof(SchemaClassFieldData_t) == 0x20); + struct SchemaStaticFieldData_t { const char* m_pszName; // 0x0000 CSchemaType* m_pSchemaType; // 0x0008 diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 55cd8f5..11c5aa6 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -193,14 +193,14 @@ namespace codegen { self_ref static_assert_size(std::string_view type_name, int expected_size, const bool move_cursor_to_next_line = true) { assert(expected_size > 0); - return push_line(std::format("static_assert(sizeof({}) == {:#x});", type_name, expected_size)); + return push_line(std::format("static_assert(sizeof({}) == {:#x});", escape_name(type_name), expected_size)); } self_ref static_assert_offset(std::string_view class_name, std::string_view prop_name, int expected_offset, const bool move_cursor_to_next_line = true) { assert(expected_offset >= 0); - return push_line(std::format("static_assert(offsetof({}, {}) == {:#x});", class_name, prop_name, expected_offset)); + return push_line(std::format("static_assert(offsetof({}, {}) == {:#x});", escape_name(class_name), prop_name, expected_offset)); } self_ref comment(const std::string& text, const bool move_cursor_to_next_line = true) { @@ -286,7 +286,7 @@ namespace codegen { return *this; } - static std::string escape_name(const std::string& name) { + static std::string escape_name(const std::string_view name) { std::string result; result.resize(name.size()); diff --git a/sdk-static/source2sdk/source2gen_user_types.hpp b/sdk-static/source2sdk/source2gen_user_types.hpp index ab7ec21..e67c421 100644 --- a/sdk-static/source2sdk/source2gen_user_types.hpp +++ b/sdk-static/source2sdk/source2gen_user_types.hpp @@ -5,7 +5,7 @@ template using CAnimValue = char[0x08]; -using CAnimVariant = char[0x14]; +using CAnimVariant = char[0x11]; // size is a guess template using CAnimScriptParam = char[0x08]; @@ -63,7 +63,7 @@ using CUtlVector = char[0x18]; template using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; template -using CUtlLeanVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; +using CUtlLeanVectorFixedGrowable = char[0x10 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; template using C_UtlVectorEmbeddedNetworkVar = char[0x50]; template diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 64d0cdc..1e13b68 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -595,6 +595,7 @@ namespace { // @note: @es3n1n: get parent name // const std::string parent_class_name = (class_parent != nullptr) ? MaybeWithModuleName(*class_parent->m_pTypeScope, class_parent->m_pszName) : ""; + const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; // @note: @es3n1n: start class // @@ -606,13 +607,14 @@ namespace { // @note: @es3n1n: field assembling state // struct { - std::size_t last_field_size = 0ull; - std::size_t last_field_offset = 0ull; + std::optional last_field_size = std::nullopt; + std::optional last_field_offset = std::nullopt; bool assembling_bitfield = false; std::size_t total_bits_count_in_union = 0ull; std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var - } state; + // TOOD: inconsistent optional vs 0 + } state = {.last_field_size = parent_class_size}; std::list> cached_fields{}; std::list cached_datamap_fields{}; @@ -624,25 +626,13 @@ namespace { if (const auto* first_field = (class_.m_pFields == nullptr) ? nullptr : &class_.m_pFields[0]; first_field) first_field_offset = first_field->m_nSingleInheritanceOffset; - const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size; const auto expected_pad_size = first_field_offset.transform([&](auto e) { return e - parent_class_size; }).value_or(class_size_without_parent); - // @note: @es3n1n: and finally insert a pad - // - if (expected_pad_size > 0) // @fixme: @es3n1n: this is wrong, i probably should check for collisions instead - // TOOD: remove access_modifier() altogether? - builder.access_modifier("public") - .struct_padding(parent_class_size, expected_pad_size, false, true) - .reset_tabs_count() - .comment(std::format("{:#x}", parent_class_size)) - .restore_tabs_count(); - // @todo: @es3n1n: if for some mysterious reason this class describes fields // of the base class we should handle it too. if (class_parent && first_field_offset.has_value() && first_field_offset.value() < class_parent->m_nSizeOf) { - // TOOD: there' a `throw` for offset overflows down below. make error reporting consistent. builder.comment( std::format("Collision detected({:#x}->{:#x}), output may be wrong.", first_field_offset.value_or(0), class_parent->m_nSizeOf)); state.collision_end_offset = class_parent->m_nSizeOf; @@ -670,10 +660,20 @@ namespace { const auto [type_name, array_sizes] = GetType(*field.m_pSchemaType); const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); + // if (std::string_view{type_name}.contains("CUtlLeanVectorFixedGrowable")) { + // auto* p = (CSchemaType_Atomic_CollectionOfT*)field.m_pSchemaType; + // std::uint64_t a{}; + // std::uint64_t b{}; + // std::uint64_t c{}; + // const auto r = p->m_pFn(SchemaAtomicFunctionIndex::Schema_Atomic_Get_Count, &a, &b, &c); + // std::cout << "TOOD: here " << field.m_pSchemaType->m_pszName << ' ' << std::format("{} {} {} -> {:#x}", a, b, c, r) << std::endl; + // } + // @note: @es3n1n: insert padding if needed // - const auto expected_offset = state.last_field_offset + state.last_field_size; - if (state.last_field_size && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { + const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && + !state.assembling_bitfield) { builder.access_modifier("public") .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) .reset_tabs_count() @@ -687,27 +687,27 @@ namespace { if (!state.assembling_bitfield && var_info.is_bitfield()) { builder.begin_bitfield_block(expected_offset); state.assembling_bitfield = true; - state.last_field_offset = expected_offset; } // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union // if (state.assembling_bitfield && !var_info.is_bitfield()) { // TOOD: this code is duplicate - const auto expected_union_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset; - const auto expected_union_size_bits = expected_union_size_bytes * 8; + const auto expected_bitfield_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset.value_or(0); + const auto expected_union_size_bits = expected_bitfield_size_bytes * 8; const auto actual_union_size_bits = state.total_bits_count_in_union; if (expected_union_size_bits < state.total_bits_count_in_union) throw std::runtime_error( - std::format("Unexpected union size: {}. Expected: {}", state.total_bits_count_in_union, expected_union_size_bits)); + std::format("Unexpected union size: {} bits. Expected: {} bits", state.total_bits_count_in_union, expected_union_size_bits)); - if (expected_union_size_bits > state.total_bits_count_in_union) + if (expected_union_size_bits > state.total_bits_count_in_union) { builder.struct_padding(expected_offset, 0, true, false, expected_union_size_bits - actual_union_size_bits); + } - state.last_field_offset += expected_union_size_bytes; - state.last_field_size = expected_union_size_bytes; + state.last_field_offset = state.last_field_offset.value_or(0) + expected_bitfield_size_bytes; + state.last_field_size = expected_bitfield_size_bytes; builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); @@ -752,29 +752,31 @@ namespace { const auto actual_union_size_bits = state.total_bits_count_in_union; // apply 1 byte align - const auto expected_union_size_bits = actual_union_size_bits + ((8 - (actual_union_size_bits % 8)) % 8); + const auto expected_bitfield_size_bits = actual_union_size_bits + ((8 - (actual_union_size_bits % 8)) % 8); - if (expected_union_size_bits > actual_union_size_bits) { - builder.struct_padding(state.last_field_offset + state.last_field_size, 0, true, false, expected_union_size_bits - actual_union_size_bits); + if (expected_bitfield_size_bits > actual_union_size_bits) { + builder.struct_padding(state.last_field_offset.value_or(0) + state.last_field_size.value_or(0), 0, true, false, + expected_bitfield_size_bits - actual_union_size_bits); } - builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); + builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_bitfield_size_bits)).restore_tabs_count(); state.total_bits_count_in_union = 0; state.assembling_bitfield = false; + + state.last_field_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + state.last_field_size = expected_bitfield_size_bits / 8; } - // pad the class end. if this is an empty class, padding will already have been added by the code above - if (class_.m_nFieldSize != 0) { - const auto last_field_end = state.last_field_offset + state.last_field_size; - const auto end_pad = class_.GetSize() - last_field_end; + // pad the class end. + const auto last_field_end = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + const auto end_pad = class_.GetSize() - last_field_end; - if (end_pad != 0) { - builder.struct_padding(last_field_end, end_pad, true, true); - } else if (end_pad < 0) [[unlikely]] { - throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), - -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; - } + if (end_pad != 0) { + builder.struct_padding(last_field_end, end_pad, true, true); + } else if (end_pad < 0) [[unlikely]] { + throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), + -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; } // @note: @es3n1n: dump static fields @@ -818,6 +820,7 @@ namespace { std::string field_name = var_info.formatted_name(); // @note: @og: if schema dump already has this field, then just skip it + if (const auto it = std::ranges::find_if(cached_fields, [t, field_name](const auto& f) { return f.first == field_name && f.second == t->m_iOffset; }); it != cached_fields.end()) From 7f4665b66187d3a707ce554a2cb14c24480c2edd Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 11 Aug 2024 17:19:38 +0200 Subject: [PATCH 21/46] remove premake in favor of cmake --- README.md | 33 +------------------- include/sdk/interfaces/schemasystem/schema.h | 2 +- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index d8bd54d..ebb3113 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ These instructions will help you set up the project on your local machine for de #### Windows - Visual Studio 2019 or newer -- premake5 +- CMake #### Linux @@ -81,40 +81,10 @@ Possible options are: `CS2`, `SBOX`, `ARTIFACT2`, `ARTIFACT1`, `DOTA2`, `UNDERLO or -You can use premake5 options and specify which game you want to dump: - -```bash - --game=CS2 - Choose a particular game for dumping source 2 sdk; one of: - ARTIFACT1 Artifact Classic - ARTIFACT2 Artifact Foundry - CS2 Counter-Strike 2 - DESKJOB Aperture Desk Job - DOTA2 Dota 2 - HL_ALYX Half-Life: Alyx - SBOX S&BOX - THE_LAB_ROBOT_REPAIR Portal: Aperture Robot Repair - UNDERLORDS Dota Underlords -``` - -or - When using CMake, you can set `cmake -DSOURCE2GEN_GAME=CS2` ### Building the project -#### With premake5 - -- Open a command prompt or terminal in the project's root directory. -- Run the following command to generate the Visual Studio solution: - -```bash -premake5 vs2019 --game=CS2 -``` - -- Open the generated source2gen.sln file in Visual Studio. -- Build the solution in the desired configuration (Debug, Release, or Dist). - #### With CMake - Open a command prompt or terminal in the project's root directory. @@ -140,7 +110,6 @@ This project is made possible by the contributions of various individuals and pr This project also utilizes the following open-source libraries: -- **[Premake](https://github.com/premake/premake-core)** - Build configuration tool - **[CMake](https://github.com/Kitware/CMake)** - Build tool If you've contributed to the project and would like to be listed here, please submit a [pull request](https://github.com/neverlosecc/source2gen/pulls) with your information added to the credits. diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 6930192..7ae61bb 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -168,7 +168,7 @@ enum { }; #else - #error No implementation defined, please re-generate project with premake5 + #error No implementation defined, please set SOURCE2GEN_GAME and re-generate the project #endif class ISaveRestoreOps; From b54ee96270a66d4fb3da30f63f3de175a37323d1 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Thu, 15 Aug 2024 18:36:30 +0200 Subject: [PATCH 22/46] use larger integer types for consecutive bitfields --- include/tools/field_parser.h | 3 + src/sdk/sdk.cpp | 169 +++++++++++++++++++---------------- src/tools/field_parser.cpp | 41 ++++----- 3 files changed, 111 insertions(+), 102 deletions(-) diff --git a/include/tools/field_parser.h b/include/tools/field_parser.h index a2671f5..7fa9880 100644 --- a/include/tools/field_parser.h +++ b/include/tools/field_parser.h @@ -67,6 +67,9 @@ namespace field_parser { } }; + [[nodiscard]] + std::string guess_bitfield_type(std::size_t bits_count); + /// @return @ref std::nullopt if type_name is not a built-in type [[nodiscard]] std::optional type_name_to_cpp(std::string_view type_name); diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 1e13b68..9cff422 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -3,6 +3,8 @@ // ReSharper disable CppClangTidyClangDiagnosticLanguageExtensionToken #include "sdk/sdk.h" + +#include #include #include #include @@ -11,6 +13,7 @@ #include #include #include +#include #include #include @@ -577,6 +580,59 @@ namespace { return result; } + // TOOD: Move up + + struct BitfieldEntry { + std::string name{}; + std::size_t size{}; + // TOOD: document lifetime + std::vector metadata{}; + }; + + struct ClassAssemblyState { + std::optional last_field_size = std::nullopt; + std::optional last_field_offset = std::nullopt; + bool assembling_bitfield = false; + std::vector bitfield = {}; + + std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var + // TOOD: inconsistent optional vs 0 + }; + + [[nodiscard]] + ClassAssemblyState AssembleBitfield(codegen::generator_t& builder, ClassAssemblyState&& state, int expected_offset) { + state.assembling_bitfield = false; + + std::size_t exact_bitfield_size_bits = 0; + for (const auto& entry : state.bitfield) { + exact_bitfield_size_bits += entry.size; + } + const auto type_name = field_parser::guess_bitfield_type(exact_bitfield_size_bits); + + builder.begin_bitfield_block(expected_offset); + + for (const auto& entry : state.bitfield) { + // TOOD: duplicate code + for (const auto& field_metadata : entry.metadata) { + if (auto data = GetMetadataValue(field_metadata); data.empty()) + builder.comment(std::format("metadata: {}", field_metadata.m_szName)); + else + builder.comment(std::format("metadata: {} \"{}\"", field_metadata.m_szName, data)); + } + + builder.prop(type_name, entry.name, true); + } + + builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", exact_bitfield_size_bits)).restore_tabs_count(); + + state.bitfield.clear(); + state.last_field_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + // call to bit_ceil() relies on guess_bitfield_type() returning the next highest power of 2 + state.last_field_size = std::bit_ceil(std::max(std::size_t{8}, exact_bitfield_size_bits)) / 8; + + return state; + } + void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { struct cached_datamap_t { std::string type_; @@ -606,15 +662,8 @@ namespace { // @note: @es3n1n: field assembling state // - struct { - std::optional last_field_size = std::nullopt; - std::optional last_field_offset = std::nullopt; - bool assembling_bitfield = false; - std::size_t total_bits_count_in_union = 0ull; - - std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var - // TOOD: inconsistent optional vs 0 - } state = {.last_field_size = parent_class_size}; + // TOOD: inconsistent optional vs 0 + ClassAssemblyState state = {.last_field_size = parent_class_size}; std::list> cached_fields{}; std::list cached_datamap_fields{}; @@ -660,18 +709,23 @@ namespace { const auto [type_name, array_sizes] = GetType(*field.m_pSchemaType); const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); - // if (std::string_view{type_name}.contains("CUtlLeanVectorFixedGrowable")) { - // auto* p = (CSchemaType_Atomic_CollectionOfT*)field.m_pSchemaType; - // std::uint64_t a{}; - // std::uint64_t b{}; - // std::uint64_t c{}; - // const auto r = p->m_pFn(SchemaAtomicFunctionIndex::Schema_Atomic_Get_Count, &a, &b, &c); - // std::cout << "TOOD: here " << field.m_pSchemaType->m_pszName << ' ' << std::format("{} {} {} -> {:#x}", a, b, c, r) << std::endl; - // } + const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + + // Collect all bitfield entries and emit them later. We need to know + // how large the bitfield is in order to choose the right type. We + // only know how large the bitfield is once we've reached its end. + if (var_info.is_bitfield()) { + state.assembling_bitfield = true; + state.bitfield.emplace_back(BitfieldEntry{ + .name = var_info.formatted_name(), + .size = var_info.m_bitfield_size, + .metadata = std::vector(field.m_pMetadata, field.m_pMetadata + field.m_nMetadataSize), + }); + continue; + } // @note: @es3n1n: insert padding if needed // - const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { builder.access_modifier("public") @@ -682,43 +736,15 @@ namespace { .access_modifier("public"); } - // @note: @es3n1n: begin union if we're assembling bitfields - // - if (!state.assembling_bitfield && var_info.is_bitfield()) { - builder.begin_bitfield_block(expected_offset); - state.assembling_bitfield = true; - } - - // @note: @es3n1n: if we are done with bitfields we should insert a pad and finish union - // - if (state.assembling_bitfield && !var_info.is_bitfield()) { - // TOOD: this code is duplicate - const auto expected_bitfield_size_bytes = field.m_nSingleInheritanceOffset - state.last_field_offset.value_or(0); - const auto expected_union_size_bits = expected_bitfield_size_bytes * 8; - - const auto actual_union_size_bits = state.total_bits_count_in_union; - - if (expected_union_size_bits < state.total_bits_count_in_union) - throw std::runtime_error( - std::format("Unexpected union size: {} bits. Expected: {} bits", state.total_bits_count_in_union, expected_union_size_bits)); - - if (expected_union_size_bits > state.total_bits_count_in_union) { - builder.struct_padding(expected_offset, 0, true, false, expected_union_size_bits - actual_union_size_bits); - } - - state.last_field_offset = state.last_field_offset.value_or(0) + expected_bitfield_size_bytes; - state.last_field_size = expected_bitfield_size_bytes; - - builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_union_size_bits)).restore_tabs_count(); - - state.total_bits_count_in_union = 0ull; - state.assembling_bitfield = false; + // This is the end of a bitfield + if (!var_info.is_bitfield() && state.assembling_bitfield) { + state = AssembleBitfield(builder, std::move(state), expected_offset); } // @note: @es3n1n: dump metadata // for (auto j = 0; j < field.m_nMetadataSize; j++) { - auto field_metadata = field.m_pMetadata[j]; + const auto field_metadata = field.m_pMetadata[j]; if (auto data = GetMetadataValue(field_metadata); data.empty()) builder.comment(std::format("metadata: {}", field_metadata.m_szName)); @@ -732,40 +758,27 @@ namespace { state.last_field_offset = field.m_nSingleInheritanceOffset; state.last_field_size = static_cast(field_size); } - if (var_info.is_bitfield()) - state.total_bits_count_in_union += var_info.m_bitfield_size; // @note: @es3n1n: push prop // - builder.prop(var_info.m_type, var_info.formatted_name(), false); - - if (!var_info.is_bitfield()) { - builder.reset_tabs_count().comment(std::format("{:#x} ({:#x})", field.m_nSingleInheritanceOffset, field_size), false).restore_tabs_count(); - cached_fields.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); + if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { + // HACKHACK: this is a hack to get the size of template types right + builder.prop("char", std::format("{}[{:#x}]", var_info.formatted_name(), field.m_pSchemaType->GetSize().value()), false); + builder.comment(field.m_pSchemaType->m_pszName, false); + } else { + builder.prop(var_info.m_type, var_info.formatted_name(), false); } + + builder.reset_tabs_count().comment(std::format("{:#x} ({:#x})", field.m_nSingleInheritanceOffset, field_size), false).restore_tabs_count(); + cached_fields.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); + builder.next_line(); } - // @note: @es3n1n: if struct ends with union we should end union before ending the class + // @note: @es3n1n: if struct ends with bitfield we should end bitfield before ending the class // if (state.assembling_bitfield) { - const auto actual_union_size_bits = state.total_bits_count_in_union; - - // apply 1 byte align - const auto expected_bitfield_size_bits = actual_union_size_bits + ((8 - (actual_union_size_bits % 8)) % 8); - - if (expected_bitfield_size_bits > actual_union_size_bits) { - builder.struct_padding(state.last_field_offset.value_or(0) + state.last_field_size.value_or(0), 0, true, false, - expected_bitfield_size_bits - actual_union_size_bits); - } - - builder.end_bitfield_block(false).reset_tabs_count().comment(std::format("{:d} bits", expected_bitfield_size_bits)).restore_tabs_count(); - - state.total_bits_count_in_union = 0; - state.assembling_bitfield = false; - - state.last_field_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); - state.last_field_size = expected_bitfield_size_bits / 8; + state = AssembleBitfield(builder, std::move(state), state.last_field_offset.value_or(0) + state.last_field_size.value_or(0)); } // pad the class end. @@ -856,9 +869,9 @@ namespace { } } else { if (class_info.m_nFieldSize != 0) { - builder.comment(std::format( - "Cannot assert offsets of fields in {}. It is not a standard-layout class because it has a base class with non-static data members.", - class_info.m_pszName)); + builder.comment(std::format("Cannot assert offsets of fields in {}. It is not a standard-layout class because it has a base class " + "with non-static data members.", + class_info.m_pszName)); } } builder.next_line(); diff --git a/src/tools/field_parser.cpp b/src/tools/field_parser.cpp index 4917621..ec7530f 100644 --- a/src/tools/field_parser.cpp +++ b/src/tools/field_parser.cpp @@ -13,30 +13,12 @@ namespace field_parser { // @note: @es3n1n: a list of possible integral types for bitfields (would be used in `guess_bitfield_type`) // - // clang-format off - constexpr auto kBitfieldIntegralTypes = std::to_array>({ - {8, "uint8_t"}, - {16, "uint16_t"}, - {32, "uint32_t"}, - {64, "uint64_t"}, - - // @todo: @es3n1n: define uint128_t/uint256_t/... as custom structs in the very beginning of the file - {128, "uint128_t"}, - {256, "uint256_t"}, - {512, "uint512_t"}, - }); - // clang-format on - - inline std::string guess_bitfield_type(const std::size_t bits_count) { - for (auto p : kBitfieldIntegralTypes) { - if (bits_count > p.first) - continue; - - return p.second.data(); - } - - throw std::runtime_error(std::format("{} : Unable to guess bitfield type with size {}", __FUNCTION__, bits_count)); - } + constexpr auto kBitfieldIntegralTypes = std::to_array>({ + {8, "uint8_t"}, + {16, "uint16_t"}, + {32, "uint32_t"}, + {64, "uint64_t"}, + }); // clang-format off constexpr auto kTypeNameToCpp = std::to_array>({ @@ -162,6 +144,17 @@ namespace field_parser { } } // namespace detail + std::string guess_bitfield_type(const std::size_t bits_count) { + for (auto p : detail::kBitfieldIntegralTypes) { + if (bits_count > p.first) + continue; + + return p.second.data(); + } + + throw std::runtime_error(std::format("{} : Unable to guess bitfield type with size {}", __FUNCTION__, bits_count)); + } + std::optional type_name_to_cpp(std::string_view type_name) { if (const auto found = std::ranges::find(detail::kTypeNameToCpp, type_name, &decltype(detail::kTypeNameToCpp)::value_type::first); found != detail::kTypeNameToCpp.end()) { From 87e832a2edb12618896408be563e4269afef73a2 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 18 Aug 2024 18:35:59 +0200 Subject: [PATCH 23/46] replace misaligned types and fields with dummies --- include/sdk/interfaces/schemasystem/schema.h | 277 +++++++++++------- include/tools/codegen.h | 16 +- .../source2sdk/source2gen_user_types.hpp | 2 +- src/sdk/sdk.cpp | 103 +++++-- 4 files changed, 265 insertions(+), 133 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 7ae61bb..de223ef 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -2,10 +2,12 @@ // See end of file for extended copyright information. #pragma once +#include #include #include #include #include +#include #include #include #include @@ -363,8 +365,8 @@ class CSchemaType { } // @note: @og: gets size with align - [[nodiscard]] bool GetSizeWithAlignOf(int* nOutSize, std::uint8_t* unOutAlign) { - return reinterpret_cast(vftable[kSchemaType_GetSizeWithAlignOf])(this, nOutSize, unOutAlign); + [[nodiscard]] bool GetSizeWithAlignOf(int* nOutSize, std::uint8_t* unOutAlign) const { + return reinterpret_cast(vftable[kSchemaType_GetSizeWithAlignOf])(this, nOutSize, unOutAlign); } [[nodiscard]] bool CanReinterpretAs(CSchemaType* pType) { @@ -377,11 +379,19 @@ class CSchemaType { } public: - // @note: @og: wrapper around GetSizes, this one gets CSchemaClassInfo->m_nSizeOf + // @note: @og: wrapper around GetSizeWithAlignOf, this one gets CSchemaClassInfo->m_nSizeOf [[nodiscard]] std::optional GetSize() { - std::uint8_t align_of = 0; - int result = 0; - return GetSizeWithAlignOf(&result, &align_of) ? std::make_optional(result) : std::nullopt; + return GetSizeAndAlignment().transform([](auto e) { return std::get<0>(e); }); + } + + /// @return {size, alignment} + [[nodiscard]] std::optional>> GetSizeAndAlignment() const { + std::uint8_t alignment = 0; + int size = 0; + + return GetSizeWithAlignOf(&size, &alignment) ? + std::make_optional(std::make_pair(size, (alignment == 0xff) ? std::nullopt : std::make_optional(static_cast(alignment)))) : + std::nullopt; } // @todo: @og: find out to what class pointer points. @@ -646,112 +656,6 @@ struct SchemaClassInfoData_t { }; static_assert(offsetof(SchemaClassInfoData_t, m_pFn) == 0x68); -class CSchemaClassInfo : public SchemaClassInfoData_t { -public: - [[nodiscard]] std::string_view GetName() const { - if (m_pszName) - return {m_pszName}; - return {}; - } - - [[nodiscard]] std::string_view GetModule() const { - if (m_pszModule) - return {m_pszModule}; - return {}; - } - - [[nodiscard]] std::optional GetBaseClass() const { - if (m_nBaseClassSize && m_pBaseClasses) - return m_pBaseClasses->m_pClass; - return std::nullopt; - } - - // TODO: when we have AddressSanitizer, try returning std::span instead of std::vector. Repeats for other functions. - [[nodiscard]] std::vector GetFields() const { - return {m_pFields, m_pFields + m_nFieldSize}; - } - - [[nodiscard]] std::vector GetStaticFields() const { - return {m_pStaticFields, m_pStaticFields + m_nStaticFieldsSize}; - } - - [[nodiscard]] std::vector GetStaticMetadata() const { - return {m_pStaticMetadata, m_pStaticMetadata + m_nStaticMetadataSize}; - } - - [[nodiscard]] std::string_view GetPrevClassName() const { - if (!m_pBaseClasses || !m_pBaseClasses->m_pClass) - return {}; - return m_pBaseClasses->m_pClass->GetName(); - } - - [[nodiscard]] bool IsA(CSchemaType* pInheritance) const { - if (!m_pSchemaType) - return false; - - return m_pSchemaType->IsA(pInheritance); - } - - [[nodiscard]] bool HasVirtualTable() const { - return (m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0; - } - - [[nodiscard]] bool RecursiveHasVirtualTable() const { - return HasVirtualTable() || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->HasVirtualTable()); - } - - [[nodiscard]] bool IsInherits(const std::string_view from) const { - if (!m_nBaseClassSize || !m_pBaseClasses || !m_pBaseClasses->m_pClass) - return false; - if (m_pBaseClasses->m_pClass->GetName() == from) - return true; - return false; - } - - [[nodiscard]] bool IsRecursiveInherits(const std::string_view from) const { - return IsInherits(from) || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->IsRecursiveInherits(from)); - } - - [[nodiscard]] int GetSize() const { - return m_nSizeOf; - } - - [[nodiscard]] std::uint8_t GetAlignment() const { - return m_unAlignOf == std::numeric_limits::max() ? 8 : m_unAlignOf; - } - - // @note: @og: Copy instance from original to new created with all data from original, returns new_instance - auto CopyInstance(void* instance, void* new_instance) const { - return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstance, instance, new_instance); - } - - // @note: @og: Creates default instance with engine allocated memory (e.g. if SchemaClassInfoData_t is C_BaseEntity, then Instance will be - // C_BaseEntity) - [[nodiscard]] auto CreateInstance() const { - return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstance); - } - - // @note: @og: Creates default instance with your own allocated memory (e.g. if SchemaClassInfoData_t is C_BaseEntity, then Instance will be - // C_BaseEntity) - auto CreateInstance(void* memory) const { - return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstanceWithMemory, memory); - } - - // @note: @og: Destroy instance (e.g.: C_BaseEntity 1st VT fn with 0 flag) - auto DestroyInstance(void* instance) const { - return CallFunction(SchemaClassInfoFunctionIndex::kDestroyInstanceWithMemory, instance); - } - - // @note: @og: Destroy instance with de-allocating memory (e.g.: C_BaseEntity 1st VT fn with 1 flag) - auto DestroyInstanceWithMemory(void* instance) const { - return CallFunction(SchemaClassInfoFunctionIndex::kDestroyInstanceWithMemory, instance); - } - - [[nodiscard]] auto SchemaClassBinding(void* entity) const { - return CallFunction(SchemaClassInfoFunctionIndex::kSchemaDynamicBinding, entity); - } -}; - struct TypeAndCountInfo_t { int m_nElementCount; CSchemaType* m_pElementType; @@ -930,6 +834,155 @@ class CSchemaSystemTypeScope { CUtlTSHash m_EnumBindings = {}; // 0x2E50 }; +class CSchemaClassInfo : public SchemaClassInfoData_t { +public: + [[nodiscard]] std::string_view GetName() const { + if (m_pszName) + return {m_pszName}; + return {}; + } + + [[nodiscard]] std::string_view GetModule() const { + if (m_pszModule) + return {m_pszModule}; + return {}; + } + + [[nodiscard]] std::optional GetBaseClass() const { + if (m_nBaseClassSize && m_pBaseClasses) + return m_pBaseClasses->m_pClass; + return std::nullopt; + } + + // TODO: when we have AddressSanitizer, try returning std::span instead of std::vector. Repeats for other functions. + [[nodiscard]] std::vector GetFields() const { + return {m_pFields, m_pFields + m_nFieldSize}; + } + + [[nodiscard]] std::vector GetStaticFields() const { + return {m_pStaticFields, m_pStaticFields + m_nStaticFieldsSize}; + } + + [[nodiscard]] std::vector GetStaticMetadata() const { + return {m_pStaticMetadata, m_pStaticMetadata + m_nStaticMetadataSize}; + } + + [[nodiscard]] std::string_view GetPrevClassName() const { + if (!m_pBaseClasses || !m_pBaseClasses->m_pClass) + return {}; + return m_pBaseClasses->m_pClass->GetName(); + } + + [[nodiscard]] bool IsA(CSchemaType* pInheritance) const { + if (!m_pSchemaType) + return false; + + return m_pSchemaType->IsA(pInheritance); + } + + [[nodiscard]] bool HasVirtualTable() const { + return (m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0; + } + + [[nodiscard]] bool RecursiveHasVirtualTable() const { + return HasVirtualTable() || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->HasVirtualTable()); + } + + [[nodiscard]] bool IsInherits(const std::string_view from) const { + if (!m_nBaseClassSize || !m_pBaseClasses || !m_pBaseClasses->m_pClass) + return false; + if (m_pBaseClasses->m_pClass->GetName() == from) + return true; + return false; + } + + [[nodiscard]] bool IsRecursiveInherits(const std::string_view from) const { + return IsInherits(from) || (m_pBaseClasses && m_pBaseClasses->m_pClass && m_pBaseClasses->m_pClass->IsRecursiveInherits(from)); + } + + [[nodiscard]] int GetSize() const { + return m_nSizeOf; + } + + /// @return Alignment as registered in the game + [[nodiscard]] std::optional GetRegisteredAlignment() const { + return m_unAlignOf == std::numeric_limits::max() ? std::nullopt : std::make_optional(static_cast(m_unAlignOf)); + } + + /// This function recurses through all members. That's expensive. Cache the result if possible. + /// @return @ref GetRegisteredAlignment() if set. Otherwise tries to determine the alignment. + /// Returns @ref std::nullopt if one or more fields have unknown alignment. + [[nodiscard]] std::optional GetFullAlignment() const { + return GetRegisteredAlignment().or_else([this]() { + int base_alignment = 0; + + if (this->m_pBaseClasses != nullptr) { + if (const auto maybe_base_alignment = this->m_pBaseClasses->m_pClass->GetFullAlignment()) { + base_alignment = maybe_base_alignment.value(); + } else { + // we have a base class, but it has unknown alignment + return std::optional{}; + } + } + + auto field_alignments = + this->GetFields() | std::ranges::views::transform([](const SchemaClassFieldData_t& e) { + // TOOD: lookups by name are slow. I've seen some better way somewhere. + if (const auto* class_ = e.m_pSchemaType->m_pTypeScope->FindDeclaredClass(e.m_pSchemaType->m_pszName); class_ != nullptr) { + return class_->GetFullAlignment(); + } else { + return e.m_pSchemaType->GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); + } + }); + + if (field_alignments.empty()) { + // This is an empty class. The generator will add a single pad with alignment 1. + return std::make_optional((base_alignment == 0) ? 1 : base_alignment); + } else if (std::ranges::all_of(field_alignments, &std::optional::has_value)) { + int max_alignment = base_alignment; + for (const auto& e : field_alignments) { + max_alignment = std::max(max_alignment, e.value()); + } + return std::make_optional(max_alignment); + } else { + // there are fields with unknown alignment + return std::optional{}; + } + }); + } + + // @note: @og: Copy instance from original to new created with all data from original, returns new_instance + auto CopyInstance(void* instance, void* new_instance) const { + return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstance, instance, new_instance); + } + + // @note: @og: Creates default instance with engine allocated memory (e.g. if SchemaClassInfoData_t is C_BaseEntity, then Instance will be + // C_BaseEntity) + [[nodiscard]] auto CreateInstance() const { + return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstance); + } + + // @note: @og: Creates default instance with your own allocated memory (e.g. if SchemaClassInfoData_t is C_BaseEntity, then Instance will be + // C_BaseEntity) + auto CreateInstance(void* memory) const { + return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstanceWithMemory, memory); + } + + // @note: @og: Destroy instance (e.g.: C_BaseEntity 1st VT fn with 0 flag) + auto DestroyInstance(void* instance) const { + return CallFunction(SchemaClassInfoFunctionIndex::kDestroyInstanceWithMemory, instance); + } + + // @note: @og: Destroy instance with de-allocating memory (e.g.: C_BaseEntity 1st VT fn with 1 flag) + auto DestroyInstanceWithMemory(void* instance) const { + return CallFunction(SchemaClassInfoFunctionIndex::kDestroyInstanceWithMemory, instance); + } + + [[nodiscard]] auto SchemaClassBinding(void* entity) const { + return CallFunction(SchemaClassInfoFunctionIndex::kSchemaDynamicBinding, entity); + } +}; + enum SchemaTypeScope_t : std::uint8_t { SCHEMA_TYPESCOPE_GLOBAL = 0, SCHEMA_TYPESCOPE_LOCAL, diff --git a/include/tools/codegen.h b/include/tools/codegen.h index 11c5aa6..2760d97 100644 --- a/include/tools/codegen.h +++ b/include/tools/codegen.h @@ -140,13 +140,13 @@ namespace codegen { return push_line(std::vformat(sizeof(T) >= 2 ? "{} = {:#x}," : "{} = {},", std::make_format_args(name, value))); } - self_ref begin_struct(const std::string& name, const std::string& access_modifier = "public") { + self_ref begin_struct(const std::string_view name, const std::string& access_modifier = "public") { return begin_block(std::format("struct {}", escape_name(name)), access_modifier); } - self_ref begin_struct_with_base_type(const std::string& name, const std::string& base_type, const std::string& access_modifier = "public") { + self_ref begin_struct_with_base_type(const std::string_view name, const std::string& base_type, const std::string& access_modifier = "public") { if (base_type.empty()) - return begin_struct(std::cref(name), access_modifier); + return begin_struct(name, access_modifier); return begin_block(std::format("struct {} : public {}", escape_name(name), base_type), access_modifier); } @@ -203,6 +203,16 @@ namespace codegen { return push_line(std::format("static_assert(offsetof({}, {}) == {:#x});", escape_name(class_name), prop_name, expected_offset)); } + /// Not to be used for inline comments. Doing so could break support for other laguages. + self_ref begin_multi_line_comment(const bool move_cursor_to_next_line = true) { + return push_line("/*", move_cursor_to_next_line); + } + + /// Not to be used for inline comments. Doing so could break support for other laguages. + self_ref end_multi_line_comment(const bool move_cursor_to_next_line = true) { + return push_line("*/", move_cursor_to_next_line); + } + self_ref comment(const std::string& text, const bool move_cursor_to_next_line = true) { return push_line(std::format("// {}", text), move_cursor_to_next_line); } diff --git a/sdk-static/source2sdk/source2gen_user_types.hpp b/sdk-static/source2sdk/source2gen_user_types.hpp index e67c421..fc7ee65 100644 --- a/sdk-static/source2sdk/source2gen_user_types.hpp +++ b/sdk-static/source2sdk/source2gen_user_types.hpp @@ -99,7 +99,7 @@ using KeyValues = char[0x01]; using QAngle = char[0x0c]; using QuaternionStorage = char[0x10]; using Quaternion = char[0x10]; -using RadianEuler = char[0x14]; +using RadianEuler = char[0x0c]; using RenderInputLayoutField_t = char[0x04]; // we don't have a field size for this type. uses the fallback of 1. using RenderPrimitiveType_t = char[0x01]; diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 9cff422..54daf4f 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -36,6 +36,11 @@ namespace { using namespace std::string_view_literals; + // TOOD: move to a better place + std::string to_hex_string(int i) { + return std::format("{:#x}", i); + } + /** * Project structure is * @@ -130,6 +135,10 @@ namespace { FNV32("MNetworkMaxValue"), }; + void warn(std::string_view message) { + std::cerr << "warning: " << message << '\n'; + } + // @note: @es3n1n: some more utils // std::string GetMetadataValue(const SchemaMetadataEntryData_t& metadata_entry) { @@ -176,7 +185,10 @@ namespace { }; void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_info) { - builder.comment(std::format("Alignment: {}", class_info.GetAlignment())).comment(std::format("Size: {:#x}", class_info.m_nSizeOf)); + builder.comment(std::format("Registered alignment: {}", class_info.GetRegisteredAlignment().transform(&to_hex_string).value_or("unknown"))); + // TOOD: remove? this is expensive + builder.comment(std::format("Alignment: {}", class_info.GetFullAlignment().transform(&to_hex_string).value_or("unknown"))); + builder.comment(std::format("Size: {:#x}", class_info.m_nSizeOf)); if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable builder.comment("Has VTable"); @@ -580,6 +592,16 @@ namespace { return result; } + // TOOD: needs documentation. duplicate of CSchemaClass::GetFullAlignment() + [[nodiscard]] + std::optional GetFullAlignmentOfType(const CSchemaType& type) { + if (const auto* class_ = type.m_pTypeScope->FindDeclaredClass(type.m_pszName); class_ != nullptr) { + return class_->GetFullAlignment(); + } else { + return type.GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); + } + } + // TOOD: Move up struct BitfieldEntry { @@ -634,6 +656,9 @@ namespace { } void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + // TODO: when we have a CLI parser: pass this property in from the outside + const bool verbose = true; + struct cached_datamap_t { std::string type_; std::string name_; @@ -643,9 +668,28 @@ namespace { // @note: @es3n1n: get class info, assemble it // const auto* class_parent = class_.m_pBaseClasses ? class_.m_pBaseClasses->m_pClass : nullptr; + // TOOD: remove const auto& class_info = class_; + // TOOD: use these 2 variables + const auto class_size = class_.GetSize(); + const auto class_alignment = class_.GetFullAlignment(); + // TOOD: use better default alignment x3 + const auto aligned_size = class_size + (class_alignment.value_or(8) - (class_size % class_alignment.value_or(8))) % class_alignment.value_or(8); + // Source2 has alignof(max_align_t)=8, i.e. every class whose size is a multiple of 8 is aligned. + const auto class_is_aligned = (class_size % class_alignment.value_or(8)) == 0; const auto is_struct = std::string_view{class_info.m_pszName}.ends_with("_t"); + if (!class_is_aligned) { + const auto warning = std::format("Type {} is misaligned. Its size should be {:#x}, but with proper alignement it has size {:#x}.", + class_.GetName(), class_size, aligned_size); + warn(warning); + builder.comment(warning); + builder.comment("It has been replaced by a dummy. You can try uncommenting the struct below."); + builder.begin_struct(class_.GetName()); + builder.struct_padding(0, class_size); + builder.end_struct(); + } + PrintClassInfo(builder, class_info); // @note: @es3n1n: get parent name @@ -653,6 +697,10 @@ namespace { const std::string parent_class_name = (class_parent != nullptr) ? MaybeWithModuleName(*class_parent->m_pTypeScope, class_parent->m_pszName) : ""; const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; + if (!class_is_aligned) { + builder.begin_multi_line_comment(); + } + // @note: @es3n1n: start class // if (is_struct) @@ -699,10 +747,10 @@ namespace { continue; } - // @note: @es3n1n: obtaining size - // fall back to 1 because there are no 0-sized types - // - const int field_size = field.m_pSchemaType->GetSize().value_or(1); + // fall back to size=1 because there are no 0-sized types + // TOOD: why do we get no size for some types? + const auto field_size = field.m_pSchemaType->GetSize().value_or(1); + const auto field_alignment = GetFullAlignmentOfType(*field.m_pSchemaType); // @note: @es3n1n: parsing type // @@ -754,22 +802,37 @@ namespace { // @note: @es3n1n: update state // - if (field_size != 0) { - state.last_field_offset = field.m_nSingleInheritanceOffset; - state.last_field_size = static_cast(field_size); - } + state.last_field_offset = field.m_nSingleInheritanceOffset; + state.last_field_size = static_cast(field_size); // @note: @es3n1n: push prop // - if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { - // HACKHACK: this is a hack to get the size of template types right - builder.prop("char", std::format("{}[{:#x}]", var_info.formatted_name(), field.m_pSchemaType->GetSize().value()), false); - builder.comment(field.m_pSchemaType->m_pszName, false); + // TOOD: use better default alignment + if ((field.m_nSingleInheritanceOffset % field_alignment.value_or(8)) == 0) { + if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { + // HACKHACK: this is a hack to get the size of template types right + builder.prop("char", std::format("{}[{:#x}]", var_info.m_name, field_size), false); + builder.comment(field.m_pSchemaType->m_pszName, false); + } else { + builder.prop(var_info.m_type, var_info.formatted_name(), false); + } } else { - builder.prop(var_info.m_type, var_info.formatted_name(), false); + // TOOD: use warn() for all errors + warn(std::format("property {}::{} is misaligned. it has been replaced with a byte array.", class_.GetName(), field.m_pszName)); + builder.comment(std::format("property is misaligned, you can try marking {} as packed and uncommenting this property", class_.GetName())); + builder.prop("char", std::format("{}[{:#x}]", var_info.m_name, field_size), true); + builder.comment("", false).reset_tabs_count().prop(var_info.m_type, var_info.formatted_name(), false).restore_tabs_count(); } - builder.reset_tabs_count().comment(std::format("{:#x} ({:#x})", field.m_nSingleInheritanceOffset, field_size), false).restore_tabs_count(); + if (verbose) { + builder.reset_tabs_count() + .comment(std::format("offset={:#x} size={:#x} alignment={}", field.m_nSingleInheritanceOffset, field_size, + field_alignment.transform(to_hex_string).value_or("unknown")), + false) + .restore_tabs_count(); + } else { + builder.reset_tabs_count().comment(std::format("{:#x}", field.m_nSingleInheritanceOffset), false).restore_tabs_count(); + } cached_fields.emplace_back(var_info.formatted_name(), field.m_nSingleInheritanceOffset); builder.next_line(); @@ -783,7 +846,8 @@ namespace { // pad the class end. const auto last_field_end = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); - const auto end_pad = class_.GetSize() - last_field_end; + // TOOD: use aligned size? + const auto end_pad = class_size - last_field_end; if (end_pad != 0) { builder.struct_padding(last_field_end, end_pad, true, true); @@ -874,8 +938,13 @@ namespace { class_info.m_pszName)); } } + + if (!class_is_aligned) { + builder.end_multi_line_comment(); + } + builder.next_line(); - builder.static_assert_size(class_info.m_pszName, class_info.GetSize()); + builder.static_assert_size(class_info.m_pszName, class_size); } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { From d0f37ab8eb9bfd600701a0e70524435145888d36 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Thu, 22 Aug 2024 17:31:49 +0200 Subject: [PATCH 24/46] remove development notes --- include/sdk/interfaces/schemasystem/schema.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index de223ef..73b4d0b 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -138,8 +138,8 @@ enum { kSchemaType_GetSizeWithAlignOf = 3, kSchemaSystem_ValidateClasses = 35, kSchemaSystem_GetClassInfoBinaryName = 22, - kSchemaSystem_GetClassProjectName = kSchemaSystem_GetClassInfoBinaryName + 1, - kSchemaSystem_GetEnumBinaryName = kSchemaSystem_GetClassProjectName + 1, + kSchemaSystem_GetClassModuleName = kSchemaSystem_GetClassInfoBinaryName + 1, + kSchemaSystem_GetEnumBinaryName = kSchemaSystem_GetClassModuleName + 1, kSchemaSystem_GetEnumProjectName = kSchemaSystem_GetEnumBinaryName + 1, kSchemaSystemTypeScope_DeclaredClass = 14, kSchemaSystemTypeScope_DeclaredEnum = kSchemaSystemTypeScope_DeclaredClass + 1, @@ -1015,9 +1015,9 @@ class CSchemaSystem { } } - [[nodiscard]] std::string ScopedNameForClass(CSchemaClassBinding* pBinding) { + [[nodiscard]] std::string ScopedNameForClass(const CSchemaClassBinding* pBinding) { static CBufferStringGrowable<1024> szBuf; - Virtual::Get(this, 17)(this, pBinding, &szBuf); + Virtual::Get(this, 17)(this, pBinding, &szBuf); return szBuf.Get(); } @@ -1033,7 +1033,7 @@ class CSchemaSystem { } } - [[nodiscard]] std::string GetScopedNameForEnum(CSchemaEnumBinding* pBinding) { + [[nodiscard]] std::string ScopedNameForEnum(CSchemaEnumBinding* pBinding) { static CBufferStringGrowable<1024> szBuf; Virtual::Get(this, 19)(this, pBinding, &szBuf); return szBuf.Get(); @@ -1043,16 +1043,17 @@ class CSchemaSystem { return Virtual::Get(this, kSchemaSystem_GetClassInfoBinaryName)(this, pBinding); } - [[nodiscard]] const char* GetClassProjectName(CSchemaClassBinding* pBinding) { - return Virtual::Get(this, kSchemaSystem_GetClassProjectName)(this, pBinding); + [[nodiscard]] const char* GetClassModuleName(CSchemaClassBinding* pBinding) { + // Returns pBinding->m_pszModule + return Virtual::Get(this, kSchemaSystem_GetClassModuleName)(this, pBinding); } [[nodiscard]] const char* GetEnumBinaryName(CSchemaEnumBinding* pBinding) { return Virtual::Get(this, kSchemaSystem_GetEnumBinaryName)(this, pBinding); } - [[nodiscard]] const char* GetEnumProjectName(CSchemaEnumBinding* pBinding) { - return Virtual::Get(this, kSchemaSystem_GetEnumProjectName)(this, pBinding); + [[nodiscard]] const char* GetEnumProjectName(const CSchemaEnumBinding* pBinding) { + return Virtual::Get(this, kSchemaSystem_GetEnumProjectName)(this, pBinding); } CSchemaClassBinding* ValidateClasses(CSchemaClassBinding** ppBinding) { From b0321dc16e1c0987fa4070f948bf3f925fa4c947 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 19:43:09 +0200 Subject: [PATCH 25/46] better code naming, use variable for cmake project name --- CMakeLists.txt | 20 +++--- src/sdk/sdk.cpp | 165 +++++++++++++++++++++++++++++++----------------- 2 files changed, 117 insertions(+), 68 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e9c84..6b94661 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,34 +61,34 @@ if(NOT SOURCE2GEN_GAME IN_LIST SOURCE2GEN_SUPPORTED_GAMES) message(FATAL_ERROR "source2gen: Invalid value for SOURCE2GEN_GAME: ${SOURCE2GEN_GAME}") endif() -add_executable(source2gen) +add_executable(${PROJECT_NAME}) file(GLOB_RECURSE source2gen_SOURCES "src/**.cpp") file(GLOB_RECURSE source2gen_HEADERS "include/**.h" "include/**.hpp") -target_sources(source2gen PRIVATE ${source2gen_SOURCES} ${source2gen_HEADERS}) +target_sources(${PROJECT_NAME} PRIVATE ${source2gen_SOURCES} ${source2gen_HEADERS}) -target_include_directories(source2gen PUBLIC +target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src ) if(MSVC) - target_compile_options(source2gen PRIVATE /bigobj) + target_compile_options(${PROJECT_NAME} PRIVATE /bigobj) endif() if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") - target_compile_definitions(source2gen PRIVATE "source2gen_DEBUG" "_DEBUG") + target_compile_definitions(${PROJECT_NAME} PRIVATE "source2gen_DEBUG" "_DEBUG") endif() if(${CMAKE_BUILD_TYPE} STREQUAL "Release") - target_compile_definitions(source2gen PRIVATE "source2gen_RELEASE" "NDEBUG") + target_compile_definitions(${PROJECT_NAME} PRIVATE "source2gen_RELEASE" "NDEBUG") endif() if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "AppleClang" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") - target_compile_options(source2gen PRIVATE + target_compile_options(${PROJECT_NAME} PRIVATE -pedantic-errors -Wno-address-of-packed-member $<$>:-Wno-implicit-exception-spec-mismatch> @@ -96,7 +96,7 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" OR $,-Wno-attributes,-Wno-unknown-attributes> ) - target_compile_definitions(source2gen PRIVATE + target_compile_definitions(${PROJECT_NAME} PRIVATE "FORCEINLINE=" "__forceinline=" "__thiscall=" @@ -104,11 +104,11 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" OR ) endif() -target_compile_definitions(source2gen PRIVATE +target_compile_definitions(${PROJECT_NAME} PRIVATE "${SOURCE2GEN_GAME}" "_CRT_SECURE_NO_WARNINGS" "NOMINMAX" "WIN32_LEAN_AND_MEAN" "_WIN32_WINNT=0x601" "__cpp_lib_expected" # Because clangd-18 + GLIBCXX_3.4.33 doesn't know std::expected. This is a hack to make the language server work. -) \ No newline at end of file +) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 54daf4f..0992104 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -184,46 +184,46 @@ namespace { return value; }; - void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_info) { - builder.comment(std::format("Registered alignment: {}", class_info.GetRegisteredAlignment().transform(&to_hex_string).value_or("unknown"))); + void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + builder.comment(std::format("Registered alignment: {}", class_.GetRegisteredAlignment().transform(&to_hex_string).value_or("unknown"))); // TOOD: remove? this is expensive - builder.comment(std::format("Alignment: {}", class_info.GetFullAlignment().transform(&to_hex_string).value_or("unknown"))); - builder.comment(std::format("Size: {:#x}", class_info.m_nSizeOf)); + builder.comment(std::format("Alignment: {}", class_.GetFullAlignment().transform(&to_hex_string).value_or("unknown"))); + builder.comment(std::format("Size: {:#x}", class_.m_nSizeOf)); - if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable + if ((class_.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable builder.comment("Has VTable"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_IS_ABSTRACT) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_IS_ABSTRACT) != 0) builder.comment("Is Abstract"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_CONSTRUCTOR) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_CONSTRUCTOR) != 0) builder.comment("Has Trivial Constructor"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_DESTRUCTOR) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_HAS_TRIVIAL_DESTRUCTOR) != 0) builder.comment("Has Trivial Destructor"); #if defined(CS2) || defined(DOTA2) - if ((class_info.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_ALLOWED) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_ALLOWED) != 0) builder.comment("Construct allowed"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_DISALLOWED) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_CONSTRUCT_DISALLOWED) != 0) builder.comment("Construct disallowed"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MConstructibleClassBase) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MConstructibleClassBase) != 0) builder.comment("MConstructibleClassBase"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasCustomAlignedNewDelete) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasCustomAlignedNewDelete) != 0) builder.comment("MClassHasCustomAlignedNewDelete"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasEntityLimitedDataDesc) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MClassHasEntityLimitedDataDesc) != 0) builder.comment("MClassHasEntityLimitedDataDesc"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MDisableDataDescValidation) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MDisableDataDescValidation) != 0) builder.comment("MDisableDataDescValidation"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MIgnoreTypeScopeMetaChecks) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MIgnoreTypeScopeMetaChecks) != 0) builder.comment("MIgnoreTypeScopeMetaChecks"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkNoBase) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkNoBase) != 0) builder.comment("MNetworkNoBase"); - if ((class_info.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkAssumeNotNetworkable) != 0) + if ((class_.m_nClassFlags & SCHEMA_CF1_INFO_TAG_MNetworkAssumeNotNetworkable) != 0) builder.comment("MNetworkAssumeNotNetworkable"); #endif - if (class_info.m_nStaticMetadataSize > 0) + if (class_.m_nStaticMetadataSize > 0) builder.comment(""); - for (const auto& metadata : class_info.GetStaticMetadata()) { + for (const auto& metadata : class_.GetStaticMetadata()) { if (const auto value = GetMetadataValue(metadata); !value.empty()) builder.comment(std::format("static metadata: {} \"{}\"", metadata.m_szName, value)); else @@ -668,20 +668,25 @@ namespace { // @note: @es3n1n: get class info, assemble it // const auto* class_parent = class_.m_pBaseClasses ? class_.m_pBaseClasses->m_pClass : nullptr; - // TOOD: remove - const auto& class_info = class_; - // TOOD: use these 2 variables const auto class_size = class_.GetSize(); const auto class_alignment = class_.GetFullAlignment(); - // TOOD: use better default alignment x3 - const auto aligned_size = class_size + (class_alignment.value_or(8) - (class_size % class_alignment.value_or(8))) % class_alignment.value_or(8); // Source2 has alignof(max_align_t)=8, i.e. every class whose size is a multiple of 8 is aligned. const auto class_is_aligned = (class_size % class_alignment.value_or(8)) == 0; - const auto is_struct = std::string_view{class_info.m_pszName}.ends_with("_t"); + const auto is_struct = std::string_view{class_.m_pszName}.ends_with("_t"); if (!class_is_aligned) { - const auto warning = std::format("Type {} is misaligned. Its size should be {:#x}, but with proper alignement it has size {:#x}.", - class_.GetName(), class_size, aligned_size); + const auto warning = [&]() { + if (class_alignment.has_value()) { + // ceil size to next possible aligned size + const auto aligned_size = class_size + (class_alignment.value() - (class_size % class_alignment.value())) % class_alignment.value(); + + return std::format("Type {} is misaligned. Its size should be {:#x}, but with proper alignement it has size {:#x}.", class_.GetName(), + class_size, aligned_size); + } else { + return std::format("Type {} appears to be misaligned. Its alignment is unknown and it is not aligned to max_align_t (8).", + class_.GetName()); + } + }(); warn(warning); builder.comment(warning); builder.comment("It has been replaced by a dummy. You can try uncommenting the struct below."); @@ -690,7 +695,7 @@ namespace { builder.end_struct(); } - PrintClassInfo(builder, class_info); + PrintClassInfo(builder, class_); // @note: @es3n1n: get parent name // @@ -704,15 +709,18 @@ namespace { // @note: @es3n1n: start class // if (is_struct) - builder.begin_struct_with_base_type(class_info.m_pszName, parent_class_name, ""); + builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name, ""); else - builder.begin_class_with_base_type(class_info.m_pszName, parent_class_name, ""); + builder.begin_class_with_base_type(class_.m_pszName, parent_class_name, ""); // @note: @es3n1n: field assembling state // // TOOD: inconsistent optional vs 0 ClassAssemblyState state = {.last_field_size = parent_class_size}; + /// If fields cannot be emitted, e.g. because of collisions, they're added to + /// this set so we can ignore them when asserting offsets. + std::unordered_set skipped_fields{}; std::list> cached_fields{}; std::list cached_datamap_fields{}; @@ -739,16 +747,21 @@ namespace { // builder.access_modifier("public"); - for (const auto& field : class_info.GetFields()) { + for (const auto& field : class_.GetFields()) { // @fixme: @es3n1n: todo proper collision fix and remove this block if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { + skipped_fields.emplace(field.m_pszName); builder.comment( std::format("Skipped field \"{}\" @ {:#x} because of the struct collision", field.m_pszName, field.m_nSingleInheritanceOffset)); continue; } - // fall back to size=1 because there are no 0-sized types - // TOOD: why do we get no size for some types? + if (!field.m_pSchemaType->GetSize().has_value()) { + std::cout << "MISSISG SIZE OF " << field.m_pSchemaType->m_pszName << std::endl; + } + + // Fall back to size=1 because there are no 0-sized types. + // `RenderPrimitiveType_t` is the only type (in CS2 9035763) without size information. const auto field_size = field.m_pSchemaType->GetSize().value_or(1); const auto field_alignment = GetFullAlignmentOfType(*field.m_pSchemaType); @@ -757,8 +770,6 @@ namespace { const auto [type_name, array_sizes] = GetType(*field.m_pSchemaType); const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); - const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); - // Collect all bitfield entries and emit them later. We need to know // how large the bitfield is in order to choose the right type. We // only know how large the bitfield is once we've reached its end. @@ -772,6 +783,13 @@ namespace { continue; } + if (verbose) { + builder.comment(std::format("last_field_offset={} last_field_size={}", state.last_field_offset.transform(&to_hex_string).value_or("none"), + state.last_field_size.transform(&to_hex_string).value_or("none"))); + } + + const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + // @note: @es3n1n: insert padding if needed // if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && @@ -784,9 +802,26 @@ namespace { .access_modifier("public"); } - // This is the end of a bitfield + // This is the first field after a bitfield, i.e. the active bitfield has ended if (!var_info.is_bitfield() && state.assembling_bitfield) { state = AssembleBitfield(builder, std::move(state), expected_offset); + + // TOOD: this is exact duplicate code + { + const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + + // @note: @es3n1n: insert padding if needed + // + if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && + !state.assembling_bitfield) { + builder.access_modifier("public") + .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) + .reset_tabs_count() + .comment(std::format("{:#x}", expected_offset)) + .restore_tabs_count() + .access_modifier("public"); + } + } } // @note: @es3n1n: dump metadata @@ -810,7 +845,17 @@ namespace { // TOOD: use better default alignment if ((field.m_nSingleInheritanceOffset % field_alignment.value_or(8)) == 0) { if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { - // HACKHACK: this is a hack to get the size of template types right + // This is a workaround to get the size of template types right. + // There are types that have non-type template parameters, e.g. + // `CUtlLeanVectorFixedGrowable`. The non-type template parameter affects the size of the template type, but the schema system + // doesn't store information about non-type template parameters. The schema system just says `CUtlLeanVectorFixedGrowable`, which + // is insufficient to generate a `CUtlLeanVectorFixedGrowable` with correct size.` + // To still keep the rest of the class in order, we replace all template fields with char arrays. + // We're applying this workaround to all template type, even those that don't have non-type template parameters, because we can't tell + // them apart. So we're certainly commenting out more than is necessary. + builder.comment( + std::format("{} has a template type with potentially unknown template parameters. You can try uncommenting the field below.", + var_info.m_name)); builder.prop("char", std::format("{}[{:#x}]", var_info.m_name, field_size), false); builder.comment(field.m_pSchemaType->m_pszName, false); } else { @@ -853,31 +898,31 @@ namespace { builder.struct_padding(last_field_end, end_pad, true, true); } else if (end_pad < 0) [[unlikely]] { throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), - -end_pad, last_field_end, class_.GetName(), class_.GetSize())}; + -end_pad, last_field_end, class_.GetName(), class_size)}; } // @note: @es3n1n: dump static fields // - if (class_info.m_nStaticFieldsSize) { - if (class_info.m_nFieldSize) + if (class_.m_nStaticFieldsSize) { + if (class_.m_nFieldSize) builder.next_line(); builder.comment("Static fields:"); } // The current class may be defined in multiple scopes. It doesn't matter which one we use, as all definitions are the same.. // TODO: verify the above statement. Are static fields really shared between scopes? - const std::string scope_name{class_info.m_pTypeScope->BGetScopeName()}; + const std::string scope_name{class_.m_pTypeScope->BGetScopeName()}; - for (auto s = 0; s < class_info.m_nStaticFieldsSize; s++) { - auto static_field = &class_info.m_pStaticFields[s]; + for (auto s = 0; s < class_.m_nStaticFieldsSize; s++) { + auto static_field = &class_.m_pStaticFields[s]; auto [type, mod] = GetType(*static_field->m_pSchemaType); const auto var_info = field_parser::parse(type, static_field->m_pszName, mod); - builder.static_field_getter(var_info.m_type, var_info.m_name, scope_name, class_info.m_pszName, s); + builder.static_field_getter(var_info.m_type, var_info.m_name, scope_name, class_.m_pszName, s); } - if (class_info.m_pFieldMetadataOverrides && class_info.m_pFieldMetadataOverrides->m_iTypeDescriptionCount > 1) { - const auto& dm = class_info.m_pFieldMetadataOverrides; + if (class_.m_pFieldMetadataOverrides && class_.m_pFieldMetadataOverrides->m_iTypeDescriptionCount > 1) { + const auto& dm = class_.m_pFieldMetadataOverrides; for (std::uint64_t s = 0; s < dm->m_iTypeDescriptionCount; s++) { auto* t = &dm->m_pTypeDescription[s]; @@ -907,7 +952,7 @@ namespace { } if (!cached_datamap_fields.empty()) { - if (class_info.m_nFieldSize) + if (class_.m_nFieldSize) builder.next_line(); builder.comment("Datamap fields:"); @@ -917,25 +962,29 @@ namespace { } } - if (!class_info.m_nFieldSize && !class_info.m_nStaticMetadataSize) + if (!class_.m_nFieldSize && !class_.m_nStaticMetadataSize) builder.comment("No schema binary for binding"); builder.end_block(); - // TODO: when we have a CLI parse: make static_assert() generation optional because users might not care about errors in file they don't use - if (class_info.m_nBaseClassSize == 0) { - for (const auto& field : class_info.GetFields()) { + // TODO: this check is incomplete. we should also check if any of the class's fields is a non-standard-layout class. That's the case for e.g. + // soundsystem::CSosSoundEventGroupSchema + const bool is_standard_layout_class = (class_.m_nBaseClassSize == 0); + + // TODO: when we have a CLI parser: allow users to generate assertions in non-standard-layout classes. Those assertions are + // conditionally-supported. + if (is_standard_layout_class) { + for (const auto& field : + class_.GetFields() | std::ranges::views::filter([&](const auto& e) { return !skipped_fields.contains(e.m_pszName); })) { if (field.m_pSchemaType->m_unTypeCategory == ETypeCategory::Schema_Bitfield) { - builder.comment(std::format("Cannot assert offset of bitfield {}::{}", class_info.m_pszName, field.m_pszName)); + builder.comment(std::format("Cannot assert offset of bitfield {}::{}", class_.m_pszName, field.m_pszName)); } else { - builder.static_assert_offset(class_info.m_pszName, field.m_pszName, field.m_nSingleInheritanceOffset); + builder.static_assert_offset(class_.m_pszName, field.m_pszName, field.m_nSingleInheritanceOffset); } } } else { - if (class_info.m_nFieldSize != 0) { - builder.comment(std::format("Cannot assert offsets of fields in {}. It is not a standard-layout class because it has a base class " - "with non-static data members.", - class_info.m_pszName)); + if (class_.m_nFieldSize != 0) { + builder.comment(std::format("Cannot assert offsets of fields in {} because it is not a standard-layout class", class_.m_pszName)); } } @@ -944,7 +993,7 @@ namespace { } builder.next_line(); - builder.static_assert_size(class_info.m_pszName, class_size); + builder.static_assert_size(class_.m_pszName, class_size); } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { From 63bd5974cc6a724ce315fbf5c96438dd3efc8470 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 21:16:48 +0200 Subject: [PATCH 26/46] consistent optionality --- README.md | 19 ++++++++++++++++- src/sdk/sdk.cpp | 54 ++++++++++++++++++++++++------------------------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index ebb3113..b03973b 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,24 @@ to be provided by the user and expose all types listed in intend to access any of these types, you can use the dummy file [source2gen_user_types.hpp](sdk-static/source2gen_user_types.hpp). -## Getting Started +## Limitations + +### Disabled entities + +Under the following conditions, entities are either entirely omitted, or emitted +as a comment and replaced with a dummy: + +- Overlapping fields: Fields that share memory with another field +- Misaligned fields: Fields that cannot be placed at the correct in-class offset + because of their type's alignment requirements +- Misaligned types: Class types that would exceed their correct size because + padding bytes would have to be inserted to meet alignment requirements +- Fields with template types + +Some of these disabled entities can be made to work by using compiler-specific +attributes. + +## Getting Started with Development These instructions will help you set up the project on your local machine for development and testing purposes. diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 0992104..19a7464 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -34,6 +34,22 @@ namespace { auto operator<=>(const NameLookup&) const = default; }; + struct BitfieldEntry { + std::string name{}; + std::size_t size{}; + // TOOD: document lifetime - asked es3n1n + std::vector metadata{}; + }; + + struct ClassAssemblyState { + std::optional last_field_size = std::nullopt; + std::optional last_field_offset = std::nullopt; + bool assembling_bitfield = false; + std::vector bitfield = {}; + + std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var + }; + using namespace std::string_view_literals; // TOOD: move to a better place @@ -186,7 +202,6 @@ namespace { void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { builder.comment(std::format("Registered alignment: {}", class_.GetRegisteredAlignment().transform(&to_hex_string).value_or("unknown"))); - // TOOD: remove? this is expensive builder.comment(std::format("Alignment: {}", class_.GetFullAlignment().transform(&to_hex_string).value_or("unknown"))); builder.comment(std::format("Size: {:#x}", class_.m_nSizeOf)); @@ -592,7 +607,9 @@ namespace { return result; } - // TOOD: needs documentation. duplicate of CSchemaClass::GetFullAlignment() + // TOOD: Create a "ShadowClass" map to store full alignment and standard-layout-class status? + + /// @return For class types, returns @ref CSchemaClassInfo::GetFullAlignment(). Otherwise returns the immediately available size. [[nodiscard]] std::optional GetFullAlignmentOfType(const CSchemaType& type) { if (const auto* class_ = type.m_pTypeScope->FindDeclaredClass(type.m_pszName); class_ != nullptr) { @@ -602,25 +619,6 @@ namespace { } } - // TOOD: Move up - - struct BitfieldEntry { - std::string name{}; - std::size_t size{}; - // TOOD: document lifetime - std::vector metadata{}; - }; - - struct ClassAssemblyState { - std::optional last_field_size = std::nullopt; - std::optional last_field_offset = std::nullopt; - bool assembling_bitfield = false; - std::vector bitfield = {}; - - std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var - // TOOD: inconsistent optional vs 0 - }; - [[nodiscard]] ClassAssemblyState AssembleBitfield(codegen::generator_t& builder, ClassAssemblyState&& state, int expected_offset) { state.assembling_bitfield = false; @@ -700,7 +698,7 @@ namespace { // @note: @es3n1n: get parent name // const std::string parent_class_name = (class_parent != nullptr) ? MaybeWithModuleName(*class_parent->m_pTypeScope, class_parent->m_pszName) : ""; - const std::ptrdiff_t parent_class_size = class_parent ? class_parent->m_nSizeOf : 0; + const std::optional parent_class_size = class_parent ? std::make_optional(class_parent->m_nSizeOf) : std::nullopt; if (!class_is_aligned) { builder.begin_multi_line_comment(); @@ -731,9 +729,10 @@ namespace { if (const auto* first_field = (class_.m_pFields == nullptr) ? nullptr : &class_.m_pFields[0]; first_field) first_field_offset = first_field->m_nSingleInheritanceOffset; - const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size; + const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size.value_or(0); - const auto expected_pad_size = first_field_offset.transform([&](auto e) { return e - parent_class_size; }).value_or(class_size_without_parent); + const auto expected_pad_size = + first_field_offset.transform([&](auto e) { return e - parent_class_size.value_or(0); }).value_or(class_size_without_parent); // @todo: @es3n1n: if for some mysterious reason this class describes fields // of the base class we should handle it too. @@ -792,8 +791,7 @@ namespace { // @note: @es3n1n: insert padding if needed // - if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && - !state.assembling_bitfield) { + if (expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { builder.access_modifier("public") .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) .reset_tabs_count() @@ -871,8 +869,8 @@ namespace { if (verbose) { builder.reset_tabs_count() - .comment(std::format("offset={:#x} size={:#x} alignment={}", field.m_nSingleInheritanceOffset, field_size, - field_alignment.transform(to_hex_string).value_or("unknown")), + .comment(std::format("type.name=\"{}\" offset={:#x} size={:#x} alignment={}", std::string_view{field.m_pSchemaType->m_pszName}, + field.m_nSingleInheritanceOffset, field_size, field_alignment.transform(to_hex_string).value_or("unknown")), false) .restore_tabs_count(); } else { From fce7d827c24e41282b3f511e73d3a7e7b9e60d18 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 21:46:20 +0200 Subject: [PATCH 27/46] add padding before bitfields code never triggers in CS2 --- src/sdk/sdk.cpp | 73 ++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 19a7464..873b217 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -46,6 +46,7 @@ namespace { std::optional last_field_offset = std::nullopt; bool assembling_bitfield = false; std::vector bitfield = {}; + std::int32_t bitfield_start = 0; std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var }; @@ -653,6 +654,26 @@ namespace { return state; } + /// Does not insert a pad if it would have size 0 + void InsertPadUntil(codegen::generator_t::self_ref builder, const ClassAssemblyState& state, std::int32_t offset, bool verbose) { + if (verbose) { + builder.comment(std::format("last_field_offset={} last_field_size={}", state.last_field_offset.transform(&to_hex_string).value_or("none"), + state.last_field_size.transform(&to_hex_string).value_or("none"))); + } + + const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + + // insert padding only if needed + if (expected_offset < static_cast(offset) && !state.assembling_bitfield) { + builder.access_modifier("public") + .struct_padding(expected_offset, offset - expected_offset, false, true) + .reset_tabs_count() + .comment(std::format("{:#x}", expected_offset)) + .restore_tabs_count() + .access_modifier("public"); + } + } + void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { // TODO: when we have a CLI parser: pass this property in from the outside const bool verbose = true; @@ -713,7 +734,6 @@ namespace { // @note: @es3n1n: field assembling state // - // TOOD: inconsistent optional vs 0 ClassAssemblyState state = {.last_field_size = parent_class_size}; /// If fields cannot be emitted, e.g. because of collisions, they're added to @@ -773,7 +793,11 @@ namespace { // how large the bitfield is in order to choose the right type. We // only know how large the bitfield is once we've reached its end. if (var_info.is_bitfield()) { - state.assembling_bitfield = true; + if (!state.assembling_bitfield) { + state.assembling_bitfield = true; + state.bitfield_start = field.m_nSingleInheritanceOffset; + } + state.bitfield.emplace_back(BitfieldEntry{ .name = var_info.formatted_name(), .size = var_info.m_bitfield_size, @@ -782,44 +806,17 @@ namespace { continue; } - if (verbose) { - builder.comment(std::format("last_field_offset={} last_field_size={}", state.last_field_offset.transform(&to_hex_string).value_or("none"), - state.last_field_size.transform(&to_hex_string).value_or("none"))); - } + // At this point, we're never still inside a bitfield. If `assembling_bitfield` is set, that means we're at the first field following a + // bitfield, but the bitfield has not been emitted yet. + // note: in CS2, there are no types with padding before a bitfield + InsertPadUntil(builder, state, state.assembling_bitfield ? state.bitfield_start : field.m_nSingleInheritanceOffset, verbose); - const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); + // This is the first field after a bitfield, i.e. the active bitfield has ended. Emit the bitfield we have collected. + if (state.assembling_bitfield) { + state = AssembleBitfield(builder, std::move(state), state.last_field_offset.value_or(0) + state.last_field_size.value_or(0)); - // @note: @es3n1n: insert padding if needed - // - if (expected_offset < static_cast(field.m_nSingleInheritanceOffset) && !state.assembling_bitfield) { - builder.access_modifier("public") - .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) - .reset_tabs_count() - .comment(std::format("{:#x}", expected_offset)) - .restore_tabs_count() - .access_modifier("public"); - } - - // This is the first field after a bitfield, i.e. the active bitfield has ended - if (!var_info.is_bitfield() && state.assembling_bitfield) { - state = AssembleBitfield(builder, std::move(state), expected_offset); - - // TOOD: this is exact duplicate code - { - const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); - - // @note: @es3n1n: insert padding if needed - // - if (state.last_field_size.has_value() && expected_offset < static_cast(field.m_nSingleInheritanceOffset) && - !state.assembling_bitfield) { - builder.access_modifier("public") - .struct_padding(expected_offset, field.m_nSingleInheritanceOffset - expected_offset, false, true) - .reset_tabs_count() - .comment(std::format("{:#x}", expected_offset)) - .restore_tabs_count() - .access_modifier("public"); - } - } + // We need another pad here because the current loop iteration is already on a non-bitfield field which will get emitted right away. + InsertPadUntil(builder, state, field.m_nSingleInheritanceOffset, verbose); } // @note: @es3n1n: dump metadata From e6e8294085bd9247d0ff835fdd39ffabaf2e9ea8 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 21:52:19 +0200 Subject: [PATCH 28/46] more accurate misalignment warnings --- src/sdk/sdk.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 873b217..efa61ad 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -675,6 +675,8 @@ namespace { } void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + static constexpr std::size_t source2_max_align = 8; + // TODO: when we have a CLI parser: pass this property in from the outside const bool verbose = true; @@ -690,7 +692,7 @@ namespace { const auto class_size = class_.GetSize(); const auto class_alignment = class_.GetFullAlignment(); // Source2 has alignof(max_align_t)=8, i.e. every class whose size is a multiple of 8 is aligned. - const auto class_is_aligned = (class_size % class_alignment.value_or(8)) == 0; + const auto class_is_aligned = (class_size % class_alignment.value_or(source2_max_align)) == 0; const auto is_struct = std::string_view{class_.m_pszName}.ends_with("_t"); if (!class_is_aligned) { @@ -699,11 +701,11 @@ namespace { // ceil size to next possible aligned size const auto aligned_size = class_size + (class_alignment.value() - (class_size % class_alignment.value())) % class_alignment.value(); - return std::format("Type {} is misaligned. Its size should be {:#x}, but with proper alignement it has size {:#x}.", class_.GetName(), + return std::format("Type {} is misaligned. Its size should be {:#x}, but with proper alignment it has size {:#x}.", class_.GetName(), class_size, aligned_size); } else { - return std::format("Type {} appears to be misaligned. Its alignment is unknown and it is not aligned to max_align_t (8).", - class_.GetName()); + return std::format("Type {} appears to be misaligned. Its alignment is unknown and it is not aligned to max_align_t ({}).", + class_.GetName(), source2_max_align); } }(); warn(warning); @@ -837,8 +839,7 @@ namespace { // @note: @es3n1n: push prop // - // TOOD: use better default alignment - if ((field.m_nSingleInheritanceOffset % field_alignment.value_or(8)) == 0) { + if ((field.m_nSingleInheritanceOffset % field_alignment.value_or(source2_max_align)) == 0) { if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { // This is a workaround to get the size of template types right. // There are types that have non-type template parameters, e.g. @@ -857,9 +858,14 @@ namespace { builder.prop(var_info.m_type, var_info.formatted_name(), false); } } else { + const auto warning = + field_alignment.has_value() ? + std::format("Property {}::{} is misaligned.", class_.GetName(), field.m_pszName) : + std::format("Property {}::{} appears to be misaligned. Its alignment is unknown and it is not aligned to max_align_t ({}).", + class_.GetName(), field.m_pszName, source2_max_align); // TOOD: use warn() for all errors - warn(std::format("property {}::{} is misaligned. it has been replaced with a byte array.", class_.GetName(), field.m_pszName)); - builder.comment(std::format("property is misaligned, you can try marking {} as packed and uncommenting this property", class_.GetName())); + warn(warning); + builder.comment(warning); builder.prop("char", std::format("{}[{:#x}]", var_info.m_name, field_size), true); builder.comment("", false).reset_tabs_count().prop(var_info.m_type, var_info.formatted_name(), false).restore_tabs_count(); } From 1e6d12726364f995755e12449d1831a500a8ab3e Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 22:09:56 +0200 Subject: [PATCH 29/46] warn on all errors --- src/sdk/sdk.cpp | 65 ++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index efa61ad..409e420 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -727,29 +727,16 @@ namespace { builder.begin_multi_line_comment(); } - // @note: @es3n1n: start class - // - if (is_struct) - builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name, ""); - else - builder.begin_class_with_base_type(class_.m_pszName, parent_class_name, ""); - // @note: @es3n1n: field assembling state // ClassAssemblyState state = {.last_field_size = parent_class_size}; - /// If fields cannot be emitted, e.g. because of collisions, they're added to - /// this set so we can ignore them when asserting offsets. - std::unordered_set skipped_fields{}; - std::list> cached_fields{}; - std::list cached_datamap_fields{}; - // @note: @es3n1n: if we need to pad first field or if there's no fields in this class // and we need to properly pad it to make sure its size is the same as we expect it // - std::optional first_field_offset = std::nullopt; - if (const auto* first_field = (class_.m_pFields == nullptr) ? nullptr : &class_.m_pFields[0]; first_field) - first_field_offset = first_field->m_nSingleInheritanceOffset; + const auto* first_field = (class_.m_pFields == nullptr) ? nullptr : &class_.m_pFields[0]; + const std::optional first_field_offset = + (first_field != nullptr) ? std::make_optional(first_field->m_nSingleInheritanceOffset) : std::nullopt; const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size.value_or(0); @@ -758,29 +745,32 @@ namespace { // @todo: @es3n1n: if for some mysterious reason this class describes fields // of the base class we should handle it too. - if (class_parent && first_field_offset.has_value() && first_field_offset.value() < class_parent->m_nSizeOf) { - builder.comment( - std::format("Collision detected({:#x}->{:#x}), output may be wrong.", first_field_offset.value_or(0), class_parent->m_nSizeOf)); - state.collision_end_offset = class_parent->m_nSizeOf; + if ((class_parent != nullptr) && first_field_offset.has_value() && first_field_offset.value() < parent_class_size.value()) { + const auto warning = std::format("Collision detected: {} and its base {} have {:#x} overlapping byte(s)", class_.GetName(), parent_class_name, + parent_class_size.value() - first_field_offset.value()); + warn(warning); + builder.comment(warning); + state.collision_end_offset = parent_class_size.value(); } + // @note: @es3n1n: start class + // + if (is_struct) + builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name, ""); + else + builder.begin_class_with_base_type(class_.m_pszName, parent_class_name, ""); + + /// If fields cannot be emitted, e.g. because of collisions, they're added to + /// this set so we can ignore them when asserting offsets. + std::unordered_set skipped_fields{}; + std::list> cached_fields{}; + std::list cached_datamap_fields{}; + // @note: @es3n1n: begin public members // builder.access_modifier("public"); for (const auto& field : class_.GetFields()) { - // @fixme: @es3n1n: todo proper collision fix and remove this block - if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { - skipped_fields.emplace(field.m_pszName); - builder.comment( - std::format("Skipped field \"{}\" @ {:#x} because of the struct collision", field.m_pszName, field.m_nSingleInheritanceOffset)); - continue; - } - - if (!field.m_pSchemaType->GetSize().has_value()) { - std::cout << "MISSISG SIZE OF " << field.m_pSchemaType->m_pszName << std::endl; - } - // Fall back to size=1 because there are no 0-sized types. // `RenderPrimitiveType_t` is the only type (in CS2 9035763) without size information. const auto field_size = field.m_pSchemaType->GetSize().value_or(1); @@ -791,6 +781,16 @@ namespace { const auto [type_name, array_sizes] = GetType(*field.m_pSchemaType); const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); + // @fixme: @es3n1n: todo proper collision fix and remove this block + if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { + skipped_fields.emplace(field.m_pszName); + // A warning has already been logged at the start of the class + builder.comment( + std::format("Skipped field \"{}\" @ {:#x} because of the struct collision", field.m_pszName, field.m_nSingleInheritanceOffset)); + builder.comment("", false).reset_tabs_count().prop(var_info.m_type, var_info.formatted_name()).restore_tabs_count(); + continue; + } + // Collect all bitfield entries and emit them later. We need to know // how large the bitfield is in order to choose the right type. We // only know how large the bitfield is once we've reached its end. @@ -863,7 +863,6 @@ namespace { std::format("Property {}::{} is misaligned.", class_.GetName(), field.m_pszName) : std::format("Property {}::{} appears to be misaligned. Its alignment is unknown and it is not aligned to max_align_t ({}).", class_.GetName(), field.m_pszName, source2_max_align); - // TOOD: use warn() for all errors warn(warning); builder.comment(warning); builder.prop("char", std::format("{}[{:#x}]", var_info.m_name, field_size), true); From 2816bc756083a12f795c64a75b32f760e65aaf95 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Fri, 23 Aug 2024 22:15:29 +0200 Subject: [PATCH 30/46] remove mid-class access modifiers because we always use `public` because we want standard-layout-classes --- src/sdk/sdk.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 409e420..cae0c2f 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -665,12 +665,10 @@ namespace { // insert padding only if needed if (expected_offset < static_cast(offset) && !state.assembling_bitfield) { - builder.access_modifier("public") - .struct_padding(expected_offset, offset - expected_offset, false, true) + builder.struct_padding(expected_offset, offset - expected_offset, false, true) .reset_tabs_count() .comment(std::format("{:#x}", expected_offset)) - .restore_tabs_count() - .access_modifier("public"); + .restore_tabs_count(); } } @@ -756,9 +754,9 @@ namespace { // @note: @es3n1n: start class // if (is_struct) - builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name, ""); + builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name); else - builder.begin_class_with_base_type(class_.m_pszName, parent_class_name, ""); + builder.begin_class_with_base_type(class_.m_pszName, parent_class_name); /// If fields cannot be emitted, e.g. because of collisions, they're added to /// this set so we can ignore them when asserting offsets. @@ -766,10 +764,6 @@ namespace { std::list> cached_fields{}; std::list cached_datamap_fields{}; - // @note: @es3n1n: begin public members - // - builder.access_modifier("public"); - for (const auto& field : class_.GetFields()) { // Fall back to size=1 because there are no 0-sized types. // `RenderPrimitiveType_t` is the only type (in CS2 9035763) without size information. From a53defca00acc796027d25acc5e41ad11101a291 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sat, 24 Aug 2024 19:12:00 +0200 Subject: [PATCH 31/46] replace self-finding DeclaredClass with direct cast --- include/sdk/interfaces/common/CUtlTSHash.h | 5 +++- include/sdk/interfaces/schemasystem/schema.h | 30 ++++++++++++++++++-- sdk-static/CMakeLists.txt | 2 +- src/sdk/sdk.cpp | 5 ++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/include/sdk/interfaces/common/CUtlTSHash.h b/include/sdk/interfaces/common/CUtlTSHash.h index 1eb2153..0518ee1 100644 --- a/include/sdk/interfaces/common/CUtlTSHash.h +++ b/include/sdk/interfaces/common/CUtlTSHash.h @@ -2,10 +2,13 @@ // See end of file for extended copyright information. #pragma once #include "sdk/interfaces/common/CThreadSpinMutex.h" +#include "sdk/interfaces/common/CThreadSpinRWLock.h" #include "sdk/interfaces/common/CUtlMemory.h" #include "sdk/interfaces/common/CUtlMemoryPoolBase.h" +#include #include #include +#include #if defined(CS2) || defined(DOTA2) constexpr auto kUtlTsHashVersion = 2; @@ -273,7 +276,7 @@ inline std::vector CUtlTSHashV2::merge_wi std::vector merged_list = allocated_list; for (const auto& item : un_allocated_list) { - if (std::find_if(allocated_list.begin(), allocated_list.end(), [&](const T& elem) { return pred(elem, item); }) == allocated_list.end()) { + if (std::ranges::find_if(allocated_list, [&](const T& elem) { return pred(elem, item); }) == allocated_list.end()) { merged_list.push_back(item); } } diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 73b4d0b..ef1630e 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -2,6 +2,7 @@ // See end of file for extended copyright information. #pragma once +#include "sdk/interfaceregs.h" #include #include #include @@ -348,6 +349,8 @@ enum class SchemaBuiltinType_t : std::uint32_t { constexpr auto kSchemaBuiltinTypeCount = static_cast(SchemaBuiltinType_t::Schema_Builtin_count); +class CSchemaType_DeclaredClass; + class CSchemaType { public: [[nodiscard]] bool IsValid() { @@ -394,6 +397,10 @@ class CSchemaType { std::nullopt; } + /// @return @ref nullptr if this @ref GetTypeCategory() is not @ref ETypeCategory::Schema_DeclaredClass + CSchemaType_DeclaredClass* GetAsDeclaredClass(); + const CSchemaType_DeclaredClass* GetAsDeclaredClass() const; + // @todo: @og: find out to what class pointer points. [[nodiscard]] CSchemaType* GetRefClass() const; @@ -460,6 +467,7 @@ static_assert(sizeof(CSchemaType_Builtin) == 0x28); class CSchemaType_DeclaredClass : public CSchemaType { public: + /// never @ref nullptr CSchemaClassInfo* m_pClassInfo; bool m_bGlobalPromotionRequired; }; @@ -467,6 +475,21 @@ class CSchemaType_DeclaredClass : public CSchemaType { static_assert(offsetof(CSchemaType_DeclaredClass, m_pClassInfo) == 0x20); static_assert(sizeof(CSchemaType_DeclaredClass) == 0x30); +inline CSchemaType_DeclaredClass* CSchemaType::GetAsDeclaredClass() { + if (GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { + return static_cast(this); + } else { + return nullptr; + } +} +inline const CSchemaType_DeclaredClass* CSchemaType::GetAsDeclaredClass() const { + if (GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { + return static_cast(this); + } else { + return nullptr; + } +} + class CSchemaType_DeclaredEnum : public CSchemaType { public: CSchemaEnumBinding* m_pClassInfo; @@ -927,9 +950,10 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { auto field_alignments = this->GetFields() | std::ranges::views::transform([](const SchemaClassFieldData_t& e) { - // TOOD: lookups by name are slow. I've seen some better way somewhere. - if (const auto* class_ = e.m_pSchemaType->m_pTypeScope->FindDeclaredClass(e.m_pSchemaType->m_pszName); class_ != nullptr) { - return class_->GetFullAlignment(); + if (const auto* class_ = e.m_pSchemaType->GetAsDeclaredClass(); class_ != nullptr) { + assert(e.m_pSchemaType->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass); + assert(static_cast(e.m_pSchemaType)->m_pClassInfo->GetName() == e.m_pSchemaType->m_pszName); + return class_->m_pClassInfo->GetFullAlignment(); } else { return e.m_pSchemaType->GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); } diff --git a/sdk-static/CMakeLists.txt b/sdk-static/CMakeLists.txt index b800026..2214f20 100644 --- a/sdk-static/CMakeLists.txt +++ b/sdk-static/CMakeLists.txt @@ -1,4 +1,4 @@ -# TOOD: use meson +# TOOD: use meson - but meson can't glob files cmake_minimum_required(VERSION 3.30) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index cae0c2f..4661d2a 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -613,8 +613,8 @@ namespace { /// @return For class types, returns @ref CSchemaClassInfo::GetFullAlignment(). Otherwise returns the immediately available size. [[nodiscard]] std::optional GetFullAlignmentOfType(const CSchemaType& type) { - if (const auto* class_ = type.m_pTypeScope->FindDeclaredClass(type.m_pszName); class_ != nullptr) { - return class_->GetFullAlignment(); + if (const auto* class_ = type.GetAsDeclaredClass(); class_ != nullptr) { + return class_->m_pClassInfo->GetFullAlignment(); } else { return type.GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); } @@ -885,7 +885,6 @@ namespace { // pad the class end. const auto last_field_end = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); - // TOOD: use aligned size? const auto end_pad = class_size - last_field_end; if (end_pad != 0) { From fbe7ac5d8aa087a749f8e1fc1be7bb7bf8a072c2 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Sun, 25 Aug 2024 20:00:18 +0200 Subject: [PATCH 32/46] move `to_hex_string()` to `util` --- include/sdk/interfaces/schemasystem/schema.h | 69 +++++++++++--------- include/tools/util.h | 11 ++++ src/sdk/sdk.cpp | 17 ++--- 3 files changed, 55 insertions(+), 42 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index ef1630e..8878906 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -350,6 +350,7 @@ enum class SchemaBuiltinType_t : std::uint32_t { constexpr auto kSchemaBuiltinTypeCount = static_cast(SchemaBuiltinType_t::Schema_Builtin_count); class CSchemaType_DeclaredClass; +class CSchemaType_DeclaredEnum; class CSchemaType { public: @@ -398,8 +399,8 @@ class CSchemaType { } /// @return @ref nullptr if this @ref GetTypeCategory() is not @ref ETypeCategory::Schema_DeclaredClass - CSchemaType_DeclaredClass* GetAsDeclaredClass(); const CSchemaType_DeclaredClass* GetAsDeclaredClass() const; + const CSchemaType_DeclaredEnum* GetAsDeclaredEnum() const; // @todo: @og: find out to what class pointer points. [[nodiscard]] CSchemaType* GetRefClass() const; @@ -475,21 +476,6 @@ class CSchemaType_DeclaredClass : public CSchemaType { static_assert(offsetof(CSchemaType_DeclaredClass, m_pClassInfo) == 0x20); static_assert(sizeof(CSchemaType_DeclaredClass) == 0x30); -inline CSchemaType_DeclaredClass* CSchemaType::GetAsDeclaredClass() { - if (GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - return static_cast(this); - } else { - return nullptr; - } -} -inline const CSchemaType_DeclaredClass* CSchemaType::GetAsDeclaredClass() const { - if (GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { - return static_cast(this); - } else { - return nullptr; - } -} - class CSchemaType_DeclaredEnum : public CSchemaType { public: CSchemaEnumBinding* m_pClassInfo; @@ -572,6 +558,22 @@ class CSchemaType_FixedArray : public CSchemaType { CSchemaType* m_pElementType; }; +inline const CSchemaType_DeclaredClass* CSchemaType::GetAsDeclaredClass() const { + if (GetTypeCategory() == ETypeCategory::Schema_DeclaredClass) { + return static_cast(this); + } else { + return nullptr; + } +} + +inline const CSchemaType_DeclaredEnum* CSchemaType::GetAsDeclaredEnum() const { + if (GetTypeCategory() == ETypeCategory::Schema_DeclaredEnum) { + return static_cast(this); + } else { + return nullptr; + } +} + struct AtomicTypeInfo_T_t { int m_nAtomicID; CSchemaType* m_pTemplateType; @@ -740,16 +742,16 @@ class CSchemaSystemTypeScope { } } - [[nodiscard]] CSchemaType* FindSchemaTypeByName(const std::string_view szName) { + [[nodiscard]] const CSchemaType_Builtin* FindBuiltIn(const std::string_view szName) const { assert((std::strlen(szName.data()) == szName.size()) && "need a zero-terminated string"); if constexpr (kSchemaSystemVersion == 2) { - CSchemaType* schema_type; + CSchemaType_Builtin* schema_type = nullptr; - Virtual::Get(this, 4)(this, &schema_type, szName.data()); + Virtual::Get(this, 4)(this, &schema_type, szName.data()); return schema_type; } else { - return Virtual::Get(this, 4)(this, szName.data()); + return Virtual::Get(this, 4)(this, szName.data()); } } @@ -815,6 +817,9 @@ class CSchemaSystemTypeScope { [[nodiscard]] CUtlMap& GetDeclaredClasses() { return m_DeclaredClasses.m_Map; } + [[nodiscard]] const CUtlMap& GetDeclaredClasses() const { + return m_DeclaredClasses.m_Map; + } [[nodiscard]] CUtlMap& GetDeclaredEnums() { return m_DeclaredEnums.m_Map; @@ -822,7 +827,7 @@ class CSchemaSystemTypeScope { private: void* vftable = nullptr; - std::array m_szName = {}; // 0x0008 + std::array m_szName = {}; // 0x0008 CSchemaSystemTypeScope* m_pGlobalTypeScope = nullptr; // 0x0108 bool m_bBuiltinTypesInitialized = false; // 0x0110 IF_LINUX(char _pad_0x114[0x04];) @@ -836,25 +841,25 @@ class CSchemaSystemTypeScope { CSchemaPtrMap m_AtomicsTF; // 0x0408 #endif - CSchemaPtrMap m_AtomicsTT; // 0x0438 + CSchemaPtrMap m_AtomicsTT; // 0x0408 #if defined(CS2_OLD) CSchemaPtrMap m_AtomicsTTF; // 0x0468 #endif - CSchemaPtrMap m_AtomicsI; // 0x0498 - CSchemaPtrMap m_DeclaredClasses; // 0x04C8 - CSchemaPtrMap m_DeclaredEnums; // 0x04F8 - CSchemaPtrMap m_AtomicInfos; // 0x0528 - CSchemaPtrMap m_FixedArrays; // 0x0558 - CSchemaPtrMap m_Bitfields; // 0x0588 + CSchemaPtrMap m_AtomicsI; // 0x0438 + CSchemaPtrMap m_DeclaredClasses; // 0x0468 + CSchemaPtrMap m_DeclaredEnums; // 0x0498 + CSchemaPtrMap m_AtomicInfos; // 0x04C8 + CSchemaPtrMap m_FixedArrays; // 0x04F8 + CSchemaPtrMap m_Bitfields; // 0x0528 #if !defined(DOTA2) && !defined(CS2) CSchemaType_NoschemaType m_pNoschemaType = {}; #endif - CUtlTSHash m_ClassBindings = {}; // 0x05C0 - CUtlTSHash m_EnumBindings = {}; // 0x2E50 + CUtlTSHash m_ClassBindings = {}; // 0x0560 + CUtlTSHash m_EnumBindings = {}; // 0x3600 }; class CSchemaClassInfo : public SchemaClassInfoData_t { @@ -1115,8 +1120,8 @@ class CSchemaSystem { private: char pad_0x0000[kSchemaSystem_PAD0] = {}; // 0x0000 - CUtlVector m_TypeScopes = {}; // SCHEMASYSTEM_TYPE_SCOPES_OFFSET - char pad_01A0[kSchemaSystem_PAD1] = {}; // 0x01A0 + CUtlVector m_TypeScopes = {}; // linux: 0x01F0 + char pad_0x01A0[kSchemaSystem_PAD1] = {}; // 0x01A0 std::int32_t m_nRegistrations = 0; // 0x02C0 std::int32_t m_nIgnored = 0; // 0x02C4 std::int32_t m_nRedundant = 0; // 0x02C8 diff --git a/include/tools/util.h b/include/tools/util.h index 0bf3bd5..eec0a5e 100644 --- a/include/tools/util.h +++ b/include/tools/util.h @@ -2,6 +2,8 @@ // See end of file for extended copyright information. #pragma once +#include +#include #include #include @@ -19,6 +21,15 @@ namespace util { return std::to_string(num); } + + /// Useful for optional integers, e.g. + /// ```cpp + /// const std::optional offset = try_get_offset(); + /// std::cout << std::format("offset: {}\n", offset.transform(to_hex_string).value_or("unknown")); + /// ``` + inline std::string to_hex_string(int i) { + return std::format("{:#x}", i); + } } // namespace util // source2gen - Source2 games SDK generator diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 4661d2a..221eee2 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -53,11 +53,6 @@ namespace { using namespace std::string_view_literals; - // TOOD: move to a better place - std::string to_hex_string(int i) { - return std::format("{:#x}", i); - } - /** * Project structure is * @@ -202,8 +197,8 @@ namespace { }; void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { - builder.comment(std::format("Registered alignment: {}", class_.GetRegisteredAlignment().transform(&to_hex_string).value_or("unknown"))); - builder.comment(std::format("Alignment: {}", class_.GetFullAlignment().transform(&to_hex_string).value_or("unknown"))); + builder.comment(std::format("Registered alignment: {}", class_.GetRegisteredAlignment().transform(&util::to_hex_string).value_or("unknown"))); + builder.comment(std::format("Alignment: {}", class_.GetFullAlignment().transform(&util::to_hex_string).value_or("unknown"))); builder.comment(std::format("Size: {:#x}", class_.m_nSizeOf)); if ((class_.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable @@ -657,8 +652,9 @@ namespace { /// Does not insert a pad if it would have size 0 void InsertPadUntil(codegen::generator_t::self_ref builder, const ClassAssemblyState& state, std::int32_t offset, bool verbose) { if (verbose) { - builder.comment(std::format("last_field_offset={} last_field_size={}", state.last_field_offset.transform(&to_hex_string).value_or("none"), - state.last_field_size.transform(&to_hex_string).value_or("none"))); + builder.comment(std::format("last_field_offset={} last_field_size={}", + state.last_field_offset.transform(&util::to_hex_string).value_or("none"), + state.last_field_size.transform(&util::to_hex_string).value_or("none"))); } const auto expected_offset = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); @@ -866,7 +862,8 @@ namespace { if (verbose) { builder.reset_tabs_count() .comment(std::format("type.name=\"{}\" offset={:#x} size={:#x} alignment={}", std::string_view{field.m_pSchemaType->m_pszName}, - field.m_nSingleInheritanceOffset, field_size, field_alignment.transform(to_hex_string).value_or("unknown")), + field.m_nSingleInheritanceOffset, field_size, + field_alignment.transform(&util::to_hex_string).value_or("unknown")), false) .restore_tabs_count(); } else { From 182801f3db824cf7bbe54107a40ef10c8302ad7b Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 26 Aug 2024 16:27:31 +0200 Subject: [PATCH 33/46] sdk headers are generated in an `include` directory --- sdk-static/CMakeLists.txt | 11 +- .../source2sdk/source2gen_user_types.hpp | 138 ++++++++++++++++++ src/sdk/sdk.cpp | 27 ++-- 3 files changed, 157 insertions(+), 19 deletions(-) create mode 100644 sdk-static/include/source2sdk/source2gen_user_types.hpp diff --git a/sdk-static/CMakeLists.txt b/sdk-static/CMakeLists.txt index 2214f20..f62962b 100644 --- a/sdk-static/CMakeLists.txt +++ b/sdk-static/CMakeLists.txt @@ -1,4 +1,4 @@ -# TOOD: use meson - but meson can't glob files +# TOOD: try using in a playground cmake_minimum_required(VERSION 3.30) @@ -10,16 +10,11 @@ project(source2sdk file(GLOB_RECURSE source2sdk_headers "./**.hpp") -add_library(${PROJECT_NAME} ${source2sdk_headers}) -target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +add_library(${PROJECT_NAME} INTERFACE) +target_include_directories(${PROJECT_NAME} INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) -target_compile_options(${PROJECT_NAME} PRIVATE - "-Wfatal-errors" - "-pedantic-errors" -) - # Add a target that includes all headers of the generated library to check for compile-time errors foreach(el ${source2sdk_headers}) diff --git a/sdk-static/include/source2sdk/source2gen_user_types.hpp b/sdk-static/include/source2sdk/source2gen_user_types.hpp new file mode 100644 index 0000000..fc7ee65 --- /dev/null +++ b/sdk-static/include/source2sdk/source2gen_user_types.hpp @@ -0,0 +1,138 @@ +#pragma once + +#include +#include + +template +using CAnimValue = char[0x08]; +using CAnimVariant = char[0x11]; +// size is a guess +template +using CAnimScriptParam = char[0x08]; +using CBufferString = char[0x10]; +using CColorGradient = char[0x18]; +// size doesn't mapper. only used as a pointer +template +using CCompressor = char[0x01]; +using CEntityHandle = char[0x04]; +using CEntityIndex = char[0x04]; +using CGlobalSymbol = char[0x08]; +using CKV3MemberNameWithStorage = char[0x38]; +using CNetworkedQuantizedFloat = char[0x08]; +using CParticleNamedValueRef = char[0x40]; +using CPiecewiseCurve = char[0x40]; +using CPlayerSlot = char[0x04]; +using CPulseValueFullType = char[0x10]; +using CResourceName = char[0xe0]; +using CSplitScreenSlot = char[0x04]; +using CTransform = char[0x20]; +using CUtlBinaryBlock = char[0x18]; +template +using CUtlHashtable = char[0x20]; +using CUtlStringTokenWithStorage = char[0x18]; +using CUtlStringToken = char[0x04]; +using CUtlString = char[0x08]; +using CUtlSymbolLarge = char[0x08]; +using CUtlSymbol = char[0x02]; +template +using CAnimGraphParamOptionalRef = char[0x20]; +template +using CAnimGraphParamRef = char[0x20]; +template +using CBitVec = char[(N + 7) / 8]; +template +using CEntityOutputTemplate = char[0x28]; +template +using CHandle = char[0x04]; +template +using C_NetworkUtlVectorBase = char[0x18]; +template +using CNetworkUtlVectorBase = char[0x18]; +// size unknown. only used in dynamic containers. +using CSoundEventName = char[0x01]; +template +using CUtlLeanVector = char[0x10]; +template +using CUtlOrderedMap = char[0x28]; +// size doesn't mapper. only used as a pointer +template +using CUtlPair = char[0x01]; +template +using CUtlVector = char[0x18]; +// size is a guess that fits both occurences of this type in CS2 +template +using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; +template +using CUtlLeanVectorFixedGrowable = char[0x10 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; +template +using C_UtlVectorEmbeddedNetworkVar = char[0x50]; +template +using CUtlVectorEmbeddedNetworkVar = char[0x50]; +using CUtlVectorSIMDPaddedVector = char[0x18]; +template +using CSmartPtr = char[0x08]; +template +using CResourceArray = char[0x08]; +// size unknown +using CResourceString = char[0x08]; +template +using CResourcePointer = char[0x08]; +template +using CResourceNameTyped = char[0xe0]; +template +using CStrongHandle = char[0x08]; +template +using CStrongHandleCopyable = char[0x08]; +// size doesn't mapper. only used as a pointer +using CStrongHandleVoid = char[0x08]; +template +using CVariantBase = char[0x10]; +template +using CWeakHandle = char[0x18]; +using Color = char[0x04]; +using DegreeEuler = char[0x0c]; +using FourVectors = char[0x30]; +using HSCRIPT = char[0x08]; +using KeyValues3 = char[0x10]; +// size doesn't mapper. only used as a pointer +using KeyValues = char[0x01]; +using QAngle = char[0x0c]; +using QuaternionStorage = char[0x10]; +using Quaternion = char[0x10]; +using RadianEuler = char[0x0c]; +using RenderInputLayoutField_t = char[0x04]; +// we don't have a field size for this type. uses the fallback of 1. +using RenderPrimitiveType_t = char[0x01]; +using RotationVector = char[0x0c]; +template +using SphereBase_t = char[0x10]; +using Vector2D = char[0x08]; +using Vector4D = char[0x10]; +using VectorAligned = char[0x10]; +using Vector = char[0x0c]; +using WorldGroupId_t = char[0x04]; +using float32 = char[0x04]; +using fltx4 = char[0x10]; +using matrix3x4_t = char[0x30]; +using matrix3x4a_t = char[0x30]; + +// intentionally left undefined. if you want to access static fields, add your own sdk. +namespace interfaces { + struct SchemaStaticFieldData_t { + void* m_pInstance{}; + }; + + struct CSchemaClassInfo { + auto GetStaticFields() -> SchemaStaticFieldData_t**; + }; + + struct CSchemaSystemTypeScope { + auto FindDeclaredClass(std::string_view) -> CSchemaClassInfo*; + }; + + struct schema_t { + auto FindTypeScopeForModule(std::string_view) -> CSchemaSystemTypeScope*; + }; + + extern schema_t* g_schema; +} // namespace interfaces diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 221eee2..c013b8c 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -58,13 +58,13 @@ namespace { * * - CMakeLists.txt * - ... - * - - * - some_module - * - some_header.hpp + * - include/ + * - + * - some_module + * - some_header.hpp */ - constexpr std::string_view kOutDirName = "sdk"sv; - /// Include directory - constexpr std::string_view kSdkDirName = "source2sdk"sv; + constexpr std::string_view kOutDirName = "sdk"; + constexpr std::string_view kIncludeDirName = "source2sdk"; constexpr uint32_t kMaxReferencesForClassEmbed = 2; constexpr std::size_t kMinFieldCountForClassEmbed = 2; @@ -986,8 +986,13 @@ namespace { builder.static_assert_size(class_.m_pszName, class_size); } + [[nodiscard]] + std::filesystem::path GetFilePathForType(std::string_view module_name, std::string_view type_name) { + return std::format("{}/include/{}/{}/{}.hpp", kOutDirName, kIncludeDirName, module_name, DecayTypeName(type_name)); + } + void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { - const std::string out_file_path = std::format("{}/{}/{}/{}.hpp", kOutDirName, kSdkDirName, module_name, enum_.m_pszName); + const std::string out_file_path = GetFilePathForType(module_name, enum_.m_pszName); // @note: @es3n1n: init codegen // @@ -1028,7 +1033,7 @@ namespace { } void GenerateClassSdk(std::string_view module_name, const CSchemaClassBinding& class_) { - const std::string out_file_path = std::format("{}/{}/{}/{}.hpp", kOutDirName, kSdkDirName, module_name, class_.m_pszName); + const std::string out_file_path = GetFilePathForType(module_name, class_.m_pszName); // @note: @es3n1n: init codegen // @@ -1038,10 +1043,10 @@ namespace { const auto names = GetRequiredNamesForClass(class_); for (const auto& include : names | std::views::filter([](const auto& el) { return el.source == NameSource::include; })) { - builder.include(std::format("\"{}/{}/{}.hpp\"", kSdkDirName, include.module, include.type_name)); + builder.include(std::format("\"{}/{}/{}.hpp\"", kIncludeDirName, include.module, include.type_name)); } - builder.include(std::format("\"{}/source2gen_user_types.hpp\"", kSdkDirName)); + builder.include(std::format("\"{}/source2gen_user_types.hpp\"", kIncludeDirName)); builder.include(""); // for offsetof() builder.include(""); @@ -1091,7 +1096,7 @@ namespace sdk { std::cout << std::format("{}: Assembling module {} with {} enum(s) and {} class(es)", __FUNCTION__, module_name, enums.size(), classes.size()) << std::endl; - const std::filesystem::path out_directory_path = std::format("{}/{}/{}", kOutDirName, kSdkDirName, module_name); + const std::filesystem::path out_directory_path = std::format("{}/include/{}/{}", kOutDirName, kIncludeDirName, module_name); if (!std::filesystem::exists(out_directory_path)) std::filesystem::create_directories(out_directory_path); From e497dbd60bd9f3904444dde31c8e849ade27ee26 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 26 Aug 2024 17:18:18 +0200 Subject: [PATCH 34/46] cache alignment, check for standard-layout --- include/sdk/interfaces/schemasystem/schema.h | 43 ------- include/sdk/sdk.h | 20 ++- src/sdk/sdk.cpp | 129 +++++++++++++++---- src/startup/startup.cpp | 4 +- 4 files changed, 126 insertions(+), 70 deletions(-) diff --git a/include/sdk/interfaces/schemasystem/schema.h b/include/sdk/interfaces/schemasystem/schema.h index 8878906..d3d522d 100644 --- a/include/sdk/interfaces/schemasystem/schema.h +++ b/include/sdk/interfaces/schemasystem/schema.h @@ -937,49 +937,6 @@ class CSchemaClassInfo : public SchemaClassInfoData_t { return m_unAlignOf == std::numeric_limits::max() ? std::nullopt : std::make_optional(static_cast(m_unAlignOf)); } - /// This function recurses through all members. That's expensive. Cache the result if possible. - /// @return @ref GetRegisteredAlignment() if set. Otherwise tries to determine the alignment. - /// Returns @ref std::nullopt if one or more fields have unknown alignment. - [[nodiscard]] std::optional GetFullAlignment() const { - return GetRegisteredAlignment().or_else([this]() { - int base_alignment = 0; - - if (this->m_pBaseClasses != nullptr) { - if (const auto maybe_base_alignment = this->m_pBaseClasses->m_pClass->GetFullAlignment()) { - base_alignment = maybe_base_alignment.value(); - } else { - // we have a base class, but it has unknown alignment - return std::optional{}; - } - } - - auto field_alignments = - this->GetFields() | std::ranges::views::transform([](const SchemaClassFieldData_t& e) { - if (const auto* class_ = e.m_pSchemaType->GetAsDeclaredClass(); class_ != nullptr) { - assert(e.m_pSchemaType->GetTypeCategory() == ETypeCategory::Schema_DeclaredClass); - assert(static_cast(e.m_pSchemaType)->m_pClassInfo->GetName() == e.m_pSchemaType->m_pszName); - return class_->m_pClassInfo->GetFullAlignment(); - } else { - return e.m_pSchemaType->GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); - } - }); - - if (field_alignments.empty()) { - // This is an empty class. The generator will add a single pad with alignment 1. - return std::make_optional((base_alignment == 0) ? 1 : base_alignment); - } else if (std::ranges::all_of(field_alignments, &std::optional::has_value)) { - int max_alignment = base_alignment; - for (const auto& e : field_alignments) { - max_alignment = std::max(max_alignment, e.value()); - } - return std::make_optional(max_alignment); - } else { - // there are fields with unknown alignment - return std::optional{}; - } - }); - } - // @note: @og: Copy instance from original to new created with all data from original, returns new_instance auto CopyInstance(void* instance, void* new_instance) const { return CallFunction(SchemaClassInfoFunctionIndex::kCreateInstance, instance, new_instance); diff --git a/include/sdk/sdk.h b/include/sdk/sdk.h index f06b92c..e3057d1 100644 --- a/include/sdk/sdk.h +++ b/include/sdk/sdk.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -28,7 +29,24 @@ namespace sdk { inline CSchemaSystem* g_schema = nullptr; - void GenerateTypeScopeSdk(std::string_view module_name, const std::unordered_set& enums, + /// Unique identifier for a type in the source2 engine + struct TypeIdentifier { + std::string module{}; + std::string name{}; + + auto operator<=>(const TypeIdentifier&) const = default; + }; + + /// Stores results of expensive function calls, like those that recurse through classes. + struct GeneratorCache { + /// Key is {module,class} + /// If an entry exists for a class, but its value is @ref std::nullopt, we have already tried finding its alignment but couldn't figure it out. + std::map> class_alignment{}; + /// Key is {module,class} + std::map class_has_standard_layout{}; + }; + + void GenerateTypeScopeSdk(GeneratorCache& cache, std::string_view module_name, const std::unordered_set& enums, const std::unordered_set& classes); } // namespace sdk diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index c013b8c..375c276 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -196,9 +196,102 @@ namespace { return value; }; - void PrintClassInfo(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + /// https://en.cppreference.com/w/cpp/language/classes#Standard-layout_class + /// Doesn't check for all requirements, but is strict enough for what we are doing. + [[nodiscard]] bool IsStandardLayoutClass(std::map& cache, const CSchemaClassInfo& class_) { + const auto id = sdk::TypeIdentifier{.module = std::string{class_.GetModule()}, .name = std::string{class_.GetName()}}; + + if (const auto found = cache.find(id); found != cache.end()) { + return found->second; + } + + // This is a simplification of the requirement "only one class in the hierarchy has non-static data members". + // We assume that every class has non-static data members (TODO: That's wrong). + if (class_.m_pBaseClasses != 0) { + return cache.emplace(id, false).first->second; + } + + const auto has_non_standard_layout_field = + std::ranges::any_of(class_.GetFields() | std::ranges::views::transform([&](const SchemaClassFieldData_t& e) { + if (const auto* e_class = e.m_pSchemaType->GetAsDeclaredClass(); e_class != nullptr) { + return !IsStandardLayoutClass(cache, *e_class->m_pClassInfo); + } else { + // Everything that is not a class has no effect + return false; + } + }), + std::identity{}); + + if (has_non_standard_layout_field) { + return cache.emplace(id, false).first->second; + } + + return cache.emplace(id, true).first->second; + } + + /// Gets the alignment of a class by recursing through all of its fields. + /// Does not guess, the returned value is correct if set. + /// @param cache Used to look up and store alignment of fields + /// @return @ref GetRegisteredAlignment() if set. Otherwise tries to determine the alignment by recursing through all fields. + /// Returns @ref std::nullopt if one or more fields have unknown alignment. + [[nodiscard]] std::optional GetClassAlignmentRecursive(std::map>& cache, const CSchemaClassInfo& class_) { + const auto id = sdk::TypeIdentifier{.module = std::string{class_.GetModule()}, .name = std::string{class_.GetName()}}; + + if (const auto found = cache.find(id); found != cache.end()) { + return found->second; + } + + return class_.GetRegisteredAlignment().or_else([&]() { + int base_alignment = 0; + + if (class_.m_pBaseClasses != nullptr) { + if (const auto maybe_base_alignment = GetClassAlignmentRecursive(cache, *class_.m_pBaseClasses->m_pClass)) { + base_alignment = maybe_base_alignment.value(); + } else { + // we have a base class, but it has unknown alignment + return cache.emplace(id, std::nullopt).first->second; + } + } + + auto field_alignments = class_.GetFields() | std::ranges::views::transform([&](const SchemaClassFieldData_t& e) { + if (const auto* e_class = e.m_pSchemaType->GetAsDeclaredClass(); e_class != nullptr) { + return GetClassAlignmentRecursive(cache, *e_class->m_pClassInfo); + } else { + return e.m_pSchemaType->GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); + } + }); + + if (field_alignments.empty()) { + // This is an empty class. The generator will add a single pad with alignment 1. + return cache.emplace(id, std::make_optional((base_alignment == 0) ? 1 : base_alignment)).first->second; + } else if (std::ranges::all_of(field_alignments, &std::optional::has_value)) { + int max_alignment = base_alignment; + for (const auto& e : field_alignments) { + max_alignment = std::max(max_alignment, e.value()); + } + return cache.emplace(id, std::make_optional(max_alignment)).first->second; + } else { + // there are fields with unknown alignment + return cache.emplace(id, std::nullopt).first->second; + } + }); + } + + /// @return For class types, returns @ref GetClassAlignmentRecursive(). Otherwise returns the immediately available size. + [[nodiscard]] + std::optional GetAlignmentOfTypeRecursive(std::map>& cache, const CSchemaType& type) { + if (const auto* class_ = type.GetAsDeclaredClass(); class_ != nullptr) { + return GetClassAlignmentRecursive(cache, *class_->m_pClassInfo); + } else { + return type.GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); + } + } + + void PrintClassInfo(sdk::GeneratorCache& cache, codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { builder.comment(std::format("Registered alignment: {}", class_.GetRegisteredAlignment().transform(&util::to_hex_string).value_or("unknown"))); - builder.comment(std::format("Alignment: {}", class_.GetFullAlignment().transform(&util::to_hex_string).value_or("unknown"))); + builder.comment( + std::format("Alignment: {}", GetClassAlignmentRecursive(cache.class_alignment, class_).transform(&util::to_hex_string).value_or("unknown"))); + builder.comment(std::format("Standard-layout class: {}", IsStandardLayoutClass(cache.class_has_standard_layout, class_))); builder.comment(std::format("Size: {:#x}", class_.m_nSizeOf)); if ((class_.m_nClassFlags & SCHEMA_CF1_HAS_VIRTUAL_MEMBERS) != 0) // @note: @og: its means that class probably does have vtable @@ -603,18 +696,6 @@ namespace { return result; } - // TOOD: Create a "ShadowClass" map to store full alignment and standard-layout-class status? - - /// @return For class types, returns @ref CSchemaClassInfo::GetFullAlignment(). Otherwise returns the immediately available size. - [[nodiscard]] - std::optional GetFullAlignmentOfType(const CSchemaType& type) { - if (const auto* class_ = type.GetAsDeclaredClass(); class_ != nullptr) { - return class_->m_pClassInfo->GetFullAlignment(); - } else { - return type.GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); - } - } - [[nodiscard]] ClassAssemblyState AssembleBitfield(codegen::generator_t& builder, ClassAssemblyState&& state, int expected_offset) { state.assembling_bitfield = false; @@ -668,7 +749,7 @@ namespace { } } - void AssembleClass(codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { + void AssembleClass(sdk::GeneratorCache& cache, codegen::generator_t::self_ref builder, const CSchemaClassBinding& class_) { static constexpr std::size_t source2_max_align = 8; // TODO: when we have a CLI parser: pass this property in from the outside @@ -684,7 +765,7 @@ namespace { // const auto* class_parent = class_.m_pBaseClasses ? class_.m_pBaseClasses->m_pClass : nullptr; const auto class_size = class_.GetSize(); - const auto class_alignment = class_.GetFullAlignment(); + const auto class_alignment = GetClassAlignmentRecursive(cache.class_alignment, class_); // Source2 has alignof(max_align_t)=8, i.e. every class whose size is a multiple of 8 is aligned. const auto class_is_aligned = (class_size % class_alignment.value_or(source2_max_align)) == 0; const auto is_struct = std::string_view{class_.m_pszName}.ends_with("_t"); @@ -710,7 +791,7 @@ namespace { builder.end_struct(); } - PrintClassInfo(builder, class_); + PrintClassInfo(cache, builder, class_); // @note: @es3n1n: get parent name // @@ -764,7 +845,7 @@ namespace { // Fall back to size=1 because there are no 0-sized types. // `RenderPrimitiveType_t` is the only type (in CS2 9035763) without size information. const auto field_size = field.m_pSchemaType->GetSize().value_or(1); - const auto field_alignment = GetFullAlignmentOfType(*field.m_pSchemaType); + const auto field_alignment = GetAlignmentOfTypeRecursive(cache.class_alignment, *field.m_pSchemaType); // @note: @es3n1n: parsing type // @@ -957,9 +1038,7 @@ namespace { builder.end_block(); - // TODO: this check is incomplete. we should also check if any of the class's fields is a non-standard-layout class. That's the case for e.g. - // soundsystem::CSosSoundEventGroupSchema - const bool is_standard_layout_class = (class_.m_nBaseClassSize == 0); + const bool is_standard_layout_class = IsStandardLayoutClass(cache.class_has_standard_layout, class_); // TODO: when we have a CLI parser: allow users to generate assertions in non-standard-layout classes. Those assertions are // conditionally-supported. @@ -1032,7 +1111,7 @@ namespace { } } - void GenerateClassSdk(std::string_view module_name, const CSchemaClassBinding& class_) { + void GenerateClassSdk(sdk::GeneratorCache& cache, std::string_view module_name, const CSchemaClassBinding& class_) { const std::string out_file_path = GetFilePathForType(module_name, class_.m_pszName); // @note: @es3n1n: init codegen @@ -1069,7 +1148,7 @@ namespace { // @note: @es3n1n: assemble props // - AssembleClass(builder, class_); + AssembleClass(cache, builder, class_); builder.end_namespace(); @@ -1089,7 +1168,7 @@ namespace { } // namespace namespace sdk { - void GenerateTypeScopeSdk(std::string_view module_name, const std::unordered_set& enums, + void GenerateTypeScopeSdk(GeneratorCache& cache, std::string_view module_name, const std::unordered_set& enums, const std::unordered_set& classes) { // @note: @es3n1n: print debug info // @@ -1102,7 +1181,7 @@ namespace sdk { std::filesystem::create_directories(out_directory_path); std::ranges::for_each(enums, [=](const auto* el) { GenerateEnumSdk(module_name, *el); }); - std::ranges::for_each(classes, [=](const auto* el) { GenerateClassSdk(module_name, *el); }); + std::ranges::for_each(classes, [&](const auto* el) { GenerateClassSdk(cache, module_name, *el); }); } } // namespace sdk diff --git a/src/startup/startup.cpp b/src/startup/startup.cpp index bb77f41..f688189 100644 --- a/src/startup/startup.cpp +++ b/src/startup/startup.cpp @@ -171,8 +171,10 @@ namespace source2_gen { const std::unordered_map all_modules = collect_modules(std::span{type_scopes.m_pElements, static_cast(type_scopes.m_Size)}); + sdk::GeneratorCache cache{}; + for (const auto& [module_name, dump] : all_modules) { - sdk::GenerateTypeScopeSdk(module_name, dump.enums, dump.classes); + sdk::GenerateTypeScopeSdk(cache, module_name, dump.enums, dump.classes); } std::cout << std::format("Schema stats: {} registrations; {} were redundant; {} were ignored ({} bytes of ignored data)", From 308223e9cb0fdfb073b0c1296140ca77495d988b Mon Sep 17 00:00:00 2001 From: Cre3per Date: Mon, 26 Aug 2024 17:32:18 +0200 Subject: [PATCH 35/46] classes with base classes can have standard-layout --- src/sdk/sdk.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 375c276..4550045 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -205,10 +205,20 @@ namespace { return found->second; } - // This is a simplification of the requirement "only one class in the hierarchy has non-static data members". - // We assume that every class has non-static data members (TODO: That's wrong). - if (class_.m_pBaseClasses != 0) { - return cache.emplace(id, false).first->second; + // only one class in the hierarchy has non-static data members. + // assumes that source2 only has single inheritance. + { + const auto* pClass = &class_; + int classes_with_fields = 0; + do { + classes_with_fields += (pClass->m_nFieldSize != 0) ? 1 : 0; + + if (classes_with_fields > 1) { + return cache.emplace(id, false).first->second; + } + + pClass = (pClass->m_pBaseClasses == nullptr) ? nullptr : pClass->m_pBaseClasses->m_pClass; + } while (pClass != nullptr); } const auto has_non_standard_layout_field = @@ -1041,7 +1051,7 @@ namespace { const bool is_standard_layout_class = IsStandardLayoutClass(cache.class_has_standard_layout, class_); // TODO: when we have a CLI parser: allow users to generate assertions in non-standard-layout classes. Those assertions are - // conditionally-supported. + // conditionally-supported by compilers. if (is_standard_layout_class) { for (const auto& field : class_.GetFields() | std::ranges::views::filter([&](const auto& e) { return !skipped_fields.contains(e.m_pszName); })) { From cafa466e01d95255b34eb0a81400c5eb21c3ee51 Mon Sep 17 00:00:00 2001 From: Cre3per Date: Tue, 27 Aug 2024 16:28:07 +0200 Subject: [PATCH 36/46] empty classes remain empty --- src/sdk/sdk.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sdk/sdk.cpp b/src/sdk/sdk.cpp index 4550045..4305f10 100644 --- a/src/sdk/sdk.cpp +++ b/src/sdk/sdk.cpp @@ -211,7 +211,9 @@ namespace { const auto* pClass = &class_; int classes_with_fields = 0; do { - classes_with_fields += (pClass->m_nFieldSize != 0) ? 1 : 0; + // also check size because not all members are registered with + // the schema system. + classes_with_fields += ((pClass->m_nSizeOf > 1) || (pClass->m_nFieldSize != 0)) ? 1 : 0; if (classes_with_fields > 1) { return cache.emplace(id, false).first->second; @@ -719,7 +721,6 @@ namespace { builder.begin_bitfield_block(expected_offset); for (const auto& entry : state.bitfield) { - // TOOD: duplicate code for (const auto& field_metadata : entry.metadata) { if (auto data = GetMetadataValue(field_metadata); data.empty()) builder.comment(std::format("metadata: {}", field_metadata.m_szName)); @@ -975,7 +976,10 @@ namespace { const auto last_field_end = state.last_field_offset.value_or(0) + state.last_field_size.value_or(0); const auto end_pad = class_size - last_field_end; - if (end_pad != 0) { + // The `(class_size != 1)` check is here because of empty classes. If + // we generated a pad for empty classes, they'd no longer have standard-layout. + // The pad isn't necessary for such classes, because the compiler will make them have size=1. + if ((end_pad != 0) && (class_size != 1)) { builder.struct_padding(last_field_end, end_pad, true, true); } else if (end_pad < 0) [[unlikely]] { throw std::runtime_error{std::format("{} overflows by {:#x} byte(s). Its last field ends at {:#x}, but {} ends at {:#x}", class_.GetName(), From bdcd7e35d9e14995b0e833146384056536e810ad Mon Sep 17 00:00:00 2001 From: Cre3per Date: Tue, 27 Aug 2024 16:52:09 +0200 Subject: [PATCH 37/46] add conan to generated sdk --- sdk-static/CMakeLists.txt | 8 +- sdk-static/conanfile.py | 51 +++++++ .../source2sdk/source2gen_user_types.hpp | 138 ------------------ 3 files changed, 57 insertions(+), 140 deletions(-) create mode 100644 sdk-static/conanfile.py delete mode 100644 sdk-static/source2sdk/source2gen_user_types.hpp diff --git a/sdk-static/CMakeLists.txt b/sdk-static/CMakeLists.txt index f62962b..174b823 100644 --- a/sdk-static/CMakeLists.txt +++ b/sdk-static/CMakeLists.txt @@ -1,5 +1,3 @@ -# TOOD: try using in a playground - cmake_minimum_required(VERSION 3.30) set(CMAKE_EXPORT_COMPILE_COMMANDS On) @@ -12,9 +10,15 @@ file(GLOB_RECURSE source2sdk_headers "./**.hpp") add_library(${PROJECT_NAME} INTERFACE) target_include_directories(${PROJECT_NAME} INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) +set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${source2sdk_headers}") set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) +install( + DIRECTORY "${CMAKE_SOURCE_DIR}/include" + DESTINATION . +) + # Add a target that includes all headers of the generated library to check for compile-time errors foreach(el ${source2sdk_headers}) diff --git a/sdk-static/conanfile.py b/sdk-static/conanfile.py new file mode 100644 index 0000000..46956e0 --- /dev/null +++ b/sdk-static/conanfile.py @@ -0,0 +1,51 @@ +from conan import ConanFile +from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps + + +# TODO: We should set `version` and `name` to reflect what game this sdk is for +class source2sdkRecipe(ConanFile): + name = "source2sdk" + version = "0.0.0" + package_type = "library" + + author = "source2gen" + url = "https://github.com/neverlosecc/source2gen" + description = "Source2 SDK" + topics = ("source2") + + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + exports_sources = "CMakeLists.txt", "include/*" + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self) + + def generate(self): + deps = CMakeDeps(self) + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.bindirs = [] + self.cpp_info.libdirs = [] + diff --git a/sdk-static/source2sdk/source2gen_user_types.hpp b/sdk-static/source2sdk/source2gen_user_types.hpp deleted file mode 100644 index fc7ee65..0000000 --- a/sdk-static/source2sdk/source2gen_user_types.hpp +++ /dev/null @@ -1,138 +0,0 @@ -#pragma once - -#include -#include - -template -using CAnimValue = char[0x08]; -using CAnimVariant = char[0x11]; -// size is a guess -template -using CAnimScriptParam = char[0x08]; -using CBufferString = char[0x10]; -using CColorGradient = char[0x18]; -// size doesn't mapper. only used as a pointer -template -using CCompressor = char[0x01]; -using CEntityHandle = char[0x04]; -using CEntityIndex = char[0x04]; -using CGlobalSymbol = char[0x08]; -using CKV3MemberNameWithStorage = char[0x38]; -using CNetworkedQuantizedFloat = char[0x08]; -using CParticleNamedValueRef = char[0x40]; -using CPiecewiseCurve = char[0x40]; -using CPlayerSlot = char[0x04]; -using CPulseValueFullType = char[0x10]; -using CResourceName = char[0xe0]; -using CSplitScreenSlot = char[0x04]; -using CTransform = char[0x20]; -using CUtlBinaryBlock = char[0x18]; -template -using CUtlHashtable = char[0x20]; -using CUtlStringTokenWithStorage = char[0x18]; -using CUtlStringToken = char[0x04]; -using CUtlString = char[0x08]; -using CUtlSymbolLarge = char[0x08]; -using CUtlSymbol = char[0x02]; -template -using CAnimGraphParamOptionalRef = char[0x20]; -template -using CAnimGraphParamRef = char[0x20]; -template -using CBitVec = char[(N + 7) / 8]; -template -using CEntityOutputTemplate = char[0x28]; -template -using CHandle = char[0x04]; -template -using C_NetworkUtlVectorBase = char[0x18]; -template -using CNetworkUtlVectorBase = char[0x18]; -// size unknown. only used in dynamic containers. -using CSoundEventName = char[0x01]; -template -using CUtlLeanVector = char[0x10]; -template -using CUtlOrderedMap = char[0x28]; -// size doesn't mapper. only used as a pointer -template -using CUtlPair = char[0x01]; -template -using CUtlVector = char[0x18]; -// size is a guess that fits both occurences of this type in CS2 -template -using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; -template -using CUtlLeanVectorFixedGrowable = char[0x10 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; -template -using C_UtlVectorEmbeddedNetworkVar = char[0x50]; -template -using CUtlVectorEmbeddedNetworkVar = char[0x50]; -using CUtlVectorSIMDPaddedVector = char[0x18]; -template -using CSmartPtr = char[0x08]; -template -using CResourceArray = char[0x08]; -// size unknown -using CResourceString = char[0x08]; -template -using CResourcePointer = char[0x08]; -template -using CResourceNameTyped = char[0xe0]; -template -using CStrongHandle = char[0x08]; -template -using CStrongHandleCopyable = char[0x08]; -// size doesn't mapper. only used as a pointer -using CStrongHandleVoid = char[0x08]; -template -using CVariantBase = char[0x10]; -template -using CWeakHandle = char[0x18]; -using Color = char[0x04]; -using DegreeEuler = char[0x0c]; -using FourVectors = char[0x30]; -using HSCRIPT = char[0x08]; -using KeyValues3 = char[0x10]; -// size doesn't mapper. only used as a pointer -using KeyValues = char[0x01]; -using QAngle = char[0x0c]; -using QuaternionStorage = char[0x10]; -using Quaternion = char[0x10]; -using RadianEuler = char[0x0c]; -using RenderInputLayoutField_t = char[0x04]; -// we don't have a field size for this type. uses the fallback of 1. -using RenderPrimitiveType_t = char[0x01]; -using RotationVector = char[0x0c]; -template -using SphereBase_t = char[0x10]; -using Vector2D = char[0x08]; -using Vector4D = char[0x10]; -using VectorAligned = char[0x10]; -using Vector = char[0x0c]; -using WorldGroupId_t = char[0x04]; -using float32 = char[0x04]; -using fltx4 = char[0x10]; -using matrix3x4_t = char[0x30]; -using matrix3x4a_t = char[0x30]; - -// intentionally left undefined. if you want to access static fields, add your own sdk. -namespace interfaces { - struct SchemaStaticFieldData_t { - void* m_pInstance{}; - }; - - struct CSchemaClassInfo { - auto GetStaticFields() -> SchemaStaticFieldData_t**; - }; - - struct CSchemaSystemTypeScope { - auto FindDeclaredClass(std::string_view) -> CSchemaClassInfo*; - }; - - struct schema_t { - auto FindTypeScopeForModule(std::string_view) -> CSchemaSystemTypeScope*; - }; - - extern schema_t* g_schema; -} // namespace interfaces From ab0dde9f479b9597f5b4f8941934ab6b9c3f40d5 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Mon, 9 Sep 2024 21:39:37 +0200 Subject: [PATCH 38/46] build: fix compilation on windows --- source2gen/include/tools/util.h | 2 +- source2gen/src/sdk/sdk.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source2gen/include/tools/util.h b/source2gen/include/tools/util.h index 519c068..51d9f3c 100644 --- a/source2gen/include/tools/util.h +++ b/source2gen/include/tools/util.h @@ -29,7 +29,7 @@ namespace util { /// const std::optional offset = try_get_offset(); /// std::cout << std::format("offset: {}\n", offset.transform(to_hex_string).value_or("unknown")); /// ``` - inline std::string to_hex_string(int i) { + inline std::string to_hex_string(const std::uintptr_t i) { return std::format("{:#x}", i); } diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index d23256f..25b64cf 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -1088,7 +1088,7 @@ namespace { } void GenerateEnumSdk(std::string_view module_name, const CSchemaEnumBinding& enum_) { - const std::string out_file_path = GetFilePathForType(module_name, enum_.m_pszName); + const auto out_file_path = GetFilePathForType(module_name, enum_.m_pszName).string(); // @note: @es3n1n: init codegen // @@ -1129,7 +1129,7 @@ namespace { } void GenerateClassSdk(sdk::GeneratorCache& cache, std::string_view module_name, const CSchemaClassBinding& class_) { - const std::string out_file_path = GetFilePathForType(module_name, class_.m_pszName); + const auto out_file_path = GetFilePathForType(module_name, class_.m_pszName).string(); // @note: @es3n1n: init codegen // From 80cc781cf1e2c656d9869011b61d08ae62fd40e4 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 18:02:30 +0200 Subject: [PATCH 39/46] refactor: cleanup code a bit --- source2gen/src/sdk/sdk.cpp | 60 ++++++++++++++------------------------ 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index 25b64cf..566f059 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -42,8 +42,8 @@ namespace { }; struct ClassAssemblyState { - std::optional last_field_size = std::nullopt; - std::optional last_field_offset = std::nullopt; + std::optional last_field_size = std::nullopt; + std::optional last_field_offset = std::nullopt; bool assembling_bitfield = false; std::vector bitfield = {}; std::int32_t bitfield_start = 0; @@ -51,8 +51,6 @@ namespace { std::ptrdiff_t collision_end_offset = 0ull; // @fixme: @es3n1n: todo proper collision fix and remove this var }; - using namespace std::string_view_literals; - /** * Project structure is * @@ -66,10 +64,6 @@ namespace { constexpr std::string_view kOutDirName = "sdk"; constexpr std::string_view kIncludeDirName = "source2sdk"; - constexpr uint32_t kMaxReferencesForClassEmbed = 2; - constexpr std::size_t kMinFieldCountForClassEmbed = 2; - constexpr std::size_t kMaxFieldCountForClassEmbed = 12; - constinit std::array string_metadata_entries = { FNV32("MCellForDomain"), FNV32("MCustomFGDMetadata"), @@ -158,22 +152,18 @@ namespace { const auto value_hash_name = fnv32::hash_runtime(metadata_entry.m_szName); - // clang-format off - if (std::ranges::find(var_name_string_class_metadata_entries, value_hash_name) != var_name_string_class_metadata_entries.end()) - { - const auto &var_value = metadata_entry.m_pNetworkValue->m_VarValue; + if (std::ranges::find(var_name_string_class_metadata_entries, value_hash_name) != var_name_string_class_metadata_entries.end()) { + const auto& var_value = metadata_entry.m_pNetworkValue->m_VarValue; if (var_value.m_pszType && var_value.m_pszName) value = std::format("{} {}", var_value.m_pszType, var_value.m_pszName); else if (var_value.m_pszName && !var_value.m_pszType) value = var_value.m_pszName; else if (!var_value.m_pszName && var_value.m_pszType) value = var_value.m_pszType; - } - else if (std::ranges::find(string_class_metadata_entries, value_hash_name) != string_class_metadata_entries.end()) - { + } else if (std::ranges::find(string_class_metadata_entries, value_hash_name) != string_class_metadata_entries.end()) { auto clean_string = [](const std::string_view& input) { std::string result; - for (const char &ch : input) { + for (const char& ch : input) { if (std::isalpha(static_cast(ch))) { result += ch; } else { @@ -184,17 +174,16 @@ namespace { }; value = clean_string(metadata_entry.m_pNetworkValue->m_szValue.data()); - } - else if (std::ranges::find(string_metadata_entries, value_hash_name) != string_metadata_entries.end()) + } else if (std::ranges::find(string_metadata_entries, value_hash_name) != string_metadata_entries.end()) { value = metadata_entry.m_pNetworkValue->m_pszValue; - else if (std::ranges::find(integer_metadata_entries, value_hash_name) != integer_metadata_entries.end()) + } else if (std::ranges::find(integer_metadata_entries, value_hash_name) != integer_metadata_entries.end()) { value = std::to_string(metadata_entry.m_pNetworkValue->m_nValue); - else if (std::ranges::find(float_metadata_entries, value_hash_name) != float_metadata_entries.end()) + } else if (std::ranges::find(float_metadata_entries, value_hash_name) != float_metadata_entries.end()) { value = std::to_string(metadata_entry.m_pNetworkValue->m_fValue); - // clang-format on + } return value; - }; + } /// https://en.cppreference.com/w/cpp/language/classes#Standard-layout_class /// Doesn't check for all requirements, but is strict enough for what we are doing. @@ -460,7 +449,7 @@ namespace { } return {base_type, sizes}; - }; + } /// @return Lifetime is bound to string viewed by @p type_name [[nodiscard]] @@ -499,7 +488,7 @@ namespace { } else { return std::nullopt; } - }; + } [[nodiscard]] std::string EscapeTypeName(const std::string_view type_name) { // TODO: when we have a package manager: use a library @@ -526,7 +515,7 @@ namespace { return GetModuleOfTypeInScope(scope, type_name) .transform([&](const auto module_name) { return std::format("{}::{}", module_name, escaped_type_name); }) .value_or(escaped_type_name); - }; + } /// Decomposes a templated type into its components, keeping template /// syntax for later reassembly by @ref ReassembleRetypedTemplate(). @@ -536,13 +525,13 @@ namespace { std::vector> DecomposeTemplate(std::string_view type_name) { // TODO: use a library for this once we have a package manager const auto trim = [](std::string_view str) { - if (const auto found = str.find_first_not_of(" "); found != std::string_view::npos) { + if (const auto found = str.find_first_not_of(' '); found != std::string_view::npos) { str.remove_prefix(found); } else { return std::string_view{}; } - if (const auto found = str.find_last_not_of(" "); found != std::string_view::npos) { + if (const auto found = str.find_last_not_of(' '); found != std::string_view::npos) { str.remove_suffix(str.size() - (found + 1)); } @@ -584,7 +573,7 @@ namespace { // remove the topmost type and all syntax entries for (const auto& el : DecomposeTemplate(type_name)) { if (std::holds_alternative(el)) { - result.emplace_back(std::move(std::get(el))); + result.emplace_back(std::get(el)); } } @@ -631,7 +620,7 @@ namespace { return {type_name_with_modules, array_sizes}; return {type_name_with_modules, {}}; - }; + } // We assume that everything that is not a pointer is odr-used. // This assumption not correct, e.g. template classes that internally store pointers are @@ -711,7 +700,7 @@ namespace { } [[nodiscard]] - ClassAssemblyState AssembleBitfield(codegen::generator_t& builder, ClassAssemblyState&& state, int expected_offset) { + ClassAssemblyState AssembleBitfield(codegen::generator_t& builder, ClassAssemblyState&& state, std::ptrdiff_t expected_offset) { state.assembling_bitfield = false; std::size_t exact_bitfield_size_bits = 0; @@ -766,7 +755,7 @@ namespace { static constexpr std::size_t source2_max_align = 8; // TODO: when we have a CLI parser: pass this property in from the outside - const bool verbose = false; + constexpr bool verbose = false; struct cached_datamap_t { std::string type_; @@ -826,11 +815,6 @@ namespace { const std::optional first_field_offset = (first_field != nullptr) ? std::make_optional(first_field->m_nSingleInheritanceOffset) : std::nullopt; - const auto class_size_without_parent = class_.m_nSizeOf - parent_class_size.value_or(0); - - const auto expected_pad_size = - first_field_offset.transform([&](auto e) { return e - parent_class_size.value_or(0); }).value_or(class_size_without_parent); - // @todo: @es3n1n: if for some mysterious reason this class describes fields // of the base class we should handle it too. if ((class_parent != nullptr) && first_field_offset.has_value() && first_field_offset.value() < parent_class_size.value()) { @@ -954,7 +938,7 @@ namespace { builder.comment("", false).reset_tabs_count().prop(var_info.m_type, var_info.formatted_name(), false).restore_tabs_count(); } - if (verbose) { + if constexpr (verbose) { builder.reset_tabs_count() .comment(std::format("type.name=\"{}\" offset={:#x} size={:#x} alignment={}", std::string_view{field.m_pSchemaType->m_pszName}, field.m_nSingleInheritanceOffset, field_size, @@ -1020,7 +1004,7 @@ namespace { if (t->GetFieldName().empty()) continue; - const auto var_info = field_parser::parse(t->m_iFieldType, t->GetFieldName().data(), t->m_nFieldSize); + const auto var_info = field_parser::parse(t->m_iFieldType, t->GetFieldName(), t->m_nFieldSize); std::string field_type = var_info.m_type; if (t->m_iFieldType == fieldtype_t::FIELD_EMBEDDED) { From 17b0d5f286882088d22840bc574d7a0f0e9dd4bd Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 18:17:27 +0200 Subject: [PATCH 40/46] docs: add BitfieldEntry lifetime doc --- source2gen/include/sdk/interfaces/schemasystem/schema.h | 5 +++-- source2gen/src/sdk/sdk.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source2gen/include/sdk/interfaces/schemasystem/schema.h b/source2gen/include/sdk/interfaces/schemasystem/schema.h index 256248e..14773a8 100644 --- a/source2gen/include/sdk/interfaces/schemasystem/schema.h +++ b/source2gen/include/sdk/interfaces/schemasystem/schema.h @@ -657,8 +657,9 @@ struct SchemaClassInfoData_t { std::uint8_t m_unAlignOf; // 0x0022 std::int8_t m_nBaseClassSize; // 0x0023 - std::int16_t - m_nMultipleInheritanceDepth; // 0x0024 // @note: @og: if there is no derived or base class, then it will be 1 otherwise derived class size + 1. + + // @note: @og: if there is no derived or base class, then it will be 1 otherwise derived class size + 1. + std::int16_t m_nMultipleInheritanceDepth; // 0x0024 std::int16_t m_nSingleInheritanceDepth; // 0x0026 SchemaClassFieldData_t* m_pFields; // 0x0028 diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index 566f059..f02ee77 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -37,7 +37,7 @@ namespace { struct BitfieldEntry { std::string name{}; std::size_t size{}; - // TOOD: document lifetime - asked es3n1n + /// Lifetime of fields' pointers bound to the source2's @ref CSchemaClassInfo std::vector metadata{}; }; From 7166122c84ce38a30271d31fdcccf9e267d19b33 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 19:22:02 +0200 Subject: [PATCH 41/46] refactor: rename source2gen_user_types to source2gen --- README.md | 6 +++--- sdk-static/conanfile.py | 2 +- .../{source2gen_user_types.hpp => source2gen.hpp} | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) rename sdk-static/include/source2sdk/{source2gen_user_types.hpp => source2gen.hpp} (99%) diff --git a/README.md b/README.md index 8d0b402..87e45bb 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,11 @@ Linux. ### Using the generated SDK -The sdk depends on a file/module called "source2gen_user_types". This file has +The sdk depends on a file/module called `source2gen.hpp`. This file has to be provided by the user and expose all types listed in -[source2gen_user_types.hpp](sdk-static/source2gen_user_types.hpp). If you don't +[source2gen.hpp](sdk-static/include/source2sdk/source2gen.hpp). If you don't intend to access any of these types, you can use the dummy file -[source2gen_user_types.hpp](sdk-static/source2gen_user_types.hpp). +[source2gen.hpp](sdk-static/include/source2sdk/source2gen.hpp). ## Limitations diff --git a/sdk-static/conanfile.py b/sdk-static/conanfile.py index 46956e0..60d309a 100644 --- a/sdk-static/conanfile.py +++ b/sdk-static/conanfile.py @@ -11,7 +11,7 @@ class source2sdkRecipe(ConanFile): author = "source2gen" url = "https://github.com/neverlosecc/source2gen" description = "Source2 SDK" - topics = ("source2") + topics = ("source2",) settings = "os", "compiler", "build_type", "arch" options = {"shared": [True, False], "fPIC": [True, False]} diff --git a/sdk-static/include/source2sdk/source2gen_user_types.hpp b/sdk-static/include/source2sdk/source2gen.hpp similarity index 99% rename from sdk-static/include/source2sdk/source2gen_user_types.hpp rename to sdk-static/include/source2sdk/source2gen.hpp index fc7ee65..66d7084 100644 --- a/sdk-static/include/source2sdk/source2gen_user_types.hpp +++ b/sdk-static/include/source2sdk/source2gen.hpp @@ -1,5 +1,4 @@ #pragma once - #include #include From 0d7647157982044ca23743a9e22b3176481927b1 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 19:35:04 +0200 Subject: [PATCH 42/46] refactor(codegen): minor refactoring --- source2gen/include/tools/codegen.h | 38 +++++++++++++++++++----------- source2gen/src/sdk/sdk.cpp | 25 +++++++++++--------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/source2gen/include/tools/codegen.h b/source2gen/include/tools/codegen.h index 600924a..9dbb5fa 100644 --- a/source2gen/include/tools/codegen.h +++ b/source2gen/include/tools/codegen.h @@ -1,8 +1,8 @@ // Copyright (C) 2024 neverlosecc // See end of file for extended copyright information. #pragma once -#include #include +#include #include #include #include @@ -12,8 +12,9 @@ #include "tools/fnv.h" namespace codegen { - constexpr char kTabSym = '\t'; - constexpr std::size_t kTabsPerBlock = 1; // @note: @es3n1n: how many \t characters shall we place per each block + constexpr char kSpaceSym = ' '; + constexpr char kIndentWidth = 4; + constexpr std::size_t kTabsPerBlock = 1; // @note: @es3n1n: how many (kSpaceSym * kIndentWidth) characters shall we place per each block constexpr std::array kBlacklistedCharacters = {':', ';', '\\', '/'}; struct generator_t { @@ -56,6 +57,14 @@ namespace codegen { return pragma("warning(pop)"); } + self_ref pack_push(const std::size_t alignment = 1) { + return pragma(std::format("pack(push, {})", alignment)); + } + + self_ref pack_pop() { + return pragma("pack(pop)"); + } + self_ref next_line() { return push_line(""); } @@ -77,8 +86,9 @@ namespace codegen { // @note: @es3n1n: we should reset tabs count if we aren't moving cursor to // the next line const auto backup_tabs_count = _tabs_count; - if (!move_cursor_to_next_line) + if (!move_cursor_to_next_line) { _tabs_count = 0; + } push_line("{", move_cursor_to_next_line); @@ -157,7 +167,7 @@ namespace codegen { // @todo: @es3n1n: add func params self_ref begin_function(const std::string& prefix, const std::string& type_name, const std::string& func_name, const bool increment_tabs_count = true, const bool move_cursor_to_next_line = true) { - return begin_block(std::format("{}{} {}()", prefix, type_name, escape_name(func_name)), "", increment_tabs_count, move_cursor_to_next_line); + return begin_block(std::format("{}{} {}() ", prefix, type_name, escape_name(func_name)), "", increment_tabs_count, move_cursor_to_next_line); } self_ref end_function(const bool decrement_tabs_count = true, const bool move_cursor_to_next_line = true) { @@ -189,14 +199,13 @@ namespace codegen { return *this; } - self_ref static_assert_size(std::string_view type_name, int expected_size, const bool move_cursor_to_next_line = true) { + self_ref static_assert_size(std::string_view type_name, const std::size_t expected_size) { assert(expected_size > 0); return push_line(std::format("static_assert(sizeof({}) == {:#x});", escape_name(type_name), expected_size)); } - self_ref static_assert_offset(std::string_view class_name, std::string_view prop_name, int expected_offset, - const bool move_cursor_to_next_line = true) { + self_ref static_assert_offset(std::string_view class_name, std::string_view prop_name, const std::size_t expected_offset) { assert(expected_offset >= 0); return push_line(std::format("static_assert(offsetof({}, {}) == {:#x});", escape_name(class_name), prop_name, expected_offset)); @@ -246,7 +255,7 @@ namespace codegen { if (is_private_field) type_name = "[[maybe_unused]] " + type_name; - auto pad_name = pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value()) : std::format("pad_{:d}", _pads_count++); + auto pad_name = pad_offset.has_value() ? std::format("pad_{:#04x}", pad_offset.value()) : std::format("pad_{:d}", _pads_count++); if (bytes != 0) { prop(type_name, std::format("{}[{:#x}]", pad_name, bytes), move_cursor_to_next_line); @@ -254,7 +263,7 @@ namespace codegen { if (remaining_bits != 0) { auto remainder_pad_name = - pad_offset.has_value() ? std::format("pad_0x{:04x}", pad_offset.value() + bytes) : std::format("pad_{:d}", _pads_count++); + pad_offset.has_value() ? std::format("pad_{:#04x}", pad_offset.value() + bytes) : std::format("pad_{:d}", _pads_count++); prop(type_name, std::format("{}: {}", remainder_pad_name, remaining_bits), move_cursor_to_next_line); } @@ -287,11 +296,12 @@ namespace codegen { private: self_ref push_line(const std::string& line, bool move_cursor_to_next_line = true) { - for (std::size_t i = 0; i < _tabs_count; i++) - _stream << kTabSym; - _stream << line; - if (move_cursor_to_next_line) + _stream << std::string(_tabs_count * kIndentWidth, kSpaceSym) // insert spaces + << line; + + if (move_cursor_to_next_line) { _stream << std::endl; + } return *this; } diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index f02ee77..f2847ec 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -825,8 +825,8 @@ namespace { state.collision_end_offset = parent_class_size.value(); } - // @note: @es3n1n: start class - // + /// Start the class + builder.pack_push(1); // we are aligning stuff ourselves if (is_struct) builder.begin_struct_with_base_type(class_.m_pszName, parent_class_name); else @@ -1038,6 +1038,8 @@ namespace { builder.comment("No schema binary for binding"); builder.end_block(); + builder.pack_pop(); + builder.next_line(); const bool is_standard_layout_class = IsStandardLayoutClass(cache.class_has_standard_layout, class_); @@ -1126,18 +1128,11 @@ namespace { builder.include(util::EscapePath(std::format("\"{}/{}/{}.hpp\"", kIncludeDirName, include.module, include.type_name))); } - builder.include(std::format("\"{}/source2gen_user_types.hpp\"", kIncludeDirName)); + builder.include(std::format("\"{}/source2gen.hpp\"", kIncludeDirName)); builder.include(""); // for offsetof() builder.include(""); - for (const auto& forward_declaration : names | std::views::filter([](const auto& el) { return el.source == NameSource::forward_declaration; })) { - builder.begin_namespace(std::format("source2sdk::{}", forward_declaration.module)); - builder.forward_declaration(forward_declaration.type_name); - builder.end_namespace(); - } - - // @note: @es3n1n: print banner - // + /// print banner builder.next_line() .comment("/////////////////////////////////////////////////////////////") .comment(std::format("Module: {}", module_name)) @@ -1145,6 +1140,14 @@ namespace { .comment("/////////////////////////////////////////////////////////////") .next_line(); + /// Insert forward declarations + for (const auto& forward_declaration : names | std::views::filter([](const auto& el) { return el.source == NameSource::forward_declaration; })) { + builder.begin_namespace(std::format("source2sdk::{}", forward_declaration.module)); + builder.forward_declaration(forward_declaration.type_name); + builder.end_namespace(); + builder.next_line(); + } + builder.begin_namespace(std::format("source2sdk::{}", module_name)); // @note: @es3n1n: assemble props From 6b9c071e9c6b358068f299f7d2ccc7ea7a70108c Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 19:39:42 +0200 Subject: [PATCH 43/46] feat(static): add smart prop attributes --- sdk-static/include/source2sdk/source2gen.hpp | 70 ++++++++++++-------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/sdk-static/include/source2sdk/source2gen.hpp b/sdk-static/include/source2sdk/source2gen.hpp index 66d7084..9b96177 100644 --- a/sdk-static/include/source2sdk/source2gen.hpp +++ b/sdk-static/include/source2sdk/source2gen.hpp @@ -2,16 +2,16 @@ #include #include -template +template using CAnimValue = char[0x08]; using CAnimVariant = char[0x11]; // size is a guess -template +template using CAnimScriptParam = char[0x08]; using CBufferString = char[0x10]; using CColorGradient = char[0x18]; // size doesn't mapper. only used as a pointer -template +template using CCompressor = char[0x01]; using CEntityHandle = char[0x04]; using CEntityIndex = char[0x04]; @@ -26,67 +26,67 @@ using CResourceName = char[0xe0]; using CSplitScreenSlot = char[0x04]; using CTransform = char[0x20]; using CUtlBinaryBlock = char[0x18]; -template +template using CUtlHashtable = char[0x20]; using CUtlStringTokenWithStorage = char[0x18]; using CUtlStringToken = char[0x04]; using CUtlString = char[0x08]; using CUtlSymbolLarge = char[0x08]; using CUtlSymbol = char[0x02]; -template +template using CAnimGraphParamOptionalRef = char[0x20]; -template +template using CAnimGraphParamRef = char[0x20]; template using CBitVec = char[(N + 7) / 8]; -template +template using CEntityOutputTemplate = char[0x28]; -template +template using CHandle = char[0x04]; -template +template using C_NetworkUtlVectorBase = char[0x18]; -template +template using CNetworkUtlVectorBase = char[0x18]; // size unknown. only used in dynamic containers. using CSoundEventName = char[0x01]; -template +template using CUtlLeanVector = char[0x10]; -template +template using CUtlOrderedMap = char[0x28]; // size doesn't mapper. only used as a pointer -template +template using CUtlPair = char[0x01]; -template +template using CUtlVector = char[0x18]; // size is a guess that fits both occurences of this type in CS2 -template -using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; -template -using CUtlLeanVectorFixedGrowable = char[0x10 + ((sizeof(T) < 4) ? 4 : sizeof(T))]; -template +template +using CUtlVectorFixedGrowable = char[0x18 + ((sizeof(Ty) < 4) ? 4 : sizeof(Ty))]; +template +using CUtlLeanVectorFixedGrowable = char[0x10 + ((sizeof(Ty) < 4) ? 4 : sizeof(Ty))]; +template using C_UtlVectorEmbeddedNetworkVar = char[0x50]; -template +template using CUtlVectorEmbeddedNetworkVar = char[0x50]; using CUtlVectorSIMDPaddedVector = char[0x18]; -template +template using CSmartPtr = char[0x08]; -template +template using CResourceArray = char[0x08]; // size unknown using CResourceString = char[0x08]; -template +template using CResourcePointer = char[0x08]; -template +template using CResourceNameTyped = char[0xe0]; -template +template using CStrongHandle = char[0x08]; -template +template using CStrongHandleCopyable = char[0x08]; // size doesn't mapper. only used as a pointer using CStrongHandleVoid = char[0x08]; -template +template using CVariantBase = char[0x10]; -template +template using CWeakHandle = char[0x18]; using Color = char[0x04]; using DegreeEuler = char[0x0c]; @@ -103,7 +103,7 @@ using RenderInputLayoutField_t = char[0x04]; // we don't have a field size for this type. uses the fallback of 1. using RenderPrimitiveType_t = char[0x01]; using RotationVector = char[0x0c]; -template +template using SphereBase_t = char[0x10]; using Vector2D = char[0x08]; using Vector4D = char[0x10]; @@ -114,6 +114,18 @@ using float32 = char[0x04]; using fltx4 = char[0x10]; using matrix3x4_t = char[0x30]; using matrix3x4a_t = char[0x30]; +using CSmartPropAttributeVector = char[0x40]; +using CSmartPropAttributeFloat = char[0x40]; +using CSmartPropAttributeBool = char[0x40]; +using CSmartPropAttributeColor = char[0x40]; +using CSmartPropAttributeInt = char[0x40]; +using CSmartPropAttributeModelName = char[0x40]; +using CSmartPropAttributeMaterialGroup = char[0x40]; +using CSmartPropAttributeVector2D = char[0x40]; +using CSmartPropVariableComparison = char[0x20]; +using CSmartPropAttributeAngles = char[0x40]; +using CSmartPropAttributeStateName = char[0x40]; +using CSmartPropAttributeVariableValue = char[0x40]; // intentionally left undefined. if you want to access static fields, add your own sdk. namespace interfaces { From 15d1eb14e89cf33c1fa33846e3d5a6ef575103d6 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 19:50:40 +0200 Subject: [PATCH 44/46] refactor(static): move stuff around --- sdk-static/include/source2sdk/source2gen.hpp | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/sdk-static/include/source2sdk/source2gen.hpp b/sdk-static/include/source2sdk/source2gen.hpp index 9b96177..539bf9b 100644 --- a/sdk-static/include/source2sdk/source2gen.hpp +++ b/sdk-static/include/source2sdk/source2gen.hpp @@ -10,7 +10,7 @@ template using CAnimScriptParam = char[0x08]; using CBufferString = char[0x10]; using CColorGradient = char[0x18]; -// size doesn't mapper. only used as a pointer +// size doesn't matter. only used as a pointer template using CCompressor = char[0x01]; using CEntityHandle = char[0x04]; @@ -26,7 +26,7 @@ using CResourceName = char[0xe0]; using CSplitScreenSlot = char[0x04]; using CTransform = char[0x20]; using CUtlBinaryBlock = char[0x18]; -template +template using CUtlHashtable = char[0x20]; using CUtlStringTokenWithStorage = char[0x18]; using CUtlStringToken = char[0x04]; @@ -53,7 +53,7 @@ template using CUtlLeanVector = char[0x10]; template using CUtlOrderedMap = char[0x28]; -// size doesn't mapper. only used as a pointer +// size doesn't matter. only used as a pointer template using CUtlPair = char[0x01]; template @@ -82,18 +82,30 @@ template using CStrongHandle = char[0x08]; template using CStrongHandleCopyable = char[0x08]; -// size doesn't mapper. only used as a pointer +// size doesn't matter. only used as a pointer using CStrongHandleVoid = char[0x08]; template using CVariantBase = char[0x10]; template using CWeakHandle = char[0x18]; +using CSmartPropAttributeVector = char[0x40]; +using CSmartPropAttributeFloat = char[0x40]; +using CSmartPropAttributeBool = char[0x40]; +using CSmartPropAttributeColor = char[0x40]; +using CSmartPropAttributeInt = char[0x40]; +using CSmartPropAttributeModelName = char[0x40]; +using CSmartPropAttributeMaterialGroup = char[0x40]; +using CSmartPropAttributeVector2D = char[0x40]; +using CSmartPropVariableComparison = char[0x20]; +using CSmartPropAttributeAngles = char[0x40]; +using CSmartPropAttributeStateName = char[0x40]; +using CSmartPropAttributeVariableValue = char[0x40]; using Color = char[0x04]; using DegreeEuler = char[0x0c]; using FourVectors = char[0x30]; using HSCRIPT = char[0x08]; using KeyValues3 = char[0x10]; -// size doesn't mapper. only used as a pointer +// size doesn't matter. only used as a pointer using KeyValues = char[0x01]; using QAngle = char[0x0c]; using QuaternionStorage = char[0x10]; @@ -114,18 +126,6 @@ using float32 = char[0x04]; using fltx4 = char[0x10]; using matrix3x4_t = char[0x30]; using matrix3x4a_t = char[0x30]; -using CSmartPropAttributeVector = char[0x40]; -using CSmartPropAttributeFloat = char[0x40]; -using CSmartPropAttributeBool = char[0x40]; -using CSmartPropAttributeColor = char[0x40]; -using CSmartPropAttributeInt = char[0x40]; -using CSmartPropAttributeModelName = char[0x40]; -using CSmartPropAttributeMaterialGroup = char[0x40]; -using CSmartPropAttributeVector2D = char[0x40]; -using CSmartPropVariableComparison = char[0x20]; -using CSmartPropAttributeAngles = char[0x40]; -using CSmartPropAttributeStateName = char[0x40]; -using CSmartPropAttributeVariableValue = char[0x40]; // intentionally left undefined. if you want to access static fields, add your own sdk. namespace interfaces { From 9257deb68b2e3f5c50280d3811be70ee52ef9605 Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 21:07:38 +0200 Subject: [PATCH 45/46] fix: check m_pClassInfo for nullptr --- source2gen/src/sdk/sdk.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index f2847ec..8b0824c 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -212,16 +212,16 @@ namespace { } while (pClass != nullptr); } - const auto has_non_standard_layout_field = - std::ranges::any_of(class_.GetFields() | std::ranges::views::transform([&](const SchemaClassFieldData_t& e) { - if (const auto* e_class = e.m_pSchemaType->GetAsDeclaredClass(); e_class != nullptr) { - return !IsStandardLayoutClass(cache, *e_class->m_pClassInfo); - } else { - // Everything that is not a class has no effect - return false; - } - }), - std::identity{}); + const auto has_non_standard_layout_field = std::ranges::any_of( + class_.GetFields() | std::ranges::views::transform([&](const SchemaClassFieldData_t& e) { + if (const auto* e_class = e.m_pSchemaType->GetAsDeclaredClass(); e_class != nullptr && e_class->m_pClassInfo != nullptr) { + return !IsStandardLayoutClass(cache, *e_class->m_pClassInfo); + } else { + // Everything that is not a class has no effect + return false; + } + }), + std::identity{}); if (has_non_standard_layout_field) { return cache.emplace(id, false).first->second; @@ -281,7 +281,7 @@ namespace { /// @return For class types, returns @ref GetClassAlignmentRecursive(). Otherwise returns the immediately available size. [[nodiscard]] std::optional GetAlignmentOfTypeRecursive(std::map>& cache, const CSchemaType& type) { - if (const auto* class_ = type.GetAsDeclaredClass(); class_ != nullptr) { + if (const auto* class_ = type.GetAsDeclaredClass(); class_ != nullptr && class_->m_pClassInfo != nullptr) { return GetClassAlignmentRecursive(cache, *class_->m_pClassInfo); } else { return type.GetSizeAndAlignment().and_then([](const auto& e) { return std::get<1>(e); }); From 30743e0647e1d764a4a3199e6f93439f076107ec Mon Sep 17 00:00:00 2001 From: Arsenii es3n1n Date: Tue, 10 Sep 2024 21:43:12 +0200 Subject: [PATCH 46/46] fix: handle fields with no m_pClassInfo --- source2gen/src/sdk/sdk.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source2gen/src/sdk/sdk.cpp b/source2gen/src/sdk/sdk.cpp index 8b0824c..6954fb0 100644 --- a/source2gen/src/sdk/sdk.cpp +++ b/source2gen/src/sdk/sdk.cpp @@ -847,7 +847,7 @@ namespace { // @note: @es3n1n: parsing type // const auto [type_name, array_sizes] = GetType(*field.m_pSchemaType); - const auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); + auto var_info = field_parser::parse(type_name, field.m_pszName, array_sizes); // @fixme: @es3n1n: todo proper collision fix and remove this block if (state.collision_end_offset && field.m_nSingleInheritanceOffset < state.collision_end_offset) { @@ -905,8 +905,16 @@ namespace { state.last_field_offset = field.m_nSingleInheritanceOffset; state.last_field_size = static_cast(field_size); - // @note: @es3n1n: push prop - // + /// @note: @es3n1n: game bug: + /// There are some classes that have literally no info about them in schema, + /// for these fields we'll just insert a pad. + if (const auto e_class = field.m_pSchemaType->GetAsDeclaredClass(); e_class != nullptr && e_class->m_pClassInfo == nullptr) { + var_info.m_type = "std::uint8_t"; + var_info.m_array_sizes.clear(); + var_info.m_array_sizes.emplace_back(field_size); + builder.comment(std::format("game bug: prop with no declared class info ({})", e_class->m_pszName)); + } + if ((field.m_nSingleInheritanceOffset % field_alignment.value_or(source2_max_align)) == 0) { if (std::string{field.m_pSchemaType->m_pszName}.contains('<')) { // This is a workaround to get the size of template types right.