Skip to content

Commit

Permalink
Merge pull request #316 from thepowersgang/minicargo_job_system
Browse files Browse the repository at this point in the history 
Replace threads with a job manager in minicargo
  • Loading branch information
@thepowersgang
thepowersgang authored Sep 23, 2023
2 parents c70b86e + ca1096a commit 0728995
Show file tree
Hide file tree
Showing 25 changed files with 2,041 additions and 1,265 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ src/main.cpp: $(PCHS:%=src/%.gch)
@echo [CXX] -o $@
$V$(CXX) -std=c++14 -o $@ $< $(CPPFLAGS) -MMD -MP -MF $@.dep

bin/common_lib.a:
bin/common_lib.a: $(wildcard tools/common/*)
$(MAKE) -C tools/common

-include $(OBJ:%=%.dep)
Expand Down
55 changes: 33 additions & 22 deletions minicargo.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,13 @@ endif

OUTDIR := output$(OUTDIR_SUF)/

USE_MERGED_BUILD=1
ifeq ($(RUSTC_VERSION),1.19.0)
RUSTC_OUT_BIN := rustc
USE_MERGED_BUILD=0
else ifeq ($(RUSTC_VERSION),1.29.0)
RUSTC_OUT_BIN := rustc_binary
USE_MERGED_BUILD=0
else ifeq ($(RUSTC_VERSION),1.39.0)
RUSTC_OUT_BIN := rustc_binary
else
Expand Down Expand Up @@ -151,8 +154,6 @@ all: $(OUTDIR)rustc

test: $(OUTDIR)rust/test_run-pass_hello_out.txt

LIBS: $(OUTDIR)libstd.rlib $(OUTDIR)libtest.rlib $(OUTDIR)libpanic_unwind.rlib $(OUTDIR)libproc_macro.rlib

RUSTCSRC: $(RUSTC_SRC_DL)

.PHONY: rust_tests local_tests
Expand Down Expand Up @@ -190,26 +191,36 @@ $(RUSTC_SRC_DL): $(RUSTC_SRC_TARBALL) rustc-$(RUSTC_VERSION)-src.patch
# Standard library crates
# - libstd, libpanic_unwind, libtest and libgetopts
# - libproc_macro (mrustc)
$(OUTDIR)libstd.rlib: $(MRUSTC) $(MINICARGO) $(RUSTC_SRC_DL)
$(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)std --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS)
@test -e $@
$(OUTDIR)libpanic_unwind.rlib: $(OUTDIR)libstd.rlib
$(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)panic_unwind --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS)
@test -e $@
$(OUTDIR)libtest.rlib: $(OUTDIR)libstd.rlib $(OUTDIR)libpanic_unwind.rlib
$(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)test --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS)
@test -e $@
# MRustC custom version of libproc_macro
$(OUTDIR)libproc_macro.rlib: $(OUTDIR)libstd.rlib
$(MINICARGO) lib/libproc_macro --output-dir $(OUTDIR) $(MINICARGO_FLAGS)
@test -e $@
ifeq ($(USE_MERGED_BUILD),1)
$(RUSTCSRC)mrustc-stdlib/Cargo.toml: $(RUSTC_SRC_DL) minicargo.mk
@mkdir -p $(dir $@)
@echo "#![no_core]" > $(dir $@)/lib.rs
@echo "[package]" > $@
@echo "name = \"mrustc_standard_library\"" >> $@
@echo "version = \"0.0.0\"" >> $@
@echo "[lib]" >> $@
@echo "path = \"lib.rs\"" >> $@
@echo "[dependencies]" >> $@
@echo "std = { path = \"../$(RUST_LIB_PREFIX)std\" }" >> $@
@echo "panic_unwind = { path = \"../$(RUST_LIB_PREFIX)panic_unwind\" }" >> $@
@echo "test = { path = \"../$(RUST_LIB_PREFIX)test\" }" >> $@
LIBS: $(RUSTCSRC)mrustc-stdlib/Cargo.toml $(MRUSTC) $(MINICARGO)
+$(MINICARGO) --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) $(RUSTCSRC)mrustc-stdlib/
+$(MINICARGO) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) lib/libproc_macro
else
LIBS: $(MRUSTC) $(MINICARGO) $(RUSTC_SRC_DL)
+$(MINICARGO) --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) $(RUSTCSRC)$(RUST_LIB_PREFIX)std
+$(MINICARGO) --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) $(RUSTCSRC)$(RUST_LIB_PREFIX)panic_unwind
+$(MINICARGO) --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) $(RUSTCSRC)$(RUST_LIB_PREFIX)test
+$(MINICARGO) --output-dir $(OUTDIR) $(MINICARGO_FLAGS) lib/libproc_macro
endif

# Dynamically linked version of the standard library
$(OUTDIR)test/libtest.so: $(RUSTC_SRC_DL)
mkdir -p $(dir $@)
MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)std --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)panic_unwind --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)test --vendor-dir $(VENDOR_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
+MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)std --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
+MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)panic_unwind --vendor-dir $(VENDOR_DIR) --script-overrides $(OVERRIDE_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
+MINICARGO_DYLIB=1 $(MINICARGO) $(RUSTCSRC)$(RUST_LIB_PREFIX)test --vendor-dir $(VENDOR_DIR) --output-dir $(dir $@) $(MINICARGO_FLAGS)
test -e $@

RUSTC_ENV_VARS := CFG_COMPILER_HOST_TRIPLE=$(RUSTC_TARGET)
Expand All @@ -226,14 +237,14 @@ RUSTC_ENV_VARS += MRUSTC_LIBDIR=$(abspath $(OUTDIR))

$(OUTDIR)rustc: $(MRUSTC) $(MINICARGO) LIBS $(LLVM_CONFIG)
mkdir -p $(OUTDIR)rustc-build
$(RUSTC_ENV_VARS) $(MINICARGO) $(RUSTCSRC)$(SRCDIR_RUSTC) --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)rustc-build -L $(OUTDIR) $(MINICARGO_FLAGS) $(MINICARGO_FLAGS_$@)
+$(RUSTC_ENV_VARS) $(MINICARGO) $(RUSTCSRC)$(SRCDIR_RUSTC) --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)rustc-build -L $(OUTDIR) $(MINICARGO_FLAGS) $(MINICARGO_FLAGS_$@)
test -e $@ -a ! $(OUTDIR)rustc-build/$(RUSTC_OUT_BIN) -nt $@ || cp $(OUTDIR)rustc-build/$(RUSTC_OUT_BIN) $@
$(OUTDIR)rustc-build/librustc_driver.rlib: $(MRUSTC) $(MINICARGO) LIBS
mkdir -p $(OUTDIR)rustc-build
$(RUSTC_ENV_VARS) $(MINICARGO) $(RUSTCSRC)$(SRCDIR_RUSTC_DRIVER) --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)rustc-build -L $(OUTDIR) $(MINICARGO_FLAGS) $(MINICARGO_FLAGS_$(OUTDIR)rustc)
+$(RUSTC_ENV_VARS) $(MINICARGO) $(RUSTCSRC)$(SRCDIR_RUSTC_DRIVER) --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)rustc-build -L $(OUTDIR) $(MINICARGO_FLAGS) $(MINICARGO_FLAGS_$(OUTDIR)rustc)
$(OUTDIR)cargo: $(MRUSTC) LIBS
mkdir -p $(OUTDIR)cargo-build
MRUSTC_LIBDIR=$(abspath $(OUTDIR)) $(MINICARGO) $(RUSTCSRC)src/tools/cargo --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)cargo-build -L $(OUTDIR) $(MINICARGO_FLAGS)
+MRUSTC_LIBDIR=$(abspath $(OUTDIR)) $(MINICARGO) $(RUSTCSRC)src/tools/cargo --vendor-dir $(VENDOR_DIR) --output-dir $(OUTDIR)cargo-build -L $(OUTDIR) $(MINICARGO_FLAGS)
test -e $@ -a ! $(OUTDIR)cargo-build/cargo -nt $@ || cp $(OUTDIR)cargo-build/cargo $@

# Reference $(RUSTCSRC)src/bootstrap/native.rs for these values
Expand Down Expand Up @@ -386,7 +397,7 @@ RUNTIME_ARGS_$(OUTDIR)stdtest/collectionstests += --skip ::vec::overaligned_allo
#ENV_$(OUTDIR)stdtest/rustc-test += CFG_COMPILER_HOST_TRIPLE=$(RUSTC_TARGET)

$(OUTDIR)stdtest/%-test: $(RUSTCSRC)src/lib%/lib.rs LIBS
MRUSTC_LIBDIR=$(abspath $(OUTDIR)) $(MINICARGO) --test $(RUSTCSRC)src/lib$* --vendor-dir $(VENDOR_DIR) --output-dir $(dir $@) -L $(OUTDIR)
+MRUSTC_LIBDIR=$(abspath $(OUTDIR)) $(MINICARGO) --test $(RUSTCSRC)src/lib$* --vendor-dir $(VENDOR_DIR) --output-dir $(dir $@) -L $(OUTDIR)
$(OUTDIR)stdtest/collectionstests: $(OUTDIR)stdtest/alloc-test
test -e $@
$(OUTDIR)collectionstest_out.txt: $(OUTDIR)%
Expand Down
12 changes: 7 additions & 5 deletions src/mir/borrow_check.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,11 @@ namespace {
return vs_static;
}
}
throw "";
}
VarState& get_state_root_mut(const MIR::LValue::Storage& lv_root) {
TU_MATCH_HDRA( (lv_root), {)
TU_ARMA(Return, e)
TU_ARMA(Return, e)
return retval;
TU_ARMA(Local, e)
return locals.at(e);
Expand All @@ -96,6 +97,7 @@ namespace {
return it->second;
}
}
throw "";
}
const VarState& get_state(const ::MIR::TypeResolve& state, const MIR::LValue& lv) const {
const VarState* rv = &this->get_state_root(lv.m_root);
Expand Down Expand Up @@ -425,7 +427,7 @@ void MIR_BorrowCheck(const StaticTraitResolve& resolve, const ::HIR::ItemPath& p
{
const ::MIR::TypeResolve& state;
BorrowState& borrow_state;
V(const ::MIR::TypeResolve state, BorrowState& borrow_state)
V(const ::MIR::TypeResolve& state, BorrowState& borrow_state)
: state(state)
, borrow_state(borrow_state)
{}
Expand Down Expand Up @@ -550,9 +552,9 @@ void MIR_BorrowCheck(const StaticTraitResolve& resolve, const ::HIR::ItemPath& p
const auto& enm = resolve.m_crate.get_enum_by_path(state.sp, rse.path.m_path);
MonomorphStatePtr ms(nullptr, &rse.path.m_params, nullptr);
HIR::TypeRef tmp;
auto maybe_monomorph = [&](const auto& ty)->const HIR::TypeRef& {
return resolve.monomorph_expand_opt(sp, tmp, ty, ms);
};
//auto maybe_monomorph = [&](const auto& ty)->const HIR::TypeRef& {
// return resolve.monomorph_expand_opt(sp, tmp, ty, ms);
//};
if( rse.vals.size() > 0 ) {
MIR_ASSERT(state, enm.m_data.is_Data(), "");
const auto& variants = enm.m_data.as_Data();
Expand Down
1 change: 1 addition & 0 deletions tools/common/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ OBJDIR := .obj/

BIN := ../../bin/common_lib.a
OBJS = toml.o path.o debug.o
OBJS += jobserver.o

CXXFLAGS := -Wall -std=c++14 -g -O2

Expand Down
6 changes: 3 additions & 3 deletions tools/common/debug.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

static int giIndentLevel = 0;
static const char* gsDebugPhase = "";
static bool gbDebugPhaseEnabled = false;
static bool gbEnableHeaders = false;
static ::std::set<::std::string> gmDisabledDebug;
#ifndef DISABLE_MULTITHREAD
Expand Down Expand Up @@ -48,14 +49,13 @@ void Debug_ProcessEnable(const char* e)
void Debug_SetPhase(const char* phase_name)
{
gsDebugPhase = phase_name;
gbDebugPhaseEnabled = gmDisabledDebug.find(gsDebugPhase) == gmDisabledDebug.end();
if( gbEnableHeaders )
::std::cout << phase_name << ": BEGIN" << ::std::endl;
}
bool Debug_IsEnabled()
{
if( gmDisabledDebug.find(gsDebugPhase) != gmDisabledDebug.end() )
return false;
return true;
return gbDebugPhaseEnabled;
}
void Debug_DisablePhase(const char* phase_name)
{
Expand Down
Loading

0 comments on commit 0728995

Please sign in to comment.