Skip to content

Commit

Permalink
minicargo - New build system seems to be working
Browse files Browse the repository at this point in the history
  • Loading branch information
@thepowersgang
thepowersgang committed Sep 14, 2023
1 parent ad82eac commit e2a1257
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 33 deletions.
10 changes: 7 additions & 3 deletions tools/minicargo/build.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1051,6 +1051,7 @@ ::helpers::path Builder::build_build_script(const PackageManifest& manifest, boo
{
args.push_back("-g");
}
args.push_back("-O");
for(const auto& d : m_opts.lib_search_dirs)
{
args.push_back("-L");
Expand Down Expand Up @@ -1258,18 +1259,21 @@ bool spawn_process(const char* exe_name, const StringList& args, const StringLis
// - Put the exe name in the first arg
// - Update the executable name

bool rv = false;
try {
os_support::Process::spawn(exe_name, args, env, logfile, working_directory).wait();
return true;
rv = os_support::Process::spawn(exe_name, args, env, logfile, working_directory).wait();
}
catch(...)
{
}
if(!rv)
{
::std::cerr << "FAILING COMMAND: " << exe_name;
for(const auto& p : args.get_vec())
if (p != nullptr)
::std::cerr << " " << p;
::std::cerr << ::std::endl;
//::std::cerr << "See " << logfile << " for the compiler output" << ::std::endl;
return false;
}
return rv;
}
51 changes: 31 additions & 20 deletions tools/minicargo/build2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
# define EXESUF ""
# define DLLSUF ".so"
#endif
#include <target_detect.h> // tools/common/target_detect.h
#define HOST_TARGET DEFAULT_TARGET_NAME
#include <target_detect.h> // tools/common/target_detect.h
#define HOST_TARGET DEFAULT_TARGET_NAME

struct RunState
{
Expand Down Expand Up @@ -191,7 +191,7 @@ BuildList::BuildList(const PackageManifest& manifest, const BuildOptions& opts):
{
TRACE_FUNCTION_F(p.name());
// If this is a proc macro, force `is_native`
if(p.has_library() && p.get_library().m_crate_types == PackageTarget::CrateType::proc_macro ) {
if(p.has_library() && p.get_library().m_is_proc_macro ) {
is_native = true;
}
// If the package is already loaded
Expand Down Expand Up @@ -322,7 +322,7 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)
// Generate jobs for a build script
// Returns the job name (or an empty string, if no job generated)
// Populates the build script path
std::string handle_build_script(RunState& run_state, const PackageManifest& p, const helpers::path& build_script_overrides, helpers::path& build_script)
std::string handle_build_script(RunState& run_state, const PackageManifest& p, const helpers::path& build_script_overrides, helpers::path& build_script, bool is_host)
{
if( p.build_script() != "" )
{
Expand All @@ -332,6 +332,7 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)
// TODO: Should this test if it exists? or just assume and let it error?

build_script = override_file;
const_cast<PackageManifest&>(p).load_build_script( build_script.str() );
return "";
}
else
Expand All @@ -358,16 +359,19 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)
// HACK: Search for `-mmir/` in the output, remove it, and if that exists copy it to here
// - This grabs the last non-mmir execution of the script
auto tmp_out = build_script.str();
auto p = tmp_out.rfind("-mmir/");
if( p != std::string::npos )
auto mmir_pos = tmp_out.rfind("-mmir/");
if( mmir_pos != std::string::npos )
{
auto src = tmp_out.substr(0, p) + tmp_out.substr(p+5);
auto src = tmp_out.substr(0, mmir_pos) + tmp_out.substr(mmir_pos+5);
std::ifstream ifs(src);
if( ifs.good() )
{
std::cout << "HACK: Copying " << src << " to " << tmp_out << std::endl;
::std::ofstream ofs(tmp_out);
ofs << ifs.rdbuf();
{
::std::ofstream ofs(tmp_out);
ofs << ifs.rdbuf();
}
const_cast<PackageManifest&>(p).load_build_script( build_script.str() );
return name_bs_build;
}
}
Expand All @@ -376,9 +380,21 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)

auto output_ts = Timestamp::for_file(build_script);
this->handle_dep(job_bs_run->m_dependencies, output_ts, name_bs_build);
p.iter_main_dependencies([&](const PackageRef& dep) {
if( !dep.is_disabled() )
{
auto k = run_state.get_key(dep.get_package(), false, is_host);
bs_is_dirty |= this->handle_dep(job_bs_run->m_dependencies, output_ts, k);
}
});
bool bs_needs_run = bs_is_dirty || output_ts < script_ts;
auto rv = bs_needs_run ? job_bs_run->name() : ::std::string();
this->add_job(std::move(job_bs_run), output_ts, bs_needs_run);
// If the script is not being run, then it still needs to be loaded
if(!bs_needs_run)
{
const_cast<PackageManifest&>(p).load_build_script( build_script.str() );
}
return rv;
}
}
Expand All @@ -398,7 +414,7 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)
auto output_ts = Timestamp::for_file(job->get_outfile());
bool is_dirty = run_state.outfile_needs_rebuild(job->get_outfile());
// Handle build script
auto bs_job_name = convert_state.handle_build_script(run_state, p, opts.build_script_overrides, job->m_build_script);
auto bs_job_name = convert_state.handle_build_script(run_state, p, opts.build_script_overrides, job->m_build_script, e.is_host);
if( bs_job_name != "" ) {
job->m_dependencies.push_back(bs_job_name);
is_dirty = true;
Expand All @@ -418,7 +434,7 @@ bool BuildList::build(BuildOptions opts, unsigned num_jobs)
helpers::path build_script;
if( !m_root_manifest.has_library() )
{
bs_job_name = convert_state.handle_build_script(run_state, m_root_manifest, opts.build_script_overrides, build_script);
bs_job_name = convert_state.handle_build_script(run_state, m_root_manifest, opts.build_script_overrides, build_script, !cross_compiling);
}

auto push_root_target = [&](const PackageTarget& target) {
Expand Down Expand Up @@ -636,7 +652,7 @@ bool RunState::outfile_needs_rebuild(const helpers::path& outfile) const
if( ts_result < dep_ts )
{
has_new_file = true;
DEBUG("Rebuilding " << outfile << ", older than " << f);
DEBUG("Rebuilding " << outfile << ", older than " << f << " (" << ts_result << " < " << dep_ts << ")");
break;
}
}
Expand Down Expand Up @@ -769,11 +785,6 @@ helpers::path Job_BuildTarget::get_outfile() const
}
RunnableJob Job_BuildTarget::start()
{
if( m_build_script.is_valid() )
{
const_cast<PackageManifest&>(m_manifest).load_build_script( m_build_script.str() );
}

const char* crate_type;
::std::string crate_suffix;
auto outfile = parent.get_crate_path(m_manifest, m_target, m_is_for_host, &crate_type, &crate_suffix);
Expand Down Expand Up @@ -947,7 +958,7 @@ RunnableJob Job_RunScript::start()
env.push_back(fn, "1");
}
//env.push_back("CARGO_CFG_RELEASE", "");
env.push_back("OUT_DIR", out_dir);
env.push_back("OUT_DIR", out_dir.to_absolute());

push_env_common(env, m_manifest);

Expand All @@ -970,7 +981,7 @@ RunnableJob Job_RunScript::start()
else
{
m_script_exe_abs = ::std::move(script_exe_abs);
return RunnableJob(m_script_exe_abs.str().c_str(), {}, std::move(env), ::std::move(out_file), m_manifest.directory());
return RunnableJob(m_script_exe_abs.str().c_str(), {}, std::move(env), out_file.to_absolute(), m_manifest.directory());
}
}
bool Job_RunScript::complete(bool was_success)
Expand All @@ -979,7 +990,7 @@ bool Job_RunScript::complete(bool was_success)
if(was_success)
{
// TODO: Parse the script here? Or just keep the parsing in the downstream build
//const_cast<PackageManifest&>(m_manifest).load_build_script( out_file.str() );
const_cast<PackageManifest&>(m_manifest).load_build_script( out_file.str() );
return true;
}
else
Expand Down
34 changes: 30 additions & 4 deletions tools/minicargo/jobs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,36 @@ bool JobList::run_all()
auto total_job_count = this->waiting_jobs.size();
bool failed = false;
size_t num_complete = 0;
bool force_wait = false;

auto dump_state = [&]() {
::std::cerr
<< " ("
<< std::fixed << std::setprecision(1) << (100 * static_cast<double>(num_complete) / total_job_count) << "% "
<< this->running_jobs.size()+1 << "r," << this->runnable_jobs.size() << "w," << this->waiting_jobs.size() << "b/" << total_job_count << "t"
<< "):"
;
for(const auto& rj : this->running_jobs) {
if(&rj != &this->running_jobs.front())
::std::cerr << ",";
::std::cerr << " " << rj.job->name();
}
::std::cerr << "\n";
};

while( !this->waiting_jobs.empty() || !this->runnable_jobs.empty() || !this->running_jobs.empty() )
{
// Wait until a running job stops
while( this->running_jobs.size() >= this->num_jobs )
while( (force_wait || this->running_jobs.size() >= this->num_jobs) && !this->running_jobs.empty() )
{
if( !wait_one() ) {
dump_state();
failed = true;
break;
}
dump_state();
num_complete += 1;
force_wait = false;
}
if(failed) {
break;
Expand All @@ -51,9 +71,7 @@ bool JobList::run_all()
// Update the runnable list.
for(auto& slot : this->waiting_jobs)
{
if(!slot) {
continue;
}
assert(slot);
const auto& deps = slot->dependencies();
if( std::all_of(deps.begin(), deps.end(), [&](const std::string& s){ return completed_jobs.count(s) > 0; }) && slot->is_runnable() )
{
Expand All @@ -62,14 +80,19 @@ bool JobList::run_all()
}
auto new_end = std::remove_if(waiting_jobs.begin(), waiting_jobs.end(), [](const job_t& j){ return !j; });
waiting_jobs.erase(new_end, waiting_jobs.end());

// Is nothing runnable?
if( this->runnable_jobs.empty() ) {
// Is nothing running?
if( this->running_jobs.empty()) {
// BUG if there are jobs on the queue
if( !this->waiting_jobs.empty() ) {
::std::cerr << "BUG: Nothing runnable or running, but jobs are still waiting\n";
}
break ;
}
else {
force_wait = true;
continue ;
}
}
Expand All @@ -90,10 +113,12 @@ bool JobList::run_all()

auto handle = this->spawn(rjob);
this->running_jobs.push_back(RunningJob { handle, std::move(job), std::move(rjob) });
dump_state();
}
while( !this->running_jobs.empty() )
{
failed |= !wait_one();
dump_state();
num_complete += 1;
}
return !failed;
Expand Down Expand Up @@ -163,6 +188,7 @@ bool JobList::wait_one()
}
else
{
::std::cout << "Completed " << rjob.job->name() << std::endl;
this->completed_jobs.insert(rjob.job->name());
}
rv &= rjob.job->complete(rv);
Expand Down
Loading

0 comments on commit e2a1257

Please sign in to comment.