From 964c9cdb31f5b76189962af45180edfd21e2b943 Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 23 Oct 2023 21:56:19 +0200 Subject: [PATCH 1/5] vere: add retry logic to _king_curl_bytes If the curl fails or receives a non-2xx status code, it retries, backing off a little bit, until it either succeeds, or exceeds five attempts. --- pkg/vere/king.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/pkg/vere/king.c b/pkg/vere/king.c index 7446ae7199..74ff7f782d 100644 --- a/pkg/vere/king.c +++ b/pkg/vere/king.c @@ -251,6 +251,7 @@ _king_curl_bytes(c3_c* url_c, c3_w* len_w, c3_y** hun_y, c3_t veb_t) CURL *cul_u; CURLcode res_i; long cod_i; + c3_y try_y = 0; uv_buf_t buf_u = uv_buf_init(c3_malloc(1), 0); if ( !(cul_u = curl_easy_init()) ) { @@ -263,29 +264,34 @@ _king_curl_bytes(c3_c* url_c, c3_w* len_w, c3_y** hun_y, c3_t veb_t) curl_easy_setopt(cul_u, CURLOPT_WRITEFUNCTION, _king_curl_alloc); curl_easy_setopt(cul_u, CURLOPT_WRITEDATA, (void*)&buf_u); - res_i = curl_easy_perform(cul_u); - curl_easy_getinfo(cul_u, CURLINFO_RESPONSE_CODE, &cod_i); + while ( 5 > try_y ) { + sleep(try_y++); + res_i = curl_easy_perform(cul_u); + curl_easy_getinfo(cul_u, CURLINFO_RESPONSE_CODE, &cod_i); - // XX retry? - // - if ( CURLE_OK != res_i ) { - if ( veb_t ) { - u3l_log("curl: failed %s: %s", url_c, curl_easy_strerror(res_i)); + if ( CURLE_OK != res_i ) { + if ( veb_t ) { + u3l_log("curl: failed to fetch %s: %s", + url_c, curl_easy_strerror(res_i)); + } + ret_i = -1; } - ret_i = -1; - } - if ( 300 <= cod_i ) { - if ( veb_t ) { - u3l_log("curl: error %s: HTTP %ld", url_c, cod_i); + else if ( 300 <= cod_i ) { + if ( veb_t ) { + u3l_log("curl: error fetching %s: HTTP %ld", url_c, cod_i); + } + ret_i = -2; + } + else { + *len_w = buf_u.len; + *hun_y = (c3_y*)buf_u.base; + ret_i = 0; + break; } - ret_i = -2; } curl_easy_cleanup(cul_u); - *len_w = buf_u.len; - *hun_y = (c3_y*)buf_u.base; - return ret_i; } @@ -376,6 +382,7 @@ u3_king_next(c3_c* pac_c, c3_c** out_c) } // skip printfs on failed requests (/next is usually not present) + //REVIEW new retry logic means this case will take longer. make retries optional? // if ( _king_curl_bytes(url_c, &len_w, &hun_y, 0) ) { c3_free(url_c); From e5bbeed7295d6b838271d4c84ef3c4a4c9870832 Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 23 Oct 2023 22:13:00 +0200 Subject: [PATCH 2/5] vere: bring your own boot sequence Adds support for --prop-file, --prop-url and --prop-name, which let one augment the boot sequence of a ship with additional events. Back-ports the vere changes from urbit/urbit#5470 onto current vere. See that for additional details/history. This is a mostly straightforward, copy-paste port of the changes from that original PR, with the following exceptions: - Boot sequence construction now happens in _pier_boot_make. - We don't modify the way the default boot sequence is acquired (yet). We will want to follow up on the latter, eventually. The %autoprop "prop builder" that was included in #5470 should still operate fine and be useful for automating keeping certain kinds of props up to date. (Save for minimal kelvin compatability disrepair it has fallen into due to the flow of time.) --- pkg/c3/motes.h | 1 + pkg/vere/king.c | 51 ++++++++++++++++++++++++++++++++++++++++++++-- pkg/vere/main.c | 46 +++++++++++++++++++++++++++++++++++++++++ pkg/vere/pier.c | 54 ++++++++++++++++++++++++++++++++++++++++++++----- pkg/vere/vere.h | 12 ++++++++++- 5 files changed, 156 insertions(+), 8 deletions(-) diff --git a/pkg/c3/motes.h b/pkg/c3/motes.h index 0000d5c593..4241c7f09d 100644 --- a/pkg/c3/motes.h +++ b/pkg/c3/motes.h @@ -936,6 +936,7 @@ # define c3__pril c3_s4('p','r','i','l') # define c3__pro c3_s3('p','r','o') # define c3__prod c3_s4('p','r','o','d') +# define c3__prop c3_s4('p','r','o','p') # define c3__prof c3_s4('p','r','o','f') # define c3__prox c3_s4('p','r','o','x') # define c3__psdg c3_s4('p','s','d','g') diff --git a/pkg/vere/king.c b/pkg/vere/king.c index 74ff7f782d..44979102b3 100644 --- a/pkg/vere/king.c +++ b/pkg/vere/king.c @@ -90,6 +90,8 @@ void _king_doom(u3_noun doom); void _king_fake(u3_noun ship, u3_noun pill, u3_noun path); void _king_pier(u3_noun pier); +static u3_noun _king_get_atom(c3_c* url_c); + /* _king_defy_fate(): invalid fate */ void @@ -158,6 +160,49 @@ _king_boot(u3_noun bul) u3z(bul); } +/* _king_prop(): events from prop arguments +*/ +u3_noun +_king_prop() +{ + u3_noun mor = u3_nul; + while ( 0 != u3_Host.ops_u.vex_u ) { + u3_even* vex_u = u3_Host.ops_u.vex_u; + switch ( vex_u->kin_i ) { + case 1: { // file + u3_atom jam = u3m_file(vex_u->loc_c); + mor = u3nc(u3ke_cue(jam), mor); + } break; + + case 2: { // url + u3l_log("boot: downloading prop %s", vex_u->loc_c); + u3_atom jam = _king_get_atom(vex_u->loc_c); + mor = u3nc(u3ke_cue(jam), mor); + } break; + + case 3: { // name + //NOTE this implementation limits us to max 38 char prop names + c3_c url_c[80]; + sprintf(url_c, + //TODO should maybe respect ops_u.url_c + "https://bootstrap.urbit.org/props/" URBIT_VERSION "/%s.jam", + vex_u->loc_c); + u3l_log("boot: downloading prop %s", url_c); + u3_atom jam = _king_get_atom(url_c); + mor = u3nc(u3ke_cue(jam), mor); + } break; + + default: { + u3l_log("invalid prop source %d", vex_u->kin_i); + exit(1); + } + } + + u3_Host.ops_u.vex_u = vex_u->pre_u; + } + return mor; +} + /* _king_fake(): boot with fake keys */ void @@ -166,7 +211,8 @@ _king_fake(u3_noun ship, u3_noun pill, u3_noun path) // XX link properly // u3_noun vent = u3nc(c3__fake, u3k(ship)); - u3K.pir_u = u3_pier_boot(sag_w, ship, vent, pill, path, u3_none); + u3K.pir_u = u3_pier_boot(sag_w, ship, vent, pill, path, + u3_none, _king_prop()); } /* _king_come(): mine a comet under star (unit) @@ -201,7 +247,8 @@ _king_dawn(u3_noun feed, u3_noun pill, u3_noun path) u3_noun vent = u3_dawn_vent(u3k(ship), u3k(feed)); // XX link properly // - u3K.pir_u = u3_pier_boot(sag_w, u3k(ship), vent, pill, path, feed); + u3K.pir_u = u3_pier_boot(sag_w, u3k(ship), vent, pill, path, + feed, _king_prop()); // disable ivory slog printfs // diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 9a1680347c..9114cb4f74 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -215,6 +215,19 @@ _main_pier_run(c3_c* bin_c) return dir_c; } +/* _main_add_prop(): add a boot prop to u3_Host.ops_u.vex_u. +*/ +u3_even* +_main_add_prop(c3_i kin_i, c3_c* loc_c) +{ + u3_even* nex_u = c3_calloc(sizeof(*nex_u)); + nex_u->kin_i = kin_i; + nex_u->loc_c = loc_c; // XX _main_repath where appropriate? + nex_u->pre_u = u3_Host.ops_u.vex_u; + u3_Host.ops_u.vex_u = nex_u; + return nex_u; +} + /* _main_getopt(): extract option map from command line. */ static u3_noun @@ -269,6 +282,10 @@ _main_getopt(c3_i argc, c3_c** argv) { "scry-into", required_argument, NULL, 'Y' }, { "scry-format", required_argument, NULL, 'Z' }, // + { "prop-file", required_argument, NULL, 1 }, + { "prop-url", required_argument, NULL, 2 }, + { "prop-name", required_argument, NULL, 3 }, + // { "urth-loom", required_argument, NULL, 5 }, { "no-demand", no_argument, NULL, 6 }, { "swap", no_argument, NULL, 7 }, @@ -283,6 +300,10 @@ _main_getopt(c3_i argc, c3_c** argv) lop_u, &lid_i)) ) { switch ( ch_i ) { + case 1: case 2: case 3: { // prop-* + _main_add_prop(ch_i, strdup(optarg)); + break; + } case 5: { // urth-loom if (_main_readw_loom("urth-loom", &u3_Host.ops_u.lut_y)) { return c3n; @@ -591,6 +612,7 @@ _main_getopt(c3_i argc, c3_c** argv) if ( hyphen_c ) { *hyphen_c = '\0'; } + //TODO use brass pill from b.u.org/props/etc eventually c3_i res_i = asprintf(&u3_Host.ops_u.url_c, "https://bootstrap.urbit.org/urbit-v%s.pill", version_c); @@ -616,6 +638,18 @@ _main_getopt(c3_i argc, c3_c** argv) } } + if ( u3_Host.ops_u.vex_u != 0 ) { + struct stat s; + u3_even* vex_u = u3_Host.ops_u.vex_u; + while ( vex_u != 0 ) { + if ( vex_u->kin_i == 1 && stat(vex_u->loc_c, &s) != 0 ) { + fprintf(stderr, "events file %s not found\n", vex_u->loc_c); + return c3n; + } + vex_u = vex_u->pre_u; + } + } + struct sockaddr_in t; if ( u3_Host.ops_u.bin_c != 0 && inet_pton(AF_INET, u3_Host.ops_u.bin_c, &t.sin_addr) == 0 ) { fprintf(stderr, "-b invalid IP address\n"); @@ -630,6 +664,15 @@ _main_getopt(c3_i argc, c3_c** argv) } } + //TODO split up "default distribution" packages eventually + // // if we're not in lite mode, include the default props + // // + // if ( u3_Host.ops_u.lit == c3n ) { + // _main_add_prop(3, "landscape"); + // _main_add_prop(3, "webterm"); + // _main_add_prop(3, "groups"); + // } + return c3y; } @@ -790,6 +833,9 @@ u3_ve_usage(c3_i argc, c3_c** argv) " --no-dock Skip binary \"docking\" on boot\n", " --swap Use an explicit ephemeral (swap-like) file\n", " --swap-to FILE Specify ephemeral file location\n", + " --prop-file FILE Add a prop into the boot sequence\n" + " --prop-url URL Download a prop into the boot sequence\n", + " --prop-name NAME Download a prop from bootstrap.urbit.org\n", "\n", "Development Usage:\n", " To create a development ship, use a fakezod:\n", diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index bba434c4fd..b195291707 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -1820,7 +1820,8 @@ _pier_boot_make(u3_noun who, u3_noun wyr, u3_noun ven, u3_noun pil, - u3_weak fed) + u3_weak fed, + u3_noun mor) { u3_boot bot_u = _pier_pill_parse(pil); // transfer @@ -1872,7 +1873,48 @@ _pier_boot_make(u3_noun who, bot_u.use = u3nc(u3nc(wir, cad), bot_u.use); } + // prepend & append additional boot enhancements to the userspace sequence + // + { + u3_noun mos = mor; + u3_noun pre = u3_nul; + u3_noun aft = u3_nul; + while ( u3_nul != mos ) { + u3_noun mot = u3h(mos); + + switch ( u3h(mot) ) { + case c3__prop: { + u3_noun ter, met, ves; + + if ( c3n == u3r_trel(u3t(mot), &met, &ter, &ves) ) { + u3m_p("invalid prop", u3t(mot)); + break; + } + + if ( c3__fore == ter ) { + u3m_p("prop: fore", met); + pre = u3kb_weld(pre, u3k(ves)); + } + else if ( c3__hind == ter ) { + u3m_p("prop: hind", met); + aft = u3kb_weld(aft, u3k(ves)); + } + else { + u3m_p("unrecognized prop tier", ter); + } + } break; + + default: u3m_p("unrecognized boot sequence enhancement", u3h(mot)); + } + + mos = u3t(mos); + } + + bot_u.use = u3kb_weld(pre, u3kb_weld(bot_u.use, aft)); // transfer + } + u3z(fed); + u3z(mor); return bot_u; } @@ -1883,7 +1925,8 @@ _pier_boot_plan(u3_pier* pir_u, u3_noun who, u3_noun ven, u3_noun pil, - u3_weak fed) + u3_weak fed, + u3_noun mor) { u3_boot bot_u; { @@ -1891,7 +1934,7 @@ _pier_boot_plan(u3_pier* pir_u, pir_u->fak_o = ( c3__fake == u3h(ven) ) ? c3y : c3n; u3r_chubs(0, 2, pir_u->who_d, who); - bot_u = _pier_boot_make(who, _pier_wyrd_card(pir_u), ven, pil, fed); + bot_u = _pier_boot_make(who, _pier_wyrd_card(pir_u), ven, pil, fed, mor); pir_u->lif_w = u3qb_lent(bot_u.bot); } @@ -1967,7 +2010,8 @@ u3_pier_boot(c3_w wag_w, // config flags u3_noun ven, // boot event u3_noun pil, // type-of/path-to pill u3_noun pax, // path to pier - u3_weak fed) // extra private keys + u3_weak fed, // extra private keys + u3_noun mor) // extra boot sequence props { u3_pier* pir_u; @@ -1979,7 +2023,7 @@ u3_pier_boot(c3_w wag_w, // config flags // XX must be called from on_lord_live // - if ( c3n == _pier_boot_plan(pir_u, who, ven, pil, fed) ) { + if ( c3n == _pier_boot_plan(pir_u, who, ven, pil, fed, mor) ) { fprintf(stderr, "pier: boot plan fail\r\n"); // XX dispose // diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index b2b78b5e81..6361a1c3d2 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -258,6 +258,14 @@ struct _u3_auto* car_u; // driver hack } u3_utty; + /* u3_even: boot event specifier + */ + typedef struct _u3_even { + c3_i kin_i; + c3_c* loc_c; + struct _u3_even* pre_u; + } u3_even; + /* u3_opts: command line configuration. */ typedef struct _u3_opts { @@ -308,6 +316,7 @@ c3_o map; // --no-demand (reversed) c3_o eph; // --swap, use ephemeral file c3_o tos; // --toss, discard ephemeral + u3_even* vex_u; // --prop-*, boot enhancements } u3_opts; /* u3_host: entire host. @@ -1417,7 +1426,8 @@ u3_noun ven, // boot event u3_noun pil, // type-of/path-to pill u3_noun pax, // path to pier - u3_weak fed); // extra private keys + u3_weak fed, // extra private keys + u3_noun mor); // extra boot seq props /* u3_pier_stay(): restart the pier. */ From 232a5580e6a923def3316e332296273f09ed3f2e Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 16 Jan 2024 21:12:08 +0100 Subject: [PATCH 3/5] vere: include prop type definition --- pkg/vere/pier.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index b195291707..db1fc08b0f 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -1875,6 +1875,10 @@ _pier_boot_make(u3_noun who, // prepend & append additional boot enhancements to the userspace sequence // + // +$ prop [%prop meta tier (list ovum)] + // +$ meta term + // +$ tier ?(%fore %hind) :: before or after userspace + // { u3_noun mos = mor; u3_noun pre = u3_nul; From c686f91bcbfdafcfea375c5e1d489ad197e0cf21 Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 16 Jan 2024 21:13:28 +0100 Subject: [PATCH 4/5] vere: no need to call _main_repath here --- pkg/vere/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 9114cb4f74..6ac32a5587 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -222,7 +222,7 @@ _main_add_prop(c3_i kin_i, c3_c* loc_c) { u3_even* nex_u = c3_calloc(sizeof(*nex_u)); nex_u->kin_i = kin_i; - nex_u->loc_c = loc_c; // XX _main_repath where appropriate? + nex_u->loc_c = loc_c; nex_u->pre_u = u3_Host.ops_u.vex_u; u3_Host.ops_u.vex_u = nex_u; return nex_u; From 8b092267fe11b33afa02056bdf8251f741832737 Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 16 Jan 2024 21:38:13 +0100 Subject: [PATCH 5/5] vere: sprintf -> snprintf For safety. --- pkg/vere/king.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/vere/king.c b/pkg/vere/king.c index 44979102b3..66181c6f6a 100644 --- a/pkg/vere/king.c +++ b/pkg/vere/king.c @@ -181,12 +181,12 @@ _king_prop() } break; case 3: { // name - //NOTE this implementation limits us to max 38 char prop names - c3_c url_c[80]; - sprintf(url_c, - //TODO should maybe respect ops_u.url_c - "https://bootstrap.urbit.org/props/" URBIT_VERSION "/%s.jam", - vex_u->loc_c); + //NOTE this implementation limits us to max 213 char prop names + c3_c url_c[256]; + snprintf(url_c, 255, + //TODO should maybe respect ops_u.url_c + "https://bootstrap.urbit.org/props/" URBIT_VERSION "/%s.jam", + vex_u->loc_c); u3l_log("boot: downloading prop %s", url_c); u3_atom jam = _king_get_atom(url_c); mor = u3nc(u3ke_cue(jam), mor);