diff --git a/.github/workflows/next.yml b/.github/workflows/next.yml index e0dd8feeec..07250449e3 100644 --- a/.github/workflows/next.yml +++ b/.github/workflows/next.yml @@ -20,7 +20,3 @@ jobs: upload: true next: ${{ github.ref_name }} secrets: inherit - - docker: - uses: ./.github/workflows/docker-shared.yml - secrets: inherit diff --git a/.gitignore b/.gitignore index 251a0d7f0f..da5bf8e6c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# IDEs. +/.vscode +/.idea + # Bazel. /.user.bazelrc /bazel-* diff --git a/VERSION b/VERSION index 3e162f02ea..9f55b2ccb5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12 +3.0 diff --git a/flake.lock b/flake.lock index 12ffaf6f77..45c4d968fb 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1677383253, - "narHash": "sha256-UfpzWfSxkfXHnb4boXZNaKsAcUrZT9Hw+tao1oZxd08=", + "lastModified": 1683236849, + "narHash": "sha256-Y7PNBVLOBvZrmrFmHgXUBUA1lM72tl6JGIn1trOeuyE=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9952d6bc395f5841262b006fbace8dd7e143b634", + "rev": "374ffe54403c3c42d97a513ac7a14ce1b5b86e30", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1675933616, - "narHash": "sha256-/rczJkJHtx16IFxMmAWu5nNYcSXNg1YYXTHoGjLrLUA=", + "lastModified": 1682984683, + "narHash": "sha256-fSMthG+tp60AHhNmaHc4StT3ltfHkQsJtN8GhfLWmtI=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "47478a4a003e745402acf63be7f9a092d51b83d7", + "rev": "86684881e184f41aa322e653880e497b66429f3e", "type": "github" }, "original": { diff --git a/pkg/c3/defs.h b/pkg/c3/defs.h index 38b39263a9..bbf8d91ef7 100644 --- a/pkg/c3/defs.h +++ b/pkg/c3/defs.h @@ -149,6 +149,8 @@ mkdir(a, b);}) # define c3_rmdir(a) ({ \ rmdir(a);}) +# define c3_link(a, b) ({ \ + link(a, b);}) # define c3_unlink(a) ({ \ unlink(a);}) # define c3_fopen(a, b) ({ \ diff --git a/pkg/noun/events.c b/pkg/noun/events.c index dc1ba965d0..d4f6c92ed0 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -398,23 +398,14 @@ _ce_ephemeral_open(c3_i* eph_i) /* _ce_image_open(): open or create image. */ static c3_o -_ce_image_open(u3e_image* img_u) +_ce_image_open(u3e_image* img_u, c3_c* ful_c) { c3_i mod_i = O_RDWR | O_CREAT; - c3_c ful_c[8193]; - - snprintf(ful_c, 8192, "%s", u3P.dir_c); - c3_mkdir(ful_c, 0700); - - snprintf(ful_c, 8192, "%s/.urb", u3P.dir_c); - c3_mkdir(ful_c, 0700); - - snprintf(ful_c, 8192, "%s/.urb/chk", u3P.dir_c); - c3_mkdir(ful_c, 0700); - snprintf(ful_c, 8192, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c); - if ( -1 == (img_u->fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + c3_c pax_c[8192]; + snprintf(pax_c, 8192, "%s/%s.bin", ful_c, img_u->nam_c); + if ( -1 == (img_u->fid_i = c3_open(pax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", pax_c, strerror(errno)); return c3n; } else if ( c3n == _ce_image_stat(img_u, &img_u->pgs_w) ) { @@ -787,14 +778,16 @@ _ce_patch_sync(u3_ce_patch* pat_u) /* _ce_image_sync(): make sure image is synced to disk. */ -static void +static c3_o _ce_image_sync(u3e_image* img_u) { if ( -1 == c3_sync(img_u->fid_i) ) { fprintf(stderr, "loom: image (%s) sync failed: %s\r\n", img_u->nam_c, strerror(errno)); - u3_assert(!"loom: image sync"); + return c3n; } + + return c3y; } /* _ce_image_resize(): resize image, truncating if it shrunk. @@ -1345,54 +1338,77 @@ _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) return c3y; } -/* u3e_backup(): copy snapshot to .urb/bhk (if it doesn't exist yet). +/* u3e_backup(): copy snapshot from [pux_c] to [pax_c], + * overwriting optionally. note that image files must + * be named "north" and "south". */ c3_o -u3e_backup(c3_o ovw_o) +u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) { - u3e_image nop_u = { .nam_c = "north", .pgs_w = 0 }; - u3e_image sop_u = { .nam_c = "south", .pgs_w = 0 }; - c3_i mod_i = O_RDWR | O_CREAT; // XX O_TRUNC ? - c3_c ful_c[8193]; + // source image files from [pux_c] + u3e_image nux_u = { .nam_c = "north", .pgs_w = 0 }; + u3e_image sux_u = { .nam_c = "south", .pgs_w = 0 }; + + // destination image files to [pax_c] + u3e_image nax_u = { .nam_c = "north", .pgs_w = 0 }; + u3e_image sax_u = { .nam_c = "south", .pgs_w = 0 }; - snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); + c3_i mod_i = O_RDWR | O_CREAT; + + if ( !pux_c || !pax_c ) { + fprintf(stderr, "loom: image backup: bad path\r\n"); + return c3n; + } - if ( (c3n == ovw_o) && c3_mkdir(ful_c, 0700) ) { + if ( (c3n == ovw_o) && c3_mkdir(pax_c, 0700) ) { if ( EEXIST != errno ) { fprintf(stderr, "loom: image backup: %s\r\n", strerror(errno)); } return c3n; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); - - if ( -1 == (nop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + // open source image files if they exist + // + c3_c nux_c[8193]; + snprintf(nux_c, 8192, "%s/%s.bin", pux_c, nux_u.nam_c); + if ( (0 != access(nux_c, F_OK)) || (c3n == _ce_image_open(&nux_u, pux_c)) ) { + fprintf(stderr, "loom: couldn't open north image at %s\r\n", pux_c); + return c3n; + } + c3_c sux_c[8193]; + snprintf(sux_c, 8192, "%s/%s.bin", pux_c, sux_u.nam_c); + if ( (0 != access(sux_c, F_OK)) || (c3n == _ce_image_open(&sux_u, pux_c)) ) { + fprintf(stderr, "loom: couldn't open south image at %s\r\n", pux_c); return c3n; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, sop_u.nam_c); - - if ( -1 == (sop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + // open destination image files + c3_c nax_c[8193]; + snprintf(nax_c, 8192, "%s/%s.bin", pax_c, nax_u.nam_c); + if ( -1 == (nax_u.fid_i = c3_open(nax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", nax_c, strerror(errno)); + return c3n; + } + c3_c sax_c[8193]; + snprintf(sax_c, 8192, "%s/%s.bin", pax_c, sax_u.nam_c); + if ( -1 == (sax_u.fid_i = c3_open(sax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", sax_c, strerror(errno)); return c3n; } - if ( (c3n == _ce_image_copy(&u3P.nor_u, &nop_u)) - || (c3n == _ce_image_copy(&u3P.sou_u, &sop_u)) ) + if ( (c3n == _ce_image_copy(&nux_u, &nax_u)) + || (c3n == _ce_image_copy(&sux_u, &sax_u)) + || (c3n == _ce_image_sync(&nax_u)) + || (c3n == _ce_image_sync(&sax_u)) ) { - - c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); - c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); - c3_rmdir(ful_c); + c3_unlink(nax_c); + c3_unlink(sax_c); fprintf(stderr, "loom: image backup failed\r\n"); return c3n; } - close(nop_u.fid_i); - close(sop_u.fid_i); + close(nax_u.fid_i); + close(sax_u.fid_i); fprintf(stderr, "loom: image backup complete\r\n"); return c3y; } @@ -1462,8 +1478,9 @@ u3e_save(u3_post low_p, u3_post hig_p) _ce_patch_apply(pat_u); - _ce_image_sync(&u3P.nor_u); - _ce_image_sync(&u3P.sou_u); + u3_assert( c3y == _ce_image_sync(&u3P.nor_u) ); + u3_assert( c3y == _ce_image_sync(&u3P.sou_u) ); + _ce_patch_free(pat_u); _ce_patch_delete(); @@ -1502,8 +1519,6 @@ u3e_save(u3_post low_p, u3_post hig_p) } u3e_toss(low_p, hig_p); - - u3e_backup(c3n); } /* _ce_toss_pages(): discard ephemeral pages. @@ -1573,8 +1588,10 @@ u3e_live(c3_o nuu_o, c3_c* dir_c) // Open image files. // - if ( (c3n == _ce_image_open(&u3P.nor_u)) || - (c3n == _ce_image_open(&u3P.sou_u)) ) + c3_c chk_c[8193]; + snprintf(chk_c, 8193, "%s/.urb/chk", u3P.dir_c); + if ( (c3n == _ce_image_open(&u3P.nor_u, chk_c)) || + (c3n == _ce_image_open(&u3P.sou_u, chk_c)) ) { fprintf(stderr, "boot: image failed\r\n"); exit(1); @@ -1587,8 +1604,8 @@ u3e_live(c3_o nuu_o, c3_c* dir_c) */ if ( 0 != (pat_u = _ce_patch_open()) ) { _ce_patch_apply(pat_u); - _ce_image_sync(&u3P.nor_u); - _ce_image_sync(&u3P.sou_u); + u3_assert( c3y == _ce_image_sync(&u3P.nor_u) ); + u3_assert( c3y == _ce_image_sync(&u3P.sou_u) ); _ce_patch_free(pat_u); _ce_patch_delete(); } @@ -1659,6 +1676,9 @@ u3e_stop(void) close(u3P.eph_i); unlink(u3C.eph_c); } + + close(u3P.sou_u.fid_i); + close(u3P.sou_u.fid_i); } /* u3e_yolo(): disable dirty page tracking, read/write whole loom. @@ -1698,6 +1718,8 @@ u3e_init(void) { u3P.pag_w = u3C.wor_i >> u3a_page; + u3P.nor_u.fid_i = u3P.sou_u.fid_i = -1; + u3e_foul(); #ifdef U3_GUARD_PAGE diff --git a/pkg/noun/events.h b/pkg/noun/events.h index b17dcba65d..7442c9df94 100644 --- a/pkg/noun/events.h +++ b/pkg/noun/events.h @@ -74,10 +74,11 @@ /** Functions. **/ - /* u3e_backup(): copy the snapshot from chk to bhk. + /* u3e_backup(): copy the snapshot from [pux_c] to [pax_c], + * overwriting optional. */ - c3_o - u3e_backup(c3_o ovw_o); + c3_o + u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o); /* u3e_fault(): handle a memory fault. */ diff --git a/pkg/noun/jets/c/dor.c b/pkg/noun/jets/c/dor.c index dc463f7ae6..0c1e220afc 100644 --- a/pkg/noun/jets/c/dor.c +++ b/pkg/noun/jets/c/dor.c @@ -7,8 +7,8 @@ u3_noun - u3qc_dor(u3_atom a, - u3_atom b) + u3qc_dor(u3_noun a, + u3_noun b) { if ( c3y == u3r_sing(a, b) ) { return c3y; diff --git a/pkg/noun/jets/c/mor.c b/pkg/noun/jets/c/mor.c index 6967a8a1b6..97ba1410bd 100644 --- a/pkg/noun/jets/c/mor.c +++ b/pkg/noun/jets/c/mor.c @@ -7,8 +7,8 @@ u3_noun - u3qc_mor(u3_atom a, - u3_atom b) + u3qc_mor(u3_noun a, + u3_noun b) { c3_w c_w = u3r_mug(u3r_mug(a)); c3_w d_w = u3r_mug(u3r_mug(b)); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index c15e29ba8e..b0264327d1 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2224,11 +2224,8 @@ static u3j_core _139_qua_d[] = { "mole", 7, _140_qua_mole_a, 0, no_hashes }, { "mule", 7, _140_qua_mule_a, 0, no_hashes }, - // XX disabled, implicated in memory corruption - // write tests and re-enable - // - // { "scot", 7, _140_qua_scot_a, 0, no_hashes }, - // { "scow", 7, _140_qua_scow_a, 0, no_hashes }, + { "scot", 7, _140_qua_scot_a, 0, no_hashes }, + { "scow", 7, _140_qua_scow_a, 0, no_hashes }, { "slaw", 7, _140_qua_slaw_a, 0, no_hashes }, {} }; diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index bd2e62cb6f..2de555d58c 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -1766,14 +1766,6 @@ _cm_limits(void) # endif } -/* u3m_backup(): copy snapshot to .urb/bhk (if it doesn't exist yet). -*/ -c3_o -u3m_backup(c3_o ovw_o) -{ - return u3e_backup(ovw_o); -} - /* u3m_fault(): handle a memory event with libsigsegv protocol. */ c3_i @@ -2094,6 +2086,42 @@ u3m_stop() u3je_secp_stop(); } +/* u3m_pier(): make a pier. +*/ +c3_c* +u3m_pier(c3_c* dir_c) +{ + c3_c ful_c[8193]; + + u3C.dir_c = dir_c; + + snprintf(ful_c, 8192, "%s", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: pier create: %s\r\n", strerror(errno)); + exit(1); + } + } + + snprintf(ful_c, 8192, "%s/.urb", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: .urb create: %s\r\n", strerror(errno)); + exit(1); + } + } + + snprintf(ful_c, 8192, "%s/.urb/chk", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: .urb/chk create: %s\r\n", strerror(errno)); + exit(1); + } + } + + return strdup(dir_c); +} + /* u3m_boot(): start the u3 system. return next event, starting from 1. */ c3_d @@ -2109,7 +2137,7 @@ u3m_boot(c3_c* dir_c, size_t len_i) /* Activate the storage system. */ - nuu_o = u3e_live(c3n, dir_c); + nuu_o = u3e_live(c3n, u3m_pier(dir_c)); /* Activate tracing. */ diff --git a/pkg/noun/manage.h b/pkg/noun/manage.h index 35ad448168..14f9cc503d 100644 --- a/pkg/noun/manage.h +++ b/pkg/noun/manage.h @@ -17,6 +17,11 @@ c3_d u3m_boot(c3_c* dir_c, size_t len_i); + /* u3m_pier(): make a pier. + */ + c3_c* + u3m_pier(c3_c* dir_c); + /* u3m_boot_lite(): start without checkpointing. */ c3_d diff --git a/pkg/noun/retrieve.c b/pkg/noun/retrieve.c index 43f001fe1f..8f9847ff92 100644 --- a/pkg/noun/retrieve.c +++ b/pkg/noun/retrieve.c @@ -9,6 +9,35 @@ #include "trace.h" #include "xtract.h" + +// declarations of inline functions +// +c3_o +u3r_cell(u3_noun a, u3_noun* b, u3_noun* c); +c3_o +u3r_trel(u3_noun a, u3_noun* b, u3_noun* c, u3_noun* d); +c3_o +u3r_qual(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e); +c3_o +u3r_quil(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e, + u3_noun* f); +c3_o +u3r_hext(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e, + u3_noun* f, + u3_noun* g); + /* _frag_word(): fast fragment/branch prediction for top word. */ static u3_weak @@ -833,27 +862,6 @@ u3r_bite(u3_noun bite, u3_atom* bloq, u3_atom *step) } } -/* u3r_cell(): -** -** Factor (a) as a cell (b c). -*/ -c3_o -u3r_cell(u3_noun a, - u3_noun* b, - u3_noun* c) -{ - u3_assert(u3_none != a); - - if ( _(u3a_is_atom(a)) ) { - return c3n; - } - else { - if ( b ) *b = u3a_h(a); - if ( c ) *c = u3a_t(a); - return c3y; - } -} - /* u3r_p(): ** ** & [0] if [a] is of the form [b *c]. @@ -937,90 +945,6 @@ u3r_pqrs(u3_noun a, else return c3n; } -/* u3r_trel(): -** -** Factor (a) as a trel (b c d). -*/ -c3_o -u3r_trel(u3_noun a, - u3_noun *b, - u3_noun *c, - u3_noun *d) -{ - u3_noun guf; - - if ( (c3y == u3r_cell(a, b, &guf)) && - (c3y == u3r_cell(guf, c, d)) ) { - return c3y; - } - else { - return c3n; - } -} - -/* u3r_qual(): -** -** Factor (a) as a qual (b c d e). -*/ -c3_o -u3r_qual(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e) -{ - u3_noun guf; - - if ( (c3y == u3r_cell(a, b, &guf)) && - (c3y == u3r_trel(guf, c, d, e)) ) { - return c3y; - } - else return c3n; -} - -/* u3r_quil(): -** -** Factor (a) as a quil (b c d e f). -*/ -c3_o -u3r_quil(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f) -{ - u3_noun guf; - - if ( (c3y == u3r_cell(a, b, &guf)) && - (c3y == u3r_qual(guf, c, d, e, f)) ) { - return c3y; - } - else return c3n; -} - -/* u3r_hext(): -** -** Factor (a) as a hext (b c d e f g) -*/ -c3_o -u3r_hext(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f, - u3_noun* g) -{ - u3_noun guf; - - if ( (c3y == u3r_cell(a, b, &guf)) && - (c3y == u3r_quil(guf, c, d, e, f, g)) ) { - return c3y; - } - else return c3n; -} - /* u3r_met(): ** ** Return the size of (b) in bits, rounded up to diff --git a/pkg/noun/retrieve.h b/pkg/noun/retrieve.h index 9a5b092c17..7423ad9f49 100644 --- a/pkg/noun/retrieve.h +++ b/pkg/noun/retrieve.h @@ -4,11 +4,107 @@ #define U3_RETRIEVE_H #include "c3.h" +#include "allocate.h" +#include "error.h" #include "gmp.h" #include "types.h" /** u3r_*: read without ever crashing. **/ + + /* u3r_cell(): factor (a) as a cell (b c). + */ + inline c3_o + u3r_cell(u3_noun a, u3_noun* b, u3_noun* c) + { + u3a_cell* cel_u; + + u3_assert(u3_none != a); + + if ( c3y == u3a_is_cell(a) ) { + cel_u = u3a_to_ptr(a); + if ( b ) *b = cel_u->hed; + if ( c ) *c = cel_u->tel; + return c3y; + } + else { + return c3n; + } + } + + /* u3r_trel(): factor (a) as a trel (b c d). + */ + inline c3_o + u3r_trel(u3_noun a, u3_noun *b, u3_noun *c, u3_noun *d) + { + u3_noun guf; + + if ( (c3y == u3r_cell(a, b, &guf)) && + (c3y == u3r_cell(guf, c, d)) ) { + return c3y; + } + else { + return c3n; + } + } + + /* u3r_qual(): factor (a) as a qual (b c d e). + */ + inline c3_o + u3r_qual(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e) + { + u3_noun guf; + + if ( (c3y == u3r_cell(a, b, &guf)) && + (c3y == u3r_trel(guf, c, d, e)) ) { + return c3y; + } + else return c3n; + } + + /* u3r_quil(): factor (a) as a quil (b c d e f). + */ + inline c3_o + u3r_quil(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e, + u3_noun* f) + { + u3_noun guf; + + if ( (c3y == u3r_cell(a, b, &guf)) && + (c3y == u3r_qual(guf, c, d, e, f)) ) { + return c3y; + } + else return c3n; + } + + /* u3r_hext(): factor (a) as a hext (b c d e f g) + */ + inline c3_o + u3r_hext(u3_noun a, + u3_noun* b, + u3_noun* c, + u3_noun* d, + u3_noun* e, + u3_noun* f, + u3_noun* g) + { + u3_noun guf; + + if ( (c3y == u3r_cell(a, b, &guf)) && + (c3y == u3r_quil(guf, c, d, e, f, g)) ) { + return c3y; + } + else return c3n; + } + /* u3r_at(): fragment `a` of `b`, or u3_none. */ u3_weak @@ -183,61 +279,6 @@ c3_o u3r_bite(u3_noun bite, u3_atom* bloq, u3_atom *step); - /* u3r_cell(): - ** - ** Divide `a` as a cell `[b c]`. - */ - c3_o - u3r_cell(u3_noun a, - u3_noun* b, - u3_noun* c); - - /* u3r_trel(): - ** - ** Divide `a` as a trel `[b c d]`. - */ - c3_o - u3r_trel(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d); - - /* u3r_qual(): - ** - ** Divide (a) as a qual [b c d e]. - */ - c3_o - u3r_qual(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e); - - /* u3r_quil(): - ** - ** Divide (a) as a quil [b c d e f]. - */ - c3_o - u3r_quil(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f); - - /* u3r_hext(): - ** - ** Divide (a) as a hext [b c d e f g]. - */ - c3_o - u3r_hext(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f, - u3_noun* g); - /* u3r_p(): ** ** & [0] if [a] is of the form [b *c]. diff --git a/pkg/noun/trace.c b/pkg/noun/trace.c index 8768531d62..fd33169e8d 100644 --- a/pkg/noun/trace.c +++ b/pkg/noun/trace.c @@ -299,7 +299,7 @@ u3t_flee(void) /* u3t_trace_open(): opens a trace file and writes the preamble. */ void -u3t_trace_open(c3_c* dir_c) +u3t_trace_open(const c3_c* dir_c) { c3_c fil_c[2048]; diff --git a/pkg/noun/trace.h b/pkg/noun/trace.h index 667cdfae4b..695a5fe133 100644 --- a/pkg/noun/trace.h +++ b/pkg/noun/trace.h @@ -103,7 +103,7 @@ /* u3t_trace_open(): opens the path for writing tracing information. */ void - u3t_trace_open(c3_c *dir_c); + u3t_trace_open(const c3_c *dir_c); /* u3t_trace_close(): closes the trace file. optional. */ diff --git a/pkg/noun/version.h b/pkg/noun/version.h index bc81c83f5a..5cc6acb86d 100644 --- a/pkg/noun/version.h +++ b/pkg/noun/version.h @@ -19,4 +19,10 @@ typedef c3_w u3e_version; #define U3E_VER1 1 #define U3E_VERLAT U3E_VER1 +/* DISK FORMAT + */ + +#define U3D_VER1 1 +#define U3D_VERLAT U3L_VER1 + #endif /* ifndef U3_VERSION_H */ diff --git a/pkg/noun/vortex.c b/pkg/noun/vortex.c index 5981068b87..0bb9af8c29 100644 --- a/pkg/noun/vortex.c +++ b/pkg/noun/vortex.c @@ -39,18 +39,12 @@ c3_o u3v_boot(u3_noun eve) { c3_d len_d; - { u3_noun len = u3qb_lent(eve); u3_assert( c3y == u3r_safe_chub(len, &len_d) ); u3z(len); } - // ensure zero-initialized kernel - // - u3A->roc = 0; - u3A->eve_d = 0; - { u3_noun pro = u3m_soft(0, u3v_life, eve); @@ -59,6 +53,7 @@ u3v_boot(u3_noun eve) return c3n; } + u3z(u3A->roc); u3A->roc = u3k(u3t(pro)); u3A->eve_d = len_d; u3z(pro); diff --git a/pkg/noun/xtract.c b/pkg/noun/xtract.c index fc59f463d8..a869eb9547 100644 --- a/pkg/noun/xtract.c +++ b/pkg/noun/xtract.c @@ -5,21 +5,8 @@ #include "manage.h" #include "retrieve.h" -/* u3x_good(): test for u3_none. -*/ u3_noun -u3x_good(u3_weak som) -{ - return ( u3_none == som ) ? u3m_bail(c3__exit) : som; -} - -/* u3x_at (u3at): fragment. -*/ -u3_noun -u3x_at(u3_noun axe, u3_noun som) -{ - return u3x_good(u3r_at(axe, som)); -} +u3x_good(u3_weak som); /* u3x_mean(): ** @@ -40,94 +27,3 @@ u3x_mean(u3_noun som, ...) u3m_bail(c3__exit); } } - -/* u3x_bite(): xtract/default $bloq and $step from $bite. -*/ -void -u3x_bite(u3_noun bite, u3_atom* bloq, u3_atom *step) -{ - if ( c3n == u3r_bite(bite, bloq, step) ) { - u3m_bail(c3__exit); - } -} - -/* u3x_cell(): -** -** Divide `a` as a cell `[b c]`. -*/ -void -u3x_cell(u3_noun a, - u3_noun* b, - u3_noun* c) -{ - if ( c3n == u3r_cell(a, b, c) ) { - u3m_bail(c3__exit); - } -} - -/* u3x_trel(): -** -** Divide `a` as a trel `[b c d]`, or bail. -*/ -void -u3x_trel(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d) -{ - if ( c3n == u3r_trel(a, b, c, d) ) { - u3m_bail(c3__exit); - } -} - -/* u3x_qual(): -** -** Divide `a` as a quadruple `[b c d e]`. -*/ -void -u3x_qual(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e) -{ - if ( c3n == u3r_qual(a, b, c, d, e) ) { - u3m_bail(c3__exit); - } -} - -/* u3x_quil(): -** -** Divide `a` as a quintuple `[b c d e f]`. -*/ -void -u3x_quil(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f) -{ - if ( c3n == u3r_quil(a, b, c, d, e, f) ) { - u3m_bail(c3__exit); - } -} - -/* u3x_hext(): -** -** Divide `a` as a hextuple `[b c d e f g]`. -*/ -void -u3x_hext(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f, - u3_noun* g) -{ - if ( c3n == u3r_hext(a, b, c, d, e, f, g) ) { - u3m_bail(c3__exit); - } -} - diff --git a/pkg/noun/xtract.h b/pkg/noun/xtract.h index cf39011463..343690a8f2 100644 --- a/pkg/noun/xtract.h +++ b/pkg/noun/xtract.h @@ -5,6 +5,8 @@ #include "c3.h" #include "types.h" +#include "allocate.h" +#include "manage.h" /** Constants. **/ @@ -40,6 +42,21 @@ **/ /* Word axis macros. For 31-bit axes only. */ + + /* u3x_at (u3at): fragment. + */ +# define u3x_at(a, b) u3x_good(u3r_at(a, b)) +# define u3at(a, b) u3x_at(a, b) + + /* u3x_bite(): xtract/default $bloq and $step from $bite. + */ +# define u3x_bite(a, b, c) \ + do { \ + if ( c3n == u3r_bite(a, b, c) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) + /* u3x_dep(): number of axis bits. */ # define u3x_dep(a_w) (c3_bits_word(a_w) - 1) @@ -61,25 +78,70 @@ # define u3x_peg(a_w, b_w) \ ( (a_w << u3x_dep(b_w)) | (b_w &~ (1 << u3x_dep(b_w))) ) - /* u3x_atom(): atom or exit. + /* u3x_cell(): divide `a` as a cell `[b c]`. + */ +# define u3x_cell(a, b, c) \ + do { \ + if ( c3n == u3r_cell(a, b, c) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) + + /* u3x_trel(): divide `a` as a trel `[b c d]`, or bail. + */ +# define u3x_trel(a, b, c, d) \ + do { \ + if ( c3n == u3r_trel(a, b, c, d) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) + + /* u3x_qual(): divide `a` as a quadruple `[b c d e]`. */ -# define u3x_atom(a) \ - ( (c3y == u3a_is_cell(a)) ? u3m_bail(c3__exit) : a ) +# define u3x_qual(a, b, c, d, e) \ + do { \ + if ( c3n == u3r_qual(a, b, c, d, e) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) + + /* u3x_quil(): divide `a` as a quintuple `[b c d e f]`. + */ +# define u3x_quil(a, b, c, d, e, f) \ + do { \ + if ( c3n == u3r_quil(a, b, c, d, e, f) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) + + /* u3x_hext(): divide `a` as a hextuple `[b c d e f g]`. + */ +# define u3x_hext(a, b, c, d, e, f, g) \ + do { \ + if ( c3n == u3r_hext(a, b, c, d, e, f, g) ) { \ + u3m_bail(c3__exit); \ + } \ + } while (0) /** Functions. **/ /** u3x_*: read, but bail with c3__exit on a crash. **/ - /* u3x_good(): test for u3_none. + /* u3x_atom(): atom or exit. */ - u3_noun - u3x_good(u3_weak som); + inline u3_atom + u3x_atom(u3_noun a) + { + return ( c3y == u3a_is_cell(a) ) ? u3m_bail(c3__exit) : a; + } - /* u3x_at (u3at): fragment. + /* u3x_good(): test for u3_none. */ - u3_noun - u3x_at(u3_noun axe, u3_noun som); -# define u3at(axe, som) u3x_at(axe, som) + inline u3_noun + u3x_good(u3_weak som) + { + return ( u3_none == som ) ? u3m_bail(c3__exit) : som; + } /* u3x_mean(): ** @@ -89,64 +151,4 @@ void u3x_mean(u3_noun a, ...); - /* u3x_bite(): xtract/default $bloq and $step from $bite. - */ - void - u3x_bite(u3_noun bite, u3_atom* bloq, u3_atom *step); - - /* u3x_cell(): - ** - ** Divide `a` as a cell `[b c]`. - */ - void - u3x_cell(u3_noun a, - u3_noun* b, - u3_noun* c); - - /* u3x_trel(): - ** - ** Divide `a` as a trel `[b c d]`, or bail. - */ - void - u3x_trel(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d); - - /* u3x_qual(): - ** - ** Divide `a` as a quadruple `[b c d e]`. - */ - void - u3x_qual(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e); - - /* u3x_quil(): - ** - ** Divide `a` as a quintuple `[b c d e f]`. - */ - void - u3x_quil(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f); - - /* u3x_hext(): - ** - ** Divide `a` as a hextuple `[b c d e f g]`. - */ - void - u3x_hext(u3_noun a, - u3_noun* b, - u3_noun* c, - u3_noun* d, - u3_noun* e, - u3_noun* f, - u3_noun* g); - #endif /* ifndef U3_XTRACT_H */ diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 2cc270a368..45a2c21cef 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1,8 +1,11 @@ /// @file #include "noun.h" +#include "events.h" #include "vere.h" +#include "version.h" #include "db/lmdb.h" +#include struct _cd_read { uv_timer_t tim_u; @@ -28,6 +31,15 @@ struct _u3_disk_walk { c3_o liv_o; }; +// for u3_lmdb_init() calls +static const size_t siz_i = +#if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + // 500 GiB is as large as musl on aarch64 wants to allow + 0x7d00000000; +#else + 0x10000000000; +#endif + #undef VERBOSE_DISK #undef DISK_TRACE_JAM #undef DISK_TRACE_CUE @@ -960,6 +972,537 @@ u3_disk_slog(u3_disk* log_u) } } +/* _disk_epoc_meta: read metadata from epoch. +*/ +static c3_o +_disk_epoc_meta(u3_disk* log_u, + c3_d epo_d, + const c3_c* met_c, + c3_w max_w, + c3_c* buf_c) +{ + struct stat buf_u; + c3_w red_w, len_w; + c3_i ret_i, fid_i; + c3_c* pat_c; + + ret_i = asprintf(&pat_c, "%s/0i%" PRIc3_d "/%s.txt", + log_u->com_u->pax_c, epo_d, met_c); + u3_assert( ret_i > 0 ); + + fid_i = c3_open(pat_c, O_RDONLY, 0644); + c3_free(pat_c); + + if ( (fid_i < 0) || (fstat(fid_i, &buf_u) < 0) ) { + fprintf(stderr, "disk: failed to open %s.txt in epoch 0i%" PRIc3_d "\r\n", + met_c, epo_d); + return c3n; + } + else if ( buf_u.st_size >= max_w ) { + fprintf(stderr, "disk: %s.txt in epoch 0i%" PRIc3_d " too large " + "(%" PRIc3_z ")\r\n", + met_c, epo_d, (c3_z)buf_u.st_size); + return c3n; + } + + len_w = buf_u.st_size; + red_w = read(fid_i, buf_c, len_w); + close(fid_i); + + if ( len_w != red_w ) { + fprintf(stderr, "disk: failed to read %s.txt in epoch 0i%" PRIc3_d "\r\n", + met_c, epo_d); + return c3n; + } + + // trim trailing whitespace + // + do { + buf_c[len_w] = 0; + } + while ( len_w-- && isspace(buf_c[len_w]) ); + + return c3y; +} + +/* u3_disk_epoc_init: create new epoch. +*/ +c3_o +u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d) +{ + // check if any epoch directories exist + c3_d lat_d; + c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); + + // create new epoch directory if it doesn't exist + c3_c epo_c[8193]; + c3_i epo_i; + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); + c3_d ret_d = c3_mkdir(epo_c, 0700); + if ( ( ret_d < 0 ) && ( errno != EEXIST ) ) { + fprintf(stderr, "disk: create epoch dir %" PRIc3_d " failed: %s\r\n", + epo_d, strerror(errno)); + return c3n; + } + + // copy snapshot files (skip if first epoch) + if ( epo_d > 0 ) { + c3_c chk_c[8193]; + snprintf(chk_c, 8192, "%s/.urb/chk", u3_Host.dir_c); + if ( c3n == u3e_backup(chk_c, epo_c, c3y) ) { + fprintf(stderr, "disk: copy epoch snapshot failed\r\n"); + goto fail1; + } + } + + if ( -1 == (epo_i = c3_open(epo_c, O_RDONLY)) ) { + fprintf(stderr, "disk: open epoch dir %" PRIc3_d " failed: %s\r\n", + epo_d, strerror(errno)); + goto fail1; + } + + if ( -1 == c3_sync(epo_i) ) { // XX fdatasync on linux? + fprintf(stderr, "disk: sync epoch dir %" PRIc3_d " failed: %s\r\n", + epo_d, strerror(errno)); + goto fail2; + } + + // create epoch version file, overwriting any existing file + c3_c epv_c[8193]; + snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); + FILE* epv_f = fopen(epv_c, "w"); // XX errors + fprintf(epv_f, "%d", U3D_VER1); + fclose(epv_f); + + // create binary version file, overwriting any existing file + c3_c biv_c[8193]; + snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); + FILE* biv_f = fopen(biv_c, "w"); // XX errors + fprintf(biv_f, URBIT_VERSION); + fclose(biv_f); + + // get metadata from old epoch or unmigrated event log's db + c3_d who_d[2]; + c3_o fak_o; + c3_w lif_w; + if ( c3y == eps_o ) { // skip if no epochs yet + if ( c3y != u3_disk_read_meta(log_u->mdb_u, who_d, &fak_o, &lif_w) ) { + fprintf(stderr, "disk: failed to read metadata\r\n"); + goto fail3; + } + + u3_lmdb_exit(log_u->mdb_u); + log_u->mdb_u = 0; + } + + // initialize db of new epoch + if ( c3y == u3_Host.ops_u.nuu || epo_d > 0 ) { + c3_c dat_c[8193]; + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize database\r\n"); + c3_free(log_u); + goto fail3; + } + } + + // write the metadata to the database + if ( c3y == eps_o ) { + if ( c3n == u3_disk_save_meta(log_u->mdb_u, who_d, fak_o, lif_w) ) { + fprintf(stderr, "disk: failed to save metadata\r\n"); + goto fail3; + } + } + + if ( -1 == c3_sync(epo_i) ) { // XX fdatasync on linux? + fprintf(stderr, "disk: sync epoch dir %" PRIc3_d " failed: %s\r\n", + epo_d, strerror(errno)); + goto fail3; + } + + close(epo_i); + + // load new epoch directory and set it in log_u + log_u->epo_d = epo_d; + + // success + return c3y; + +fail3: + c3_unlink(epv_c); + c3_unlink(biv_c); +fail2: + close(epo_i); +fail1: + c3_rmdir(epo_c); + return c3n; +} + +/* u3_disk_epoc_kill: delete an epoch. +*/ +c3_o +u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) +{ + // get epoch directory + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); + + // delete files in epoch directory + u3_dire* dir_u = u3_foil_folder(epo_c); + u3_dent* den_u = dir_u->all_u; + while ( den_u ) { + c3_c fil_c[8193]; + snprintf(fil_c, sizeof(fil_c), "%s/%s", epo_c, den_u->nam_c); + if ( 0 != c3_unlink(fil_c) ) { + fprintf(stderr, "disk: failed to delete file in epoch directory\r\n"); + return c3n; + } + den_u = den_u->nex_u; + } + + // delete epoch directory + if ( 0 != c3_rmdir(epo_c) ) { + fprintf(stderr, "disk: failed to delete epoch directory\r\n"); + return c3n; + } + + // cleanup + u3_dire_free(dir_u); + + // success + return c3y; +} + +/* u3_disk_epoc_last: get latest epoch number. +*/ +c3_o +u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) +{ + u3_dire* die_u = u3_foil_folder(log_u->com_u->pax_c); + u3_dent* den_u = die_u->dil_u; + c3_o ret_o = c3n; + + *lat_d = 0; + + while ( den_u ) { + c3_d epo_d = 0; + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, &epo_d) ) { + ret_o = c3y; // NB: returns yes if the directory merely exists + *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number + } + den_u = den_u->nex_u; + } + + u3_dire_free(die_u); + + return ret_o; +} + +/* u3_disk_epoc_list: get descending epoch numbers, "mcut" pattern. +*/ +c3_z +u3_disk_epoc_list(u3_disk* log_u, c3_d* sot_d) +{ + u3_dire* ned_u = u3_foil_folder(log_u->com_u->pax_c); + u3_dent* den_u = ned_u->dil_u; + c3_z len_z = 0; + + while ( den_u ) { // count epochs + len_z++; + den_u = den_u->nex_u; + } + + if ( !sot_d ) { + u3_dire_free(ned_u); + return len_z; + } + + len_z = 0; + den_u = ned_u->dil_u; + + while ( den_u ) { + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, (sot_d + len_z)) ) { + len_z++; + } + den_u = den_u->nex_u; + } + + // sort sot_d naively in descending order + // + c3_d tmp_d; + for ( c3_z i_z = 0; i_z < len_z; i_z++ ) { + for ( c3_z j_z = i_z + 1; j_z < len_z; j_z++ ) { + if ( sot_d[i_z] < sot_d[j_z] ) { + tmp_d = sot_d[i_z]; + sot_d[i_z] = sot_d[j_z]; + sot_d[j_z] = tmp_d; + } + } + } + + u3_dire_free(ned_u); + return len_z; +} + +/* _disk_need_migrate: does the disk need to be migrated? +*/ +static c3_o +_disk_need_migrate(u3_disk* log_u) +{ + c3_c dut_c[8193]; + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + + // check if data.mdb is readable in log directory + // + if ( 0 == access(dut_c, F_OK)) { + return c3y; + } + + return c3n; +} + +/* _disk_migrate: migrates disk format. + */ +static c3_o +_disk_migrate(u3_disk* log_u, c3_d eve_d) +{ + /* migration steps: + * 0. detect whether we need to migrate or not + * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration + * b. if data.mdb is readable in log directory -> execute migration + * if not -> skip migration (returns yes) + * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) + * a. creates epoch directory + * b. creates epoch version file + * c. creates binary version file + * d. initializes database + * e. reads metadata from old database + * f. writes metadata to new database + * g. loads new epoch directory and sets it in log_u + * 2. create hard links to data.mdb and lock.mdb in 0i0/ + * 3. rollover to new epoch (second call to u3_disk_epoc_init()) + * a. same as 1a-g but also copies current snapshot between c/d steps + * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) + * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) + */ + + // check if lock.mdb is readable in log directory + c3_o luk_o = c3n; + c3_c luk_c[8193]; + snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + if ( 0 == access(luk_c, R_OK) ) { + luk_o = c3y; + } + + fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); + + // migrate existing pier which has either: + // - not started the migration, or + // - crashed before completing the migration + + // get first/last event numbers from pre-migrated lmdb + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + return c3n; + } + + // ensure there's a current snapshot + if ( eve_d != las_d ) { + fprintf(stderr, "disk: snapshot is out of date, please " + "start/shutdown your pier gracefully first\r\n"); + fprintf(stderr, "disk: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", + eve_d, las_d); + return c3n; + } + + // initialize first epoch "0i0" + if ( c3n == u3_disk_epoc_init(log_u, 0) ) { + fprintf(stderr, "disk: failed to initialize first epoch\r\n"); + return c3n; + } + + // create hard links to data.mdb and lock.mdb in 0i0/ + c3_c epo_c[8193], dut_c[8193], dat_c[8193], lok_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); + + if ( 0 < c3_link(dut_c, dat_c) ) { + fprintf(stderr, "disk: failed to create data.mdb hard link\r\n"); + return c3n; + } + if ( c3y == luk_o ) { // only link lock.mdb if it exists + if ( 0 < c3_link(luk_c, lok_c) ) { + fprintf(stderr, "disk: failed to create lock.mdb hard link\r\n"); + return c3n; + } + } + + // rollover to new epoch + if ( c3n == u3_disk_epoc_init(log_u, las_d) ) { + fprintf(stderr, "disk: failed to initialize new epoch\r\n"); + return c3n; + } + + // delete backup snapshot + c3_c bhk_c[8193], nop_c[8193], sop_c[8193]; + snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); + snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); + snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); + if ( c3_unlink(nop_c) ) { + fprintf(stderr, "disk: failed to delete bhk/north.bin: %s\r\n", + strerror(errno)); + } + else if ( c3_unlink(sop_c) ) { + fprintf(stderr, "disk: failed to delete bhk/south.bin: %s\r\n", + strerror(errno)); + } + else { + if ( c3_rmdir(bhk_c) ) { + fprintf(stderr, "disk: failed to delete bhk/: %s\r\n", + strerror(errno)); + } + } + + // delete old lock.mdb and data.mdb files + if ( c3_unlink(luk_c) ) { + fprintf(stderr, "disk: failed to unlink lock.mdb: %s\r\n", + strerror(errno)); + } + if ( c3_unlink(dut_c) ) { + fprintf(stderr, "disk: failed to unlink data.mdb: %s\r\n", + strerror(errno)); + return c3n; // migration succeeds only if we can unlink data.mdb + } + + // success + fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); + + return c3y; +} + +/* _disk_vere_diff(): checks if vere version mismatches latest epoch's. +*/ +static c3_o +_disk_vere_diff(u3_disk* log_u) +{ + c3_c ver_c[128]; + + if ( c3n == _disk_epoc_meta(log_u, log_u->epo_d, "vere", + sizeof(ver_c) - 1, ver_c) ) + { + return c3y; // assume mismatch if we can't read version + } + + if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { + return c3y; + } + + return c3n; +} + +/* u3_disk_kindly(): do the needful. +*/ +void +u3_disk_kindly(u3_disk* log_u, c3_d eve_d) +{ + if ( c3y == _disk_need_migrate(log_u) ) { + if ( c3n == _disk_migrate(log_u, eve_d) ) { + fprintf(stderr, "disk: failed to migrate event log\r\n"); + exit(1); + } + } + else if ( c3y == _disk_vere_diff(log_u) ) { + if ( c3n == u3_disk_epoc_init(log_u, log_u->dun_d) ) { + fprintf(stderr, "disk: failed to initialize epoch\r\n"); + exit(1); + } + } +} + +typedef enum { + _epoc_good = 0, // load successfully + _epoc_gone = 1, // version missing, total failure + _epoc_fail = 2, // transient failure (?) + _epoc_void = 3, // empty event log (cheaper to recover?) + _epoc_late = 4 // format from the future +} _epoc_kind; + +/* _disk_epoc_load(): load existing epoch, enumerating failures +*/ +static _epoc_kind +_disk_epoc_load(u3_disk* log_u, c3_d lat_d) +{ + // check latest epoc version + // + { + c3_c ver_c[8]; + c3_w ver_w; + if ( c3n == _disk_epoc_meta(log_u, lat_d, "epoc", + sizeof(ver_c) - 1, ver_c) ) + { + fprintf(stderr, "disk: failed to load epoch 0i%" PRIc3_d " version\r\n", + lat_d); + + return _epoc_gone; + } + + if ( 1 != sscanf(ver_c, "%d", &ver_w) ) { + fprintf(stderr, "disk: failed to parse epoch version: '%s'\r\n", ver_c); + return _epoc_fail; + } + + if ( U3D_VER1 != ver_w ) { + fprintf(stderr, "disk: unknown epoch version: '%s', expected '%d'\r\n", + ver_c, U3D_VER1); + return _epoc_late; + } + + log_u->ver_w = ver_w; + } + + // set path to latest epoch + c3_c epo_c[8193]; + snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, lat_d); + + // initialize latest epoch's db + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize database at %s\r\n", + epo_c); + return _epoc_fail; + } + + fprintf(stderr, "disk: loaded epoch 0i%" PRIc3_d "\r\n", lat_d); + + // get first/last event numbers from lmdb + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + u3_lmdb_exit(log_u->mdb_u); + log_u->mdb_u = 0; + return _epoc_fail; + } + + if ( (c3n == u3_Host.ops_u.nuu ) + && !fir_d + && !las_d + && (c3n == u3_disk_read_meta(log_u->mdb_u, 0, 0, 0)) ) + { + u3_lmdb_exit(log_u->mdb_u); + log_u->mdb_u = 0; + return _epoc_void; + } + + // initialize dun_d/sen_d values + log_u->dun_d = ( 0 != las_d ) ? las_d : lat_d; + log_u->sen_d = log_u->dun_d; + + return _epoc_good; +} + /* u3_disk_init(): load or create pier directories and event log. */ u3_disk* @@ -1019,66 +1562,153 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_free(dir_c); } - // create/load $pier/.urb/log, initialize db + // create/load $pier/.urb/log // { - c3_c* log_c = c3_malloc(10 + strlen(pax_c)); - - strcpy(log_c, pax_c); - strcat(log_c, "/.urb/log"); + c3_c log_c[8193]; + snprintf(log_c, sizeof(log_c), "%s/.urb/log", pax_c); if ( 0 == (log_u->com_u = u3_foil_folder(log_c)) ) { fprintf(stderr, "disk: failed to load /.urb/log in %s\r\n", pax_c); - c3_free(log_c); c3_free(log_u); return 0; } - // Arbitrarily choosing 1TB as a "large enough" mapsize + // if fresh boot, initialize disk v1 // - // per the LMDB docs: - // "[..] on 64-bit there is no penalty for making this huge (say 1TB)." - // - { - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif + if ( c3y == u3_Host.ops_u.nuu ) { + // initialize first epoch "0i0" + if ( c3n == u3_disk_epoc_init(log_u, 0) ) { + fprintf(stderr, "disk: failed to initialize first epoch\r\n"); + return 0; + } + } + else if ( c3y == _disk_need_migrate(log_u) ) { + fprintf(stderr, "disk: loading old format\r\n"); if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) { - fprintf(stderr, "disk: failed to initialize database\r\n"); - c3_free(log_c); + fprintf(stderr, "disk: failed to initialize lmdb\r\n"); + c3_free(log_u); + } + + c3_d fir_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { + fprintf(stderr, "disk: failed to load latest event from lmdb\r\n"); + // XX dispose mdb_u c3_free(log_u); return 0; } - } - c3_free(log_c); - } + // if a top-level log exists but is empty it's safe to remove. + // for extra safety, we only do this if we also have epochs. + // (the top-level log was most likely created by vere downgrade) + // + c3_d nop_d; + if ( !fir_d + && !log_u->dun_d + && (c3y == u3_disk_epoc_last(log_u, &nop_d)) ) + { + c3_c luk_c[8193]; c3_c dut_c[8193]; + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_c); + snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + + u3_lmdb_exit(log_u->mdb_u); + log_u->mdb_u = 0; + + if ( c3_unlink(dut_c) + || ( (0 == access(luk_c, R_OK)) + && c3_unlink(luk_c) )) + { + fprintf(stderr, "disk: failed to unlink empty, old log\r\n"); + } - // get the latest event number from the db - // - { - log_u->dun_d = 0; - c3_d fir_d; + // continue to epoc initialization + } + + else { + log_u->sen_d = log_u->dun_d; + log_u->ver_w = 0; + + return log_u; + } + } - if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { - fprintf(stderr, "disk: failed to load latest event from database\r\n"); + // get latest epoch number + c3_d lat_d; + if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { + fprintf(stderr, "disk: failed to load epoch number\r\n"); c3_free(log_u); return 0; } - log_u->sen_d = log_u->dun_d; - } + c3_o try_o = c3n; + +try_init: + { + _epoc_kind kin_e = _disk_epoc_load(log_u, lat_d); + + switch ( kin_e ) { + case _epoc_good: { + // mark the latest epoch directory + log_u->epo_d = lat_d; - log_u->liv_o = c3y; + // mark the log as live + log_u->liv_o = c3y; #if defined(DISK_TRACE_JAM) || defined(DISK_TRACE_CUE) - u3t_trace_open(pax_c); + u3t_trace_open(pax_c); #endif - return log_u; + return log_u; + } break; + + case _epoc_void: // XX could handle more efficiently + + case _epoc_gone: { + // XX if there is no version number, the epoc is invalid + // backup and try previous + + if ( c3y == try_o ) { + fprintf(stderr, "multiple bad epochs, bailing out\r\n"); + c3_free(log_u); + return 0; + } + + c3_z len_z = u3_disk_epoc_list(log_u, 0); + + if ( len_z <= 1 ) { + fprintf(stderr, "only epoch is bad, bailing out\r\n"); + c3_free(log_u); + return 0; + } + + c3_d* sot_d = c3_malloc(len_z * sizeof(*sot_d)); + u3_disk_epoc_list(log_u, sot_d); + + fprintf(stderr, "disk: latest epoch is 0i%" PRIc3_d " is bogus; " + "falling back to previous at 0i%" PRIc3_d "\r\n", + lat_d, sot_d[1]); + + u3_disk_epoc_kill(log_u, lat_d); + + lat_d = sot_d[1]; + try_o = c3y; + c3_free(sot_d); + goto try_init; + } break; + + case _epoc_fail: { + fprintf(stderr, "failed to load epoch\r\n"); + c3_free(log_u); + return 0; + } break; + + case _epoc_late: { + fprintf(stderr, "upgrade runtime version\r\n"); + c3_free(log_u); + return 0; + } + } + } + } } diff --git a/pkg/vere/foil.c b/pkg/vere/foil.c index 26ff8b3d04..9fae86973f 100644 --- a/pkg/vere/foil.c +++ b/pkg/vere/foil.c @@ -106,13 +106,17 @@ u3_foil_folder(const c3_c* pax_c) dir_u = u3_dire_init(pax_c); } - /* create entries for all files + /* create entries for all files and directories */ while ( UV_EOF != uv_fs_scandir_next(&ruq_u, &den_u) ) { if ( UV_DIRENT_FILE == den_u.type ) { u3_dent* det_u = u3_dent_init(den_u.name); det_u->nex_u = dir_u->all_u; dir_u->all_u = det_u; + } else if ( UV_DIRENT_DIR == den_u.type ) { + u3_dent* det_u = u3_dent_init(den_u.name); + det_u->nex_u = dir_u->dil_u; + dir_u->dil_u = det_u; } } diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index 24baa6e042..6df308239b 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -1030,17 +1030,9 @@ _ames_czar_cb(uv_getaddrinfo_t* adr_u, struct addrinfo* rai_u = aif_u; time_t now = time(0); - while ( rai_u ) { - if ( (AF_INET == rai_u->ai_family) ) { - _ames_czar_here(pac_u, now, (struct sockaddr_in *)rai_u->ai_addr); - break; - } - else { - rai_u = rai_u->ai_next; - } - } - - if ( !rai_u ) { + if ( sas_i == 0 ) { + _ames_czar_here(pac_u, now, (struct sockaddr_in *)rai_u->ai_addr); + } else { _ames_czar_gone(pac_u, now); } } @@ -1128,9 +1120,13 @@ _ames_czar(u3_pact* pac_u) uv_getaddrinfo_t* adr_u = c3_malloc(sizeof(*adr_u)); adr_u->data = pac_u; + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // only IPv4 addresses + if ( 0 != (sas_i = uv_getaddrinfo(u3L, adr_u, _ames_czar_cb, - pac_u->rut_u.dns_c, 0, 0)) ) + pac_u->rut_u.dns_c, 0, &hints)) ) { u3l_log("ames: %s", uv_strerror(sas_i)); _ames_czar_gone(pac_u, now); diff --git a/pkg/vere/main.c b/pkg/vere/main.c index c35b7ff2e4..930d8981ad 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -724,6 +724,7 @@ _cw_usage(c3_c* bin_c) " %s next %.*s request upgrade:\n", " %s queu %.*s cue state:\n", " %s chop %.*s truncate event log:\n", + " %s roll %.*s rollover to new epoch:\n", " %s vere ARGS download binary:\n", "\n run as a 'serf':\n", " %s serf " @@ -1577,11 +1578,11 @@ _cw_info(c3_i argc, c3_c* argv[]) exit(1); } - c3_d eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); fprintf(stderr, "\r\nurbit: %s at event %" PRIu64 "\r\n", - u3_Host.dir_c, eve_d); + u3_Host.dir_c, u3_Host.eve_d); u3_disk_slog(log_u); printf("\n"); @@ -1736,17 +1737,17 @@ _cw_cram(c3_i argc, c3_c* argv[]) exit(1); } - c3_d eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock c3_o ret_o; fprintf(stderr, "urbit: cram: preparing\r\n"); - if ( c3n == (ret_o = u3u_cram(u3_Host.dir_c, eve_d)) ) { + if ( c3n == (ret_o = u3u_cram(u3_Host.dir_c, u3_Host.eve_d)) ) { fprintf(stderr, "urbit: cram: unable to jam state\r\n"); } else { - fprintf(stderr, "urbit: cram: rock saved at event %" PRIu64 "\r\n", eve_d); + fprintf(stderr, "urbit: cram: rock saved at event %" PRIu64 "\r\n", u3_Host.eve_d); } // save even on failure, as we just did all the work of deduplication @@ -1848,12 +1849,11 @@ _cw_queu(c3_i argc, c3_c* argv[]) exit(1); } else { + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock fprintf(stderr, "urbit: queu: preparing\r\n"); - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - // XX can spuriously fail do to corrupt memory-image checkpoint, // need a u3m_half_boot equivalent // workaround is to delete/move the checkpoint in case of corruption @@ -1941,10 +1941,10 @@ _cw_meld(c3_i argc, c3_c* argv[]) exit(1); } - u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - u3C.wag_w |= u3o_hashless; - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock u3a_print_memory(stderr, "urbit: meld: gained", u3u_meld()); @@ -2103,9 +2103,9 @@ _cw_pack(c3_i argc, c3_c* argv[]) exit(1); } + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3a_print_memory(stderr, "urbit: pack: gained", u3m_pack()); u3m_save(); @@ -2122,6 +2122,38 @@ _cw_play_slog(u3_noun hod) u3z(hod); } +/* _cw_play_snap(): prepare snapshot for full replay. +*/ +static void +_cw_play_snap(u3_disk* log_u) +{ + c3_c chk_c[8193], epo_c[8193]; + snprintf(chk_c, 8193, "%s/.urb/chk", u3_Host.dir_c); + snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, log_u->epo_d); + + if ( 0 == log_u->epo_d ) { + // if epoch 0 is the latest, delete the snapshot files in chk/ + c3_c nor_c[8193], sop_c[8193]; + snprintf(nor_c, 8193, "%s/.urb/chk/north.bin", u3_Host.dir_c); + snprintf(sop_c, 8193, "%s/.urb/chk/south.bin", u3_Host.dir_c); + if ( c3_unlink(nor_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + nor_c, strerror(errno)); + exit(1); + } + if ( c3_unlink(sop_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + sop_c, strerror(errno)); + exit(1); + } + } + else if ( 0 != u3e_backup(epo_c, chk_c, c3y) ) { + // copy the latest epoch's snapshot files into chk/ + fprintf(stderr, "mars: failed to copy snapshot\r\n"); + exit(1); + } +} + /* _cw_play_exit(): exit immediately. */ static void @@ -2133,6 +2165,68 @@ _cw_play_exit(c3_i int_i) raise(SIGINT); } +/* _cw_play_impl(): replay events, but better. +*/ +static c3_d +_cw_play_impl(c3_d eve_d, c3_d sap_d, c3_o mel_o, c3_o sof_o, c3_o ful_o) +{ + c3_d pay_d; + + // XX handle SIGTSTP so that the lockfile is not orphaned? + // + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); + + // Handle SIGTSTP as if it was SIGINT. + // + // Configured here using signal() so as to be immediately available. + // + signal(SIGTSTP, _cw_play_exit); + + // XX source these from a shared struct ops_u + if ( c3y == mel_o ) { + u3C.wag_w |= u3o_auto_meld; + } + + if ( c3y == sof_o ) { + u3C.wag_w |= u3o_soft_mugs; + } + + u3C.wag_w |= u3o_hashless; + + if ( c3y == ful_o ) { + u3l_log("mars: preparing for full replay"); + _cw_play_snap(log_u); + } + + // XX this should check that snapshot is within epoc, + // and load from the epoc / reboot if it is not + // + u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + + u3C.slog_f = _cw_play_slog; + + { + u3_mars mar_u = { + .log_u = log_u, + .dir_c = u3_Host.dir_c, + .sen_d = u3A->eve_d, + .dun_d = u3A->eve_d, + }; + + pay_d = u3_mars_play(&mar_u, eve_d, sap_d); + u3_Host.eve_d = mar_u.dun_d; + + // migrate or rollover as needed + // + u3_disk_kindly(log_u, u3_Host.eve_d); + } + + u3_disk_exit(log_u); + u3m_stop(); + + return pay_d; +} + /* _cw_play(): replay events, but better. */ static void @@ -2225,55 +2319,9 @@ _cw_play(c3_i argc, c3_c* argv[]) exit(1); } - // XX handle SIGTSTP so that the lockfile is not orphaned? - // - u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - - // Handle SIGTSTP as if it was SIGINT. - // - // Configured here using signal() so as to be immediately available. - // - signal(SIGTSTP, _cw_play_exit); - - if ( c3y == mel_o ) { - u3C.wag_w |= u3o_auto_meld; - } - - if ( c3y == sof_o ) { - u3C.wag_w |= u3o_soft_mugs; - } - - u3C.wag_w |= u3o_hashless; - - // XX this should restore the epoch snapshot and replay that - // - if ( c3y == ful_o ) { - u3l_log("mars: preparing for full replay"); - u3m_init((size_t)1 << u3_Host.ops_u.lom_y); - u3e_live(c3n, u3_Host.dir_c); - u3m_foul(); - u3m_pave(c3y); - u3j_boot(c3y); - } - else { - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + if ( !_cw_play_impl(eve_d, sap_d, mel_o, sof_o, ful_o) ) { + fprintf(stderr, "mars: nothing to do!"); } - - u3C.slog_f = _cw_play_slog; - - { - u3_mars mar_u = { - .log_u = log_u, - .dir_c = u3_Host.dir_c, - .sen_d = u3A->eve_d, - .dun_d = u3A->eve_d, - }; - - u3_mars_play(&mar_u, eve_d, sap_d); - } - - u3_disk_exit(log_u); - u3m_stop(); } /* _cw_prep(): prepare for upgrade @@ -2281,6 +2329,8 @@ _cw_play(c3_i argc, c3_c* argv[]) static void _cw_prep(c3_i argc, c3_c* argv[]) { + // XX roll with old binary + // check that new epoch is empty, migrate snapshot in-place c3_i ch_i, lid_i; c3_w arg_w; @@ -2421,123 +2471,138 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // gracefully shutdown the pier if it's running - u3_disk* old_u = _cw_disk_init(u3_Host.dir_c); + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); - // note: this include patch applications (if any) - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_disk_kindly(log_u, u3_Host.eve_d); - // check if there's a *current* snapshot - if ( old_u->dun_d != u3A->eve_d ) { - fprintf(stderr, "chop: error: snapshot is out of date, please " - "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "chop: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", u3A->eve_d, old_u->dun_d); + // create new epoch + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "chop: failed to get first/last events\r\n"); exit(1); } - if ( c3n == u3m_backup(c3y)) { // backup current snapshot - fprintf(stderr, "chop: error: failed to backup snapshot\r\n"); + // create new epoch if latest isn't empty + if ( (fir_d != las_d) && (c3n == u3_disk_epoc_init(log_u, las_d)) ) { + fprintf(stderr, "chop: failed to create new epoch\r\n"); exit(1); } - // initialize the lmdb environment - // see disk.c:885 - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - c3_c log_c[8193]; - snprintf(log_c, sizeof(log_c), "%s/.urb/log", u3_Host.dir_c); + c3_z len_z = u3_disk_epoc_list(log_u, 0); - // get the first/last event numbers from the event log - c3_d fir_d, las_d; - if ( c3n == u3_lmdb_gulf(old_u->mdb_u, &fir_d, &las_d) ) { - fprintf(stderr, "chop: failed to load latest event from database\r\n"); - exit(1); + if ( len_z <= 2 ) { + fprintf(stderr, "chop: nothing to do, have a great day\r\n"); + exit(0); // enjoy } - // get the metadata - c3_d who_d[2]; - c3_o fak_o; - c3_w lif_w; - if ( c3y != u3_disk_read_meta(old_u->mdb_u, who_d, &fak_o, &lif_w) ) { - fprintf(stderr, "chop: failed to read metadata\r\n"); - exit(1); - } + c3_d* sot_d = c3_malloc(len_z * sizeof(c3_d)); + u3_disk_epoc_list(log_u, sot_d); - // get the last event - u3_lmdb_walk itr_u; - size_t len_i; - void* buf_v[1]; - if ( c3n == u3_lmdb_walk_init(old_u->mdb_u, &itr_u, las_d, las_d) ) { - fprintf(stderr, "chop: failed to initialize iterator\r\n"); - exit(1); + // delete all but the last two epochs + // + // XX parameterize the number of epochs to chop + // + for ( c3_z i_z = 2; i_z < len_z; i_z++ ) { + fprintf(stderr, "chop: deleting epoch 0i%" PRIc3_d "\r\n", sot_d[i_z]); + if ( c3y != u3_disk_epoc_kill(log_u, sot_d[i_z]) ) { + fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", sot_d[i_z]); + exit(1); + } } - if ( c3n == u3_lmdb_walk_next(&itr_u, &len_i, buf_v) ) { - fprintf(stderr, "chop: failed to read event\r\n"); - exit(1); + + // cleanup + c3_free(sot_d); + u3_disk_exit(log_u); + + // success + fprintf(stderr, "chop: event log truncation complete\r\n"); +} + +/* _cw_roll(): rollover to new epoch + */ +static void +_cw_roll(c3_i argc, c3_c* argv[]) +{ + c3_i ch_i, lid_i; + c3_w arg_w; + + static struct option lop_u[] = { + { "loom", required_argument, NULL, c3__loom }, + { NULL, 0, NULL, 0 } + }; + + u3_Host.dir_c = _main_pier_run(argv[0]); + + while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { + switch ( ch_i ) { + case c3__loom: { + if (_main_readw_loom("loom", &u3_Host.ops_u.lom_y)) { + exit(1); + } + } break; + + case '?': { + fprintf(stderr, "invalid argument\r\n"); + exit(1); + } break; + } } - u3_lmdb_walk_done(&itr_u); - // initialize a fresh lmdb environment in the "chop" subdir - c3_c cho_c[8193]; - snprintf(cho_c, sizeof(cho_c), "%s/chop", log_c); - if ( 0 != access(cho_c, F_OK) ) { - if ( 0 != c3_mkdir(cho_c, 0700) ) { - fprintf(stderr, "chop: failed to create chop directory\r\n"); + // argv[optind] is always "roll" + // + + if ( !u3_Host.dir_c ) { + if ( optind + 1 < argc ) { + u3_Host.dir_c = argv[optind + 1]; + } + else { + fprintf(stderr, "invalid command, pier required\r\n"); exit(1); } - } - MDB_env* new_u = u3_lmdb_init(cho_c, siz_i); - if ( !new_u ) { - fprintf(stderr, "chop: failed to initialize new database\r\n"); - exit(1); + + optind++; } - // write the metadata to the database - if ( c3n == u3_disk_save_meta(new_u, who_d, fak_o, lif_w) ) { - fprintf(stderr, "chop: failed to save metadata\r\n"); + if ( optind + 1 != argc ) { + fprintf(stderr, "invalid command\r\n"); exit(1); } - // write the last event to the database - // warning: this relies on the old database still being open - if ( c3n == u3_lmdb_save(new_u, las_d, 1, buf_v, &len_i) ) { - fprintf(stderr, "chop: failed to write last event\r\n"); + // gracefully shutdown the pier if it's running + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); + + u3_disk_kindly(log_u, u3_Host.eve_d); + + // check if there's a *current* snapshot + if ( log_u->dun_d != u3A->eve_d ) { + fprintf(stderr, "roll: error: snapshot is out of date, please " + "start/shutdown your pier gracefully first\r\n"); + fprintf(stderr, "roll: eve_d: %" PRIc3_d ", dun_d: %" PRIc3_d "\r\n", \ + u3A->eve_d, log_u->dun_d); exit(1); } - // backup the original database file - c3_c dat_c[8193], bak_c[8193]; - snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", log_c); - // "data_-.mdb.bak" - snprintf(bak_c, sizeof(bak_c), "%s/data_%" PRIu64 "-%" PRIu64 ".mdb.bak", cho_c, fir_d, las_d); - if ( 0 != c3_rename(dat_c, bak_c) ) { - fprintf(stderr, "chop: failed to backup original database file\r\n"); + // create new epoch + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "roll: failed to get first/last events\r\n"); exit(1); } - // rename new database file to be official - c3_c new_c[8193]; - snprintf(new_c, sizeof(new_c), "%s/data.mdb", cho_c); - if ( 0 != c3_rename(new_c, dat_c) ) { - fprintf(stderr, "chop: failed to rename new database file\r\n"); + if ( fir_d == las_d ) { + fprintf(stderr, "roll: latest epoch already empty\r\n"); + exit(0); + } + else if ( c3n == u3_disk_epoc_init(log_u, las_d) ) { + fprintf(stderr, "roll: failed to create new epoch\r\n"); exit(1); } - // cleanup - u3_disk_exit(old_u); - u3_lmdb_exit(new_u); - u3m_stop(); - - // success - fprintf(stderr, "chop: event log truncation complete\r\n"); - fprintf(stderr, " event log backup written to %s\r\n", bak_c); - fprintf(stderr, " WARNING: ENSURE YOU CAN RESTART YOUR SHIP BEFORE DELETING YOUR EVENT LOG BACKUP FILE!\r\n"); - fprintf(stderr, " if you can't, restore your log by running:\r\n"); - fprintf(stderr, " `mv %s %s` then try again\r\n", bak_c, dat_c); + // success + c3_d epo_d = log_u->dun_d + 1; + fprintf(stderr, "roll: epoch rollover complete\r\n"); } /* _cw_vere(): download vere @@ -2823,6 +2888,7 @@ _cw_utils(c3_i argc, c3_c* argv[]) case c3__prep: _cw_prep(argc, argv); return 2; // continue on case c3__queu: _cw_queu(argc, argv); return 1; case c3__chop: _cw_chop(argc, argv); return 1; + case c3__roll: _cw_roll(argc, argv); return 1; case c3__vere: _cw_vere(argc, argv); return 1; case c3__vile: _cw_vile(argc, argv); return 1; @@ -3012,6 +3078,13 @@ main(c3_i argc, } } + // we need the current snapshot's latest event number to + // validate whether we can execute disk migration + if ( u3_Host.ops_u.nuu == c3n ) { + _cw_play_impl(0, 0, c3n, c3n, c3n); + // XX unmap loom, else parts of the snapshot could be left in memory + } + // starting u3m configures OpenSSL memory functions, so we must do it // before any OpenSSL allocations // diff --git a/pkg/vere/mars.c b/pkg/vere/mars.c index 37933bd49e..5e940182dc 100644 --- a/pkg/vere/mars.c +++ b/pkg/vere/mars.c @@ -6,6 +6,8 @@ #include "noun.h" #include "types.h" #include "vere.h" +#include "ivory.h" +#include "ur.h" #include "db/lmdb.h" #include #include @@ -179,6 +181,29 @@ _mars_do_boot(u3_disk* log_u, c3_d eve_d) // eve = u3m_love(u3ke_cue(u3ke_jam(eve))); + // install an ivory pill to support stack traces + // + // XX support -J + // + { + c3_d len_d = u3_Ivory_pill_len; + c3_y* byt_y = u3_Ivory_pill; + u3_cue_xeno* sil_u = u3s_cue_xeno_init_with(ur_fib27, ur_fib28); + u3_weak pil; + + if ( u3_none == (pil = u3s_cue_xeno_with(sil_u, len_d, byt_y)) ) { + u3l_log("lite: unable to cue ivory pill"); + exit(1); + } + + u3s_cue_xeno_done(sil_u); + + if ( c3n == u3v_boot_lite(pil)) { + u3l_log("lite: boot failed"); + exit(1); + } + } + u3l_log("--------------- bootstrap starting ----------------"); u3l_log("boot: 1-%u", u3qb_lent(eve)); @@ -195,10 +220,11 @@ _mars_do_boot(u3_disk* log_u, c3_d eve_d) /* u3_mars_play(): replay up to [eve_d], snapshot every [sap_d]. */ -void +c3_d u3_mars_play(u3_mars* mar_u, c3_d eve_d, c3_d sap_d) { u3_disk* log_u = mar_u->log_u; + c3_d pay_d = 0; if ( !eve_d ) { eve_d = log_u->dun_d; @@ -207,17 +233,18 @@ u3_mars_play(u3_mars* mar_u, c3_d eve_d, c3_d sap_d) u3l_log("mars: already computed %" PRIu64 "", eve_d); u3l_log(" state=%" PRIu64 ", log=%" PRIu64 "", mar_u->dun_d, log_u->dun_d); - return; + return pay_d; } else { eve_d = c3_min(eve_d, log_u->dun_d); } if ( mar_u->dun_d == log_u->dun_d ) { - u3l_log("mars: nothing to do!"); - return; + return pay_d; } + pay_d = eve_d - mar_u->dun_d; + if ( !mar_u->dun_d ) { c3_w lif_w; @@ -351,4 +378,6 @@ u3_mars_play(u3_mars* mar_u, c3_d eve_d, c3_d sap_d) u3l_log("---------------- playback complete ----------------"); u3m_save(); + + return pay_d; } diff --git a/pkg/vere/mars.h b/pkg/vere/mars.h index ba97f63614..552f1a9dea 100644 --- a/pkg/vere/mars.h +++ b/pkg/vere/mars.h @@ -15,7 +15,7 @@ /* u3_mars_play(): replay up to [eve_d], snapshot every [sap_d]. */ - void + c3_d u3_mars_play(u3_mars* mar_u, c3_d eve_d, c3_d sap_d); #endif /* ifndef U3_VERE_MARS_H */ diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 8e54dbb37e..907e33c319 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -783,7 +783,7 @@ _pier_on_lord_wyrd_bail(void* ptr_v, u3_ovum* egg_u, u3_noun lud) #endif } -/* _pier_wyrd_init(): construct %wyrd. +/* _pier_wyrd_card(): construct %wyrd. */ static u3_noun _pier_wyrd_card(u3_pier* pir_u) @@ -1402,6 +1402,8 @@ _pier_on_lord_live(void* ptr_v) // XX print bootstrap commit complete // XX s/b boot_complete_cb // + // XX this codepath should never be hit due to sync replay + u3l_log("pier: warning: async replay"); _pier_play_init(pir_u, log_u->dun_d); } } @@ -1641,6 +1643,8 @@ _pier_init(c3_w wag_w, c3_c* pax_c) c3_free(pir_u); return 0; } + + u3_assert( U3D_VER1 == pir_u->log_u->ver_w ); } // initialize compute diff --git a/pkg/vere/serf.c b/pkg/vere/serf.c index 3a59005e00..cfbbc47be3 100644 --- a/pkg/vere/serf.c +++ b/pkg/vere/serf.c @@ -3,6 +3,8 @@ #include "noun.h" #include "vere.h" +#include "ivory.h" +#include "ur.h" /* |% @@ -538,11 +540,28 @@ _serf_play_life(u3_serf* sef_u, u3_noun eve) u3z(len); } - // ensure zero-initialized kernel + // install an ivory pill to support stack traces // - // XX assert? + // XX support -J // - u3A->roc = 0; + { + c3_d len_d = u3_Ivory_pill_len; + c3_y* byt_y = u3_Ivory_pill; + u3_cue_xeno* sil_u = u3s_cue_xeno_init_with(ur_fib27, ur_fib28); + u3_weak pil; + + if ( u3_none == (pil = u3s_cue_xeno_with(sil_u, len_d, byt_y)) ) { + u3l_log("lite: unable to cue ivory pill"); + exit(1); + } + + u3s_cue_xeno_done(sil_u); + + if ( c3n == u3v_boot_lite(pil)) { + u3l_log("lite: boot failed"); + exit(1); + } + } gon = u3m_soft(0, u3v_life, eve); diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index 2bf976c1dd..9527246e19 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -8,6 +8,7 @@ #include "noun.h" #include "serf.h" #include "uv.h" +#include /** Quasi-tunable parameters. **/ @@ -130,6 +131,7 @@ c3_c* pax_c; // path of directory uv_file fil_u; // file, opened read-only to fsync u3_dent* all_u; // file list + u3_dent* dil_u; // directory list } u3_dire; /* u3_save: checkpoint control. @@ -314,6 +316,7 @@ typedef struct _u3_host { c3_w kno_w; // current executing stage c3_c* dir_c; // pier path (no trailing /) + c3_d eve_d; // initial current snapshot c3_c* dem_c; // daemon executable path c3_c* wrk_c; // worker executable path c3_d now_d; // event tick @@ -539,9 +542,11 @@ u3_dire* urb_u; // urbit system data u3_dire* com_u; // log directory c3_o liv_o; // live - void* mdb_u; // lmdb environment. + c3_w ver_w; // pier version + void* mdb_u; // lmdb env of current epoch c3_d sen_d; // commit requested c3_d dun_d; // committed + c3_d epo_d; // current epoch number u3_disk_cb cb_u; // callbacks u3_read* red_u; // read requests union { // write thread/request @@ -1003,6 +1008,31 @@ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); + /* u3_disk_epoc_init(): create new epoch. + */ + c3_o + u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d); + + /* u3_disk_epoc_kill(): delete an epoch. + */ + c3_o + u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d); + + /* u3_disk_epoc_last(): get latest epoch number. + */ + c3_o + u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d); + + /* u3_disk_epoc_list: get descending epoch numbers, "mcut" pattern. + */ + c3_z + u3_disk_epoc_list(u3_disk* log_u, c3_d* sot_d); + + /* u3_disk_kindly(): do the needful. + */ + void + u3_disk_kindly(u3_disk* log_u, c3_d eve_d); + /* u3_disk_read_list(): synchronously read a cons list of events. */ u3_weak diff --git a/pkg/vere/ward.c b/pkg/vere/ward.c index 4c90be7179..9d6c8cbed3 100644 --- a/pkg/vere/ward.c +++ b/pkg/vere/ward.c @@ -39,6 +39,7 @@ u3_dire_init(const c3_c* pax_c) { u3_dire *dir_u = c3_malloc(sizeof *dir_u); dir_u->all_u = 0; + dir_u->dil_u = 0; dir_u->pax_c = c3_malloc(1 + strlen(pax_c)); strcpy(dir_u->pax_c, pax_c); @@ -61,6 +62,17 @@ u3_dire_free(u3_dire *dir_u) } } + { + u3_dent *det_u = dir_u->dil_u; + u3_dent *nex_u; + + while ( det_u ) { + nex_u = det_u->nex_u; + u3_dent_free(det_u); + det_u = nex_u; + } + } + c3_free(dir_u->pax_c); c3_free(dir_u); }