u3: adjust free-list sizes (#539)

This PR dedicates slot 0 to cells (`u3a_minimum`), and aligns subsequent
slots on power-of-two boundaries. Previously, slot 0 was unused, and
subsequent slots were aligned *near* power-of-two boundaries. For
example:

| words | old slot | new slot |
| --- | - | - |
|   6 | 1 | 0 |
|  31 | 3 | 2 |
|  61 | 4 | 3 |
|  62 | 4 | 3 |
|  63 | 4 | 3 |
| 121 | 5 | 4 |
| 122 | 5 | 4 |
| 123 | 5 | 4 |
| 124 | 5 | 4 |
| 125 | 5 | 4 |
| 126 | 5 | 4 |
| 127 | 5 | 4 |

The first issue dates to urbit/urbit#987. Slot 0 was reserved for sizes
less than 8, which meant that 7-word allocations on the home road were
incredibly slow (as they traversed a free list full of 6 word
allocations). The second issue has always been the case in this
allocator, due to the way that the size was rounded up on each iteration
of the "power-of-two" sizing loop.

This PR does not address the infamous size-bumping logic in the
allocator:


https://github.com/urbit/vere/blob/9bdc1af00f6650e27ac66793c6781d933dd30dfe/pkg/noun/allocate.c#L453-L467

I'm been tempted to disable that behavior on the home-road (to further
reduce fragmentation), but I'm not convinced that won't still run into
pathological performance issues. The best approach might be to start
searching the "proper" free list, but bound the number of iterations
before bumping. Either change can be made at any time, without
migrations.

This PR does requires a migration to move free space into the
now-appropriate free-list. It includes a trivial, always on migration,
which should be refactored into #508 when appropriate.

This PR helps somewhat with urbit/urbit#6805, as it reduces home-road
fragmentation.

pkg/noun/allocate.c

@@ -67,28 +67,29 @@ _box_count(c3_ws siz_ws) { }
   } while(0)
 
 /* _box_slot(): select the right free list to search for a block.
-   TODO: do we really need a loop to do this?
-
-   so our free list logic looks like this:
-   siz_w < 6 words then [0]
-   siz_w < 16      then [1]
-   siz_w < 32      then [2]
-   siz_w < 64      then [3]
-   ...
-   siz_w > 4G      then [26]
+**
+**   siz_w ==  6 words then  [0]
+**   siz_w  < 16       then  [1]
+**   siz_w  < 32       then  [2]
+**   siz_w  < 64       then  [3]
+**   ...
+**   siz_w >=  2GB     then [26]
 */
 static c3_w
 _box_slot(c3_w siz_w)
 {
-  if ( siz_w < u3a_minimum ) {
+  if ( u3a_minimum == siz_w ) {
     return 0;
   }
-
-  for (c3_w i_w = 1; i_w < u3a_fbox_no; i_w++) {
-    if ( siz_w < 16 ) return i_w;
-    siz_w = (siz_w + 1) >> 1;
+  else if ( !(siz_w >> 4) ) {
+    c3_dessert( u3a_minimum < siz_w );
+    return 1;
+  }
+  else {
+    c3_w bit_w = c3_bits_word(siz_w) - 3;
+    c3_w max_w = u3a_fbox_no - 1;
+    return c3_min(bit_w, max_w);
   }
-  return u3a_fbox_no - 1;
 }
 
 /* _box_make(): construct a box.
@@ -1113,9 +1114,7 @@ _ca_take_atom(u3a_atom* old_u)
 static inline u3_cell
 _ca_take_cell(u3a_cell* old_u, u3_noun hed, u3_noun tel)
 {
-  //  XX use u3a_celloc?
-  //
-  c3_w*     new_w = u3a_walloc(c3_wiseof(u3a_cell));
+  c3_w*     new_w = u3a_celloc();
   u3a_cell* new_u = (u3a_cell*)(void *)new_w;
   u3_cell     new = u3a_to_pom(u3a_outa(new_u));
 
@@ -2299,6 +2298,69 @@ u3a_idle(u3a_road* rod_u)
   return fre_w;
 }
 
+/* u3a_ream(): ream free-lists.
+*/
+void
+u3a_ream(void)
+{
+  u3p(u3a_fbox) lit_p;
+  u3a_fbox*     fox_u;
+  c3_w     sel_w, i_w;
+
+  for ( i_w = 0; i_w < u3a_fbox_no; i_w++ ) {
+    lit_p = u3R->all.fre_p[i_w];
+
+    while ( lit_p ) {
+      fox_u = u3to(u3a_fbox, lit_p);
+      lit_p = fox_u->nex_p;
+      sel_w = _box_slot(fox_u->box_u.siz_w);
+
+      if ( sel_w != i_w ) {
+        //  inlined _box_detach()
+        //
+        {
+          u3p(u3a_fbox) fre_p = u3of(u3a_fbox, &(fox_u->box_u));
+          u3p(u3a_fbox) pre_p = u3to(u3a_fbox, fre_p)->pre_p;
+          u3p(u3a_fbox) nex_p = u3to(u3a_fbox, fre_p)->nex_p;
+
+          if ( nex_p ) {
+            if ( u3to(u3a_fbox, nex_p)->pre_p != fre_p ) {
+              u3_assert(!"loom: corrupt");
+            }
+            u3to(u3a_fbox, nex_p)->pre_p = pre_p;
+          }
+          if ( pre_p ) {
+            if( u3to(u3a_fbox, pre_p)->nex_p != fre_p ) {
+              u3_assert(!"loom: corrupt");
+            }
+            u3to(u3a_fbox, pre_p)->nex_p = nex_p;
+          }
+          else {
+            if ( fre_p != u3R->all.fre_p[i_w] ) {
+              u3_assert(!"loom: corrupt");
+            }
+            u3R->all.fre_p[i_w] = nex_p;
+          }
+        }
+
+        //  inlined _box_attach()
+        {
+          u3p(u3a_fbox)  fre_p = u3of(u3a_fbox, &(fox_u->box_u));
+          u3p(u3a_fbox)* pfr_p = &u3R->all.fre_p[sel_w];
+          u3p(u3a_fbox)  nex_p = *pfr_p;
+
+          u3to(u3a_fbox, fre_p)->pre_p = 0;
+          u3to(u3a_fbox, fre_p)->nex_p = nex_p;
+          if ( nex_p ) {
+            u3to(u3a_fbox, nex_p)->pre_p = fre_p;
+          }
+          (*pfr_p) = fre_p;
+        }
+      }
+    }
+  }
+}
+
 /* u3a_sweep(): sweep a fully marked road.
 */
 c3_w

pkg/noun/allocate.h

@@ -662,6 +662,11 @@
           c3_w
           u3a_idle(u3a_road* rod_u);
 
+        /* u3a_ream(): ream free-lists.
+        */
+          void
+          u3a_ream(void);
+
         /* u3a_sweep(): sweep a fully marked road.
         */
           c3_w

pkg/noun/v3/allocate.h

@@ -11,6 +11,7 @@
 #     define u3R_v3              u3a_Road
 #     define u3a_v3_balign       u3a_balign
 #     define u3a_v3_road         u3a_road
+#     define u3a_v3_ream         u3a_ream
 #     define u3a_v3_walign       u3a_walign
 #     define u3a_v3_walloc       u3a_walloc
 

pkg/noun/v3/manage.c

@@ -100,6 +100,9 @@ u3m_v3_migrate()
   u3R_v3 = &u3H_v3->rod_u;
   u3H_v3->ver_w = U3V_VER3;
 
+  //  recalculate free lists
+  u3a_v3_ream();
+
   //  initialize persistent cache
   u3R_v3->cax.per_p = u3h_v3_new_cache(u3C.per_w);