Implement freelists and displacement hashmaps. Garbage collector unfinished. Builds?

src/gc.rs

@@ -0,0 +1,286 @@
+use std::hash::{Hash, Hasher};
+use std::marker::{PhantomData};
+use std::ops::{Deref, DerefMut};
+
+type Index = u32;
+
+pub struct FreeList<T> {
+  items: Vec<Option<Header<T>>>,
+  free: Vec<Index>,
+  count: u32,
+}
+
+#[derive(Clone,Debug)]
+pub struct AnyId {
+  pub(crate) index: Index
+}
+
+impl PartialEq for AnyId {
+  fn eq(&self, other: &Self) -> bool {
+    self.index == other.index
+  }
+}
+
+impl<T> From<AnyId> for Id<T> {
+  fn from(value: AnyId) -> Self {
+    Id {
+      index: value.index,
+      phantom_data: PhantomData,
+    }
+  }
+}
+
+#[derive(Debug)]
+pub struct Id<T> {
+  pub(crate) index: Index,
+  phantom_data: PhantomData<T>,
+}
+
+pub struct RefId<'a, T> {
+  pub id: Id<T>,
+  it: &'a mut T
+}
+
+impl<'a, T> Deref for RefId<'a, T> {
+  type Target = T;
+
+  fn deref(&self) -> &Self::Target {
+    self.it
+  }
+}
+
+impl<'a, T> DerefMut for RefId<'a, T> {
+
+  fn deref_mut(&mut self) -> &mut Self::Target {
+    self.it
+  }
+}
+
+impl<T> From<RefId<'_, T>> for Id<T> {
+  fn from(value: RefId<T>) -> Self {
+    value.id
+  }
+}
+
+impl<T> Clone for Id<T> {
+  fn clone(&self) -> Self {
+    Id {
+      index: self.index,
+      phantom_data: PhantomData::default(),
+    }
+  }
+}
+
+impl<T> PartialEq for Id<T> {
+  fn eq(&self, other: &Self) -> bool {
+    self.index == other.index
+  }
+}
+
+impl<T> Eq for Id<T> {}
+
+impl<T> Hash for Id<T> {
+  fn hash<H: Hasher>(&self, state: &mut H) {
+    self.index.hash(state)
+  }
+}
+
+impl<T> Id<T> {
+  pub fn maybe(self) -> MaybeId<T> {
+    MaybeId(self)
+  }
+  pub fn any(self) -> AnyId { AnyId {
+    index: self.index
+  }  }
+}
+
+#[derive(Debug)]
+pub struct MaybeId<T>(Id<T>);
+
+impl<T> From<Id<T>> for MaybeId<T> {
+  fn from(value: Id<T>) -> Self {
+    MaybeId(value)
+  }
+}
+
+impl<T> Clone for MaybeId<T> {
+  fn clone(&self) -> Self {
+    if self.0.index != u32::MAX {
+      MaybeId(self.0.clone())
+    } else {
+      MaybeId::NULL
+    }
+  }
+}
+
+impl<T> PartialEq for MaybeId<T> {
+  fn eq(&self, other: &Self) -> bool {
+    self.0 == other.0
+  }
+}
+
+impl<T> MaybeId<T> {
+  pub const NULL: MaybeId<T> = MaybeId(Id { index: u32::MAX, phantom_data: PhantomData {}, });
+  pub fn unwrap(self) -> Id<T> {
+    if self.0.index == u32::MAX {
+      panic!()
+    } else {
+      self.0
+    }
+  }
+  pub fn exists(&self) -> Option<Id<T>> {
+    if self.0.index == u32::MAX {
+      None
+    } else {
+      Some(self.0.clone())
+    }
+  }
+}
+
+impl<T> Default for FreeList<T> {
+  fn default() -> Self {
+    Self::new()
+  }
+}
+
+impl<T> FreeList<T> {
+  pub fn new() -> FreeList<T> {
+    FreeList {
+      items: Vec::new(),
+      free: Vec::new(),
+      count: 0,
+    }
+  }
+  pub fn get(&mut self, id: &Id<T>) -> &mut T {
+    &mut self.items[id.index as usize].as_mut().unwrap().it
+  }
+  pub fn get_ref(&mut self, id: Id<T>) -> RefId<'_, T> {
+    let it = self.get(&id);
+    RefId { id, it }
+  }
+  pub fn remove(&mut self, id: &Id<T>) -> T {
+    self.free.push(id.index);
+    self.items[id.index as usize].take().unwrap().it
+  }
+  pub fn make(&mut self, value: T) -> RefId<'_, T> { // todo: shouldn't panic if allocation fails
+    if let Some(free) = self.free.pop() {
+      self.items[free as usize] = Some(Header::new(value));
+      RefId {
+        id: Id {
+          index: free,
+          phantom_data: PhantomData,
+        },
+        it: &mut self.items[free as usize].as_mut().unwrap().it,
+      }
+    } else {
+      self.items.push(Some(Header::new(value)));
+      let index = (self.items.len() - 1) as Index;
+      RefId {
+        id: Id {
+          index,
+          phantom_data: PhantomData,
+        },
+        it: &mut self.items[index as usize].as_mut().unwrap().it,
+      }
+    }
+  }
+}
+
+pub struct IdNode<T> {
+  item: T,
+  next: MaybeId<IdNode<T>>,
+}
+
+impl<T,C> Traverse<C> for IdNode<T> where C: Allocates<Self> {
+  fn traverse(id: &Id<IdNode<T>>, context: &mut C) {
+    if let Some(ref next) = context.freelist().get(&id).next.exists() {
+      IdNode::traverse(next,context)
+    }
+  }
+}
+
+#[derive(Clone)]
+pub struct LinkedList<T> {
+  first: MaybeId<IdNode<T>>,
+  count: Index,
+}
+
+impl<T> Default for LinkedList<T> {
+  fn default() -> Self {
+    LinkedList {
+      first: MaybeId::NULL,
+      count: 0,
+    }
+  }
+}
+
+impl<T> LinkedList<T> {
+  pub fn new() -> LinkedList<T> {
+    LinkedList {
+      first: MaybeId::NULL,
+      count: 0,
+    }
+  }
+  pub fn add(&mut self, list: &mut FreeNodeList<T>, it: T) {
+    let old = std::mem::replace(&mut self.first,list.make(IdNode {
+      item: it,
+      next: MaybeId::NULL,
+    }).id.maybe());
+    list.get(&self.first.clone().unwrap()).next = old; // todo: this is painful
+    self.count += 1;
+  }
+  pub fn remove<F>(&mut self, list: &mut FreeNodeList<T>, criteria: F) -> T where T: PartialEq, F: Fn(&T) -> bool {
+    // Find some occurrence of 'item' in the list, remove and restitch the list, returning its id
+    let mut maybe_last = MaybeId::NULL;
+    let mut maybe_this = self.first.clone();
+    while let Some(this) = maybe_this.exists() {
+      let node = list.remove(&this);
+      let next = node.next.clone();
+      if criteria(&node.item) {
+        self.count -= 1;
+        if let Some(last) = maybe_last.exists() {
+          list.get(&last).next = node.next.clone();
+        } else {
+          self.first = node.next.clone();
+        }
+        return node.item;
+      }
+      maybe_last = maybe_this;
+      maybe_this = next;
+    }
+    panic!()
+  }
+}
+
+pub type FreeNodeList<T> = FreeList<IdNode<T>>;
+
+#[derive(Clone, Debug, Copy)]
+enum State {
+  Cold,
+  Warm,
+}
+
+#[derive(Debug, Clone)]
+struct Header<T> {
+  state: State,
+  it: T,
+}
+
+impl<T> Header<T> where {
+  fn new(it: T) -> Header<T> {
+    Header {
+      state: State::Cold,
+      it,
+    }
+  }
+}
+
+pub trait Allocates<T> where Self: Sized, T: Traverse<Self> {
+  fn collect();
+  fn freelist<'a>(&'a mut self) -> &'a mut FreeList<T>;
+}
+
+pub trait Traverse<C>: Sized {
+  fn traverse(id: &Id<Self>, context: &mut C);
+  fn collect(self);
+}

src/lib.rs

@@ -0,0 +1,2 @@
+pub mod gc;
+pub mod map;

src/map.rs

@@ -0,0 +1,297 @@
+// I've already spent 2 months on this interpreter (now GC!?), and I'm tired, so I've cut a few corners...
+/* See some of Lua's notes on this table: https://www.lua.org/source/5.5/ltable.c.html
+This table uses an 'open-addressed' hashmap https://en.wikipedia.org/wiki/Open_addressing
+where collisions are handled by 'linear-probing' and something I call 'displacement'.
+
+Motivation:
+Afaik, without having some mechanism that records or ensures known proximity between a displaced element
+that collided with other(s) in the hashmap and it's original home (at element's hash's index),
+the entire array may have to be checked just to see if an element is present. How can we fix this?
+
+Solution:
+Lua uses Brent's method, which I couldn't find a concrete explanation of, here I'm just recording
+the maximum displacement that will need to be linearly-probed from the home entry to be certain of
+any 'home' element's presence in the greater array.
+
+Notes:
+- If this value becomes large, the load factor is probably high and there would be a resize.
+- Values would only need to be shuffled around closer to their 'home' during a resize.
+- It is unlikely the displacement would large value without a resize amending that problem.
+ */
+use std::hash::{DefaultHasher, Hash, Hasher};
+use crate::gc::{AnyId, Id};
+
+type Displacement = u8;
+
+#[cfg(feature = "bighash")]
+pub type Hashed = u64;
+#[cfg(not(feature = "bighash"))]
+pub type Hashed = u32;
+
+pub trait Equivalent<T>: Sized {
+  fn matches(&self, other: &T) -> bool;
+}
+
+impl<T> Equivalent<T> for T
+where
+  T: PartialEq,
+{
+  fn matches(&self, other: &T) -> bool {
+    self == other
+  }
+}
+
+pub trait Hashes {
+  fn hashed(&self) -> Hashed;
+}
+
+impl Hashes for &str {
+  fn hashed(&self) -> Hashed {
+    let mut hasher = DefaultHasher::new();
+    self[..self.len().min(6)].hash(&mut hasher);
+    hasher.finish() as Hashed
+  }
+}
+
+impl<T> Hashes for Id<T> {
+  fn hashed(&self) -> Hashed {
+    let mut hasher = DefaultHasher::new();
+    self.index.hash(&mut hasher);
+    hasher.finish() as Hashed
+  }
+}
+
+impl Hashes for AnyId {
+  fn hashed(&self) -> Hashed {
+    let mut hasher = DefaultHasher::new();
+    self.index.hash(&mut hasher);
+    hasher.finish() as Hashed
+  }
+}
+
+pub trait TableEntry: Equivalent<Self> + Hashes {
+  fn new_vacant() -> Self;
+  fn is_vacant(&self) -> bool;
+}
+
+#[derive(Debug, Clone)]
+pub struct Entry<E> {
+  pub entry: E,
+  home: usize,
+  displacement: Displacement,
+}
+
+#[derive(Clone, Debug)]
+pub struct Map<E: TableEntry + Clone> {
+  // default representing vacancy!
+  pub map: Vec<Entry<E>>, // todo: make this private and use an iterator
+  map_bounds: std::ops::Range<usize>,
+  map_count: usize, // number of elements in table
+}
+
+impl<E: TableEntry + Clone> Map<E> {
+  fn exchange_table(&mut self, len: usize) {
+    let old = std::mem::replace(
+      &mut self.map,
+      vec![
+        Entry {
+          entry: E::new_vacant(),
+          home: 0,
+          displacement: 0,
+        };
+        len
+      ],
+    );
+    self.map_bounds = self.table_lower()..self.table_upper();
+    for Entry { entry, .. } in old {
+      if !entry.is_vacant() {
+        self.set_table(entry)
+      }
+    }
+  }
+  pub fn resize_table(&mut self, len: usize) {
+    self.exchange_table(len.max(self.map_count + (self.map_count / 3)));
+    self.map_bounds = 0..self.table_upper();
+  }
+  fn table_upper(&self) -> usize {
+    (self.map.len() / 4) * 3
+  }
+  fn table_lower(&self) -> usize {
+    self.map.len() / 3
+  }
+  fn ensure_table(&mut self) {
+    if self.map_count + 1 > self.table_upper() {
+      // free space is short
+      self.exchange_table((self.map.len() + 4) * 2);
+    } else if self.map_count < self.table_lower() {
+      // too much free space
+      self.exchange_table(self.map.len() / 2)
+    }
+  }
+  pub fn set_table(&mut self, entry: E) {
+    #[cfg(feature = "assertions")]
+    assert!(!entry.is_vacant());
+    if !self.map_bounds.contains(&(self.map_count + 1)) {
+      // make sure the table is appropriately sized
+      self.ensure_table();
+    }
+    let range = self.map.len();
+    let home = entry.hashed() as usize % range;
+    if self.map[home].entry.is_vacant() {
+      // attempt to place it directly in an empty space
+      // INSERT
+      self.map[home].home = home;
+      self.map[home].entry = entry;
+      self.map[home].displacement = self.map[home].displacement.max(0);
+      self.map_count += 1;
+    } else if self.map[home].entry.matches(&entry) {
+      // attempt to replace it directly
+      // REPLACE
+      self.map[home].entry = entry; // home.item.soft_drop()
+    } else {
+      // attempt to replace it in a collided neighbour location
+      for i in 1..self.map[home].displacement + 1 {
+        let neighbour = &mut self.map[(home + i as usize) % range]; // todo: is mod expensive?
+        if neighbour.entry.matches(&entry) {
+          // found where it was displaced to
+          // REPLACE
+          neighbour.entry = entry;
+          return;
+        }
+      }
+      // at this point it must be added, probe to place in an empty space
+      for j in self.map[home].displacement + 1..Displacement::MAX - 1 {
+        let neighbour = &mut self.map[(home + j as usize) % range];
+        if neighbour.entry.is_vacant() {
+          // new empty slot hooray!
+          // INSERT
+          neighbour.entry = entry;
+          neighbour.home = home;
+          self.map[home].displacement = j;
+          self.map_count += 1;
+          return;
+        }
+      }
+      // impossible but this still needs to be complete
+      #[cfg(feature = "messages")]
+      eprintln!("Large table collision, are the hashes ok?\n\tBrute-force probing...");
+      let mut free: Option<usize> = None;
+      for k in 0..self.map.len() {
+        let neighbour = &mut self.map[k];
+        if neighbour.entry.is_vacant() && free.is_none() {
+          free = Some(k);
+        } else if neighbour.entry.matches(&entry) {
+          // REPLACE
+          neighbour.entry = entry;
+          return;
+        }
+      }
+      if let Some(k) = free {
+        // INSERT
+        self.map[k].home = home;
+        self.map[k].entry = entry;
+        self.map[home].displacement = Displacement::MAX;
+        self.map_count += 1;
+        return;
+      }
+      // must resize
+      #[cfg(feature = "messages")]
+      eprintln!("\n\tResizing...");
+      self.resize_table((self.map.len() + 4) * 2);
+      self.set_table(entry);
+    }
+  }
+  pub fn rem_table<K>(&mut self, index: K)
+  where
+    E: Equivalent<K>,
+    K: Hashes,
+  {
+    if self.map.len() == 0 {
+      return;
+    }
+    let range = self.map.len();
+    let home = index.hashed() as usize % range;
+    if self.map[home].entry.matches(&index) {
+      self.map[home].entry = E::new_vacant();
+      self.map_count -= 1;
+    } else {
+      if self.map[home].displacement != Displacement::MAX {
+        let mut largest = 0;
+        for i in 1..self.map[home].displacement {
+          let neighbour = &mut self.map[(home + i as usize) % range]; // todo: maybe mod slows this down
+          if neighbour.entry.matches(&index) {
+            neighbour.entry = E::new_vacant();
+            self.map_count -= 1;
+            if i == self.map[home].displacement {
+              self.map[home].displacement = largest
+            }
+            return;
+          } else if !neighbour.entry.is_vacant() && neighbour.home == home {
+            largest = i;
+          }
+        }
+      } else {
+        let mut largest = 0;
+        #[cfg(feature = "assertions")]
+        let mut finished = false; // what
+        for k in 0..self.map.len() {
+          let neighbour = &mut self.map[k];
+          if neighbour.entry.matches(&index) {
+            neighbour.entry = E::new_vacant();
+            self.map_count -= 1;
+            #[cfg(feature = "assertions")]
+            {
+              assert!(!finished);
+              finished = true;
+            }
+          }
+          if neighbour.home == home && !neighbour.entry.is_vacant() {
+            largest = largest.max(if k < home {
+              k + self.map.len() - home - 1 // ?
+            } else {
+              k - home
+            })
+          }
+        }
+        self.map[home].displacement = if largest > Displacement::MAX as usize {
+          Displacement::MAX
+        } else {
+          largest as Displacement
+        }
+      }
+    }
+  }
+  pub fn get_table<K>(&mut self, index: K) -> Option<&E>
+  where
+    E: Equivalent<K>,
+    K: Hashes,
+  {
+    if self.map.len() == 0 {
+      return None;
+    }
+    let location = index.hashed() as usize % self.map.len();
+    let home = &self.map[location];
+    if home.entry.matches(&index) {
+      Some(&home.entry)
+    } else {
+      for i in 1..home.displacement as usize {
+        let neighbour = &self.map[(location + i) % self.map.len()];
+        if neighbour.entry.matches(&index) {
+          return Some(&neighbour.entry);
+        }
+      }
+      None
+    }
+  }
+  pub fn new() -> Map<E> {
+    Map {
+      map: Vec::new(),
+      map_count: 0,
+      map_bounds: (0..0).into(),
+    }
+  }
+}
+
+pub struct TableEntry
+
+pub type Table<K,T> = Map<>