[redo documentation of internals, clean up terminology dealing with smart pointers
John Meacham <john@repetae.net>**20100318230404
 Ignore-this: c0bd01a9d8129db5d4097cf2ba86e9c9
] hunk ./src/C/FromGrin2.hs 567
-    let tmp' = cast nt (f_DETAG v') -- (if vv < v0 then f_DETAG v' else v')
+    let tmp' = cast nt (f_FROM_SPTR v') -- (if vv < v0 then f_DETAG v' else v')
hunk ./src/C/FromGrin2.hs 570
-        return (nns & getHead (f_NODEP(f_DETAG v')) =* nn,emptyExpression)
+        return (nns & getHead (f_NODEP(f_FROM_SPTR v')) =* nn,emptyExpression)
hunk ./src/C/FromGrin2.hs 880
-gc_roots vs = functionCall (name "gc_frame0") (v_gc:constant (number (fromIntegral $ length vs)):vs)
-gc_end =      functionCall (name "gc_end") []
+gc_roots vs   = functionCall (name "gc_frame0") (v_gc:constant (number (fromIntegral $ length vs)):vs)
+gc_end        = functionCall (name "gc_end") []
hunk ./src/C/FromGrin2.hs 884
-f_DETAG e     = functionCall (name "DETAG") [e]
+f_FROM_SPTR e = functionCall (name "FROM_SPTR") [e]
hunk ./src/data/rts/jhc_rts2.c 2
-#define ISLAZY(x)    (((uintptr_t)(x)) & 0x1)
-#define DETAG(x)     ((uintptr_t)(x) & ~0x3)
-#define GETTAG(x)    ((uintptr_t)(x) & 0x3)
-
-#define GETHEAD(x)   (NODEP(x)->head)
-#define NODEP(x)     ((node_t *)(x))
-#define DNODEP(x)    ((dnode_t *)(x))
-#define EVALTAG(fn)  (assert(((uintptr_t)(fn) & 0x3) == 0),(sptr_t)((uintptr_t)(fn) | P_LAZY))
-#define EVALTAGC(fn) ((sptr_t)((void *)(fn) + P_LAZY))
-#define EVALFUNC(fn) ((fptr_t)((uintptr_t)(fn) + P_FUNC))
-#define VALUE(n)     ((wptr_t)(((intptr_t)(n) << 2) | P_VALUE))
-#define GETVALUE(n)  ((intptr_t)(n) >> 2)
-#define ISVALUE(n)   (assert(!ISLAZY(n)), ((uintptr_t)(n) & 0x2))
-#define PROMOTE(n)   ((wptr_t)(n))
-#define DEMOTE(n)    ((sptr_t)(n))
-#define GETWHAT(x)   (GETTAG(x) == P_VALUE ? ((uintptr_t)(x) >> 16) : DNODEP(x)->what)
-
-#define SETWHAT(x,v)   (DNODEP(x)->what = (v))
-#define RAWWHAT(w)     (wptr_t)(((uintptr_t)w << 16) | P_VALUE)
-
-
-#define P_WHNF  0x0
-#define P_LAZY  0x1
-#define P_VALUE 0x2
-#define P_FUNC  0x3
-
-#define BLACK_HOLE ((fptr_t)0xDEADBEEF)
-
hunk ./src/data/rts/jhc_rts2.c 9
-However, we call whatever conventions and binary layouts used in the generated
-executable the run time system. Since jhc generates the code anew each time, it
-can build a different 'run time' based on compiler options, trading things like
-the garbage collector as needed or changing the closure layout when we know we
-have done full program optimization. This describes the 'native' layout upon
-which other conventions are layered.
+However, back ends do have specific run-time representations of data, which can
+be affected by things like the choice of garbage collector. The following
+describes the general layout for the C based back-ends, but compiler options
+such as garbage collection method or whether we do full program analysis, will
+affect which features are used and whether certain optimized layouts are
+possible.
+
+Unboxed values directly translate to values in the target language, an unboxed
+Int will translate directly into an 'int' as an argument and an unboxed pointer
+will be a raw pointer. Unboxed values have no special interpretation and are
+_not_ followed by the garbage collector. If the target language does not
+support a feature such as multiple return values, it will have to be simulated.
+It would not be wrong to think of Grin code that only deals with unboxed values
+to be isomorphic to C-- or C augmented with multiple return values.
hunk ./src/data/rts/jhc_rts2.c 24
-A basic value in jhc is represented by a 'smart pointer' of c type sptr_t. a
+Boxed values have a standard representation and can be followed. Unlike some
+other implementation, being boxed does not imply the object is located on the
+heap. It may be on the stack, heap, or even embedded within the smart pointer
+itself. Being boxed only means that the object may be represented by a smart
+pointer, which may or may not actually be a pointer in the traditional sense.
+
+A boxed value in jhc is represented by a 'smart pointer' of c type sptr_t. a
hunk ./src/data/rts/jhc_rts2.c 32
-depending on a pair of tag bits.
+depending on a pair of tag bits, called the ptype.
hunk ./src/data/rts/jhc_rts2.c 41
-      L - lazy, this bit being set means the value is not in WHNF
+      L - lazy, this bit being set means the value is potentially not in WHNF
hunk ./src/data/rts/jhc_rts2.c 43
-A raw sptr_t on its own in the wild can only take on one of the following values:
+A sptr_t on its own in the wild can only take on one of the following forms:
hunk ./src/data/rts/jhc_rts2.c 46
-    |    raw value      | 10|
+    |    whnf raw value | 10|
hunk ./src/data/rts/jhc_rts2.c 53
-    -------------------------
-    |   lazy location   | 01|
-    -------------------------
+WHNF stands for 'Weak Head Normal Form' and means that the value is not a
+suspended function and hence not a pointer to a thunk. It may be directly
+examined and need not be evaluated. wptr_t is an alias for sptr_t that is
+guarenteed to be of one of the above forms. It is used to improve safety for
+when we can statically know that a value is WHNF and hence we can skip the
+expensive 'eval'.
hunk ./src/data/rts/jhc_rts2.c 60
-A raw value can be anything and not necessarily a pointer in general, a WHNF
-location is a pointer to some value in WHNF. The system places no restrictions
-on what is actually pointed to by a WHNF pointer, however the garbage collector
-in use may. In general, the back end is free to choose what to place in the raw
-value field or in what a WHNF points to with complete freedom. If an
-implementation sees the L bit is clear, it can pass on the smart pointer
-without examining it knowing the value is in WHNF.
+The difference between the raw value and the whnf location is that the first
+contains uninterpreted bits, while the second is a pointer to a location on the
+heap or stack and hence the garbage collector should follow it. The format of
+the memory pointed to by the whnf location is unspecified and dependent on the
+actual type being represented.
hunk ./src/data/rts/jhc_rts2.c 66
-A lazy location points to a potential closure or an indirection to a WHNF
-value. The lazy location is an allocated chunk of memory that is at least
-one pointer long. the very first location in a closure must be one of the
-following.
+Partial (unsaturated) applications are normal WHNF values. Saturated
+applications which may be 'eval'ed and updated are called thunks and must not
+be pointed to by WHNF pointers. Their representation follows.
hunk ./src/data/rts/jhc_rts2.c 71
-    | raw value or whnf  |X0|
+    |   lazy location   | 01|
hunk ./src/data/rts/jhc_rts2.c 74
-An evaluated value, interpreted exactly as above. one can always replace any occurance of a
-lazy location with an evaluated indirecton.
+A lazy location points to either a thunk, or a redirection to a WHNF value. A
+lazy location is always a pointer to an allocated block of memory which always
+begins with a restricted smart pointer. This restricted smart pointer is represented by
+the C type alias 'fptr_t'. fptr_t's only occur as the first entry in a lazy
+location, they never are passed around as objects in their own right.
+
+A fptr_t may be a whnf value or a code pointer. If a fptr_t is a whnf value (of one of
+the two forms given above) then it is called a redirection, the lazy location should be
+treated exactly as if it were the whnf given. This is used to redirect an evaluated
+thunk to its computed value.
+
+A fptr_t may also be a 'code pointer' in which case the lazy location is called
+a thunk. A code pointer is a pointer to executable machine code that evaluates
+a closure and returns a wptr_t, the returned wptr_t is then generally written
+over the code pointer, turning the thunk into a redirection. It is the
+responsibility of the code pointed to to perform this redirection.
hunk ./src/data/rts/jhc_rts2.c 96
-This is something to evaluate, code pointer is a pointer to a function that takes
-the memory location as its only argument, the called function is in charge
-of updating the location if needed.
+When debugging, the special code pointer BLACK_HOLE is also sometimes stored in
+a fptr_t to detect certain run-time errors.
hunk ./src/data/rts/jhc_rts2.c 99
-note that it is invalid to have a lazy location point to another lazy
-location. there is only ever one level of indirection allowed, and only from
-lazy locations
+Note that unlike other implementations, a fptr_t may _not_ be another lazy
+location. you can not have chained redirections, a redirection is always a
+redirection to a whnf value.
hunk ./src/data/rts/jhc_rts2.c 103
-note that a partial application is just like any other value in WHNF as far
-as the above is concered. It happens to possibly contain a code pointer.
+    sptr_t - a tagged smart pointer, may contain a whnf value or a lazy location.
+    wptr_t - a tagged smart pointer that contains a whnf value (either raw or a location)
+    fptr_t - a tagged smart pointer, may contain a whnf value indicating a redirection, or a code pointer indicating a thunk.
hunk ./src/data/rts/jhc_rts2.c 109
+#define P_WHNF  0x0
+#define P_LAZY  0x1
+#define P_VALUE 0x2
+#define P_FUNC  0x3
+
+#define IS_LAZY(x)     (((uintptr_t)(x)) & 0x1)
+#define IS_PTR(x)      (!(((uintptr_t)(x)) & 0x2))
+
+#define FROM_SPTR(x)   (typeof (x))((uintptr_t)(x) & ~0x3)  // remove a ptype from a smart pointer
+#define GET_PTYPE(x)   ((uintptr_t)(x) & 0x3)               // return the ptype associated with a smart pointer
+#define TO_SPTR(t,x)   (typeof (x))((uintptr_t)(x) | (t))   // attach a ptype to a smart pointer
+#define TO_SPTR_C(t,x) (typeof (x))((uintptr_t)(x) + (t))   // attach a ptype to a smart pointer, suitable for use by constant initialializers
+
+#define GETHEAD(x)   (NODEP(x)->head)
+#define NODEP(x)     ((node_t *)(x))
+#define DNODEP(x)    ((dnode_t *)(x))
+
+#define EVALTAG(fn)  TO_SPTR(P_LAZY,(sptr_t)fn)
+#define EVALTAGC(fn) TO_SPTR_C(P_LAZY,(sptr_t)fn)
+#define EVALFUNC(fn) TO_SPTR_C(P_FUNC,(fptr_t)fn)
+
+// #define EVALTAG(fn)  (assert(GET_PTYPE(fn) == 0),(sptr_t)((uintptr_t)(fn) | P_LAZY))
+// #define EVALTAGC(fn) ((sptr_t)((void *)(fn) + P_LAZY))
+// #define EVALFUNC(fn) ((fptr_t)((uintptr_t)(fn) + P_FUNC))
+
+#define VALUE(n)     ((wptr_t)(((intptr_t)(n) << 2) | P_VALUE))
+#define GETVALUE(n)  ((intptr_t)(n) >> 2)
+#define ISVALUE(n)   (assert(!IS_LAZY(n)), ((uintptr_t)(n) & 0x2))
+
+// demote is always safe, we must only promote when we know the argument is a WHNF
+#define PROMOTE(n)   ((wptr_t)(n))
+#define DEMOTE(n)    ((sptr_t)(n))
+
+#define GETWHAT(x)   (GET_PTYPE(x) == P_VALUE ? ((uintptr_t)(x) >> 16) : DNODEP(x)->what)
+#define SETWHAT(x,v) (DNODEP(x)->what = (v))
+#define RAWWHAT(w)   (wptr_t)(((uintptr_t)w << 16) | P_VALUE)
+
+
+
+
+#define BLACK_HOLE ((fptr_t)0xDEADBEEF)
hunk ./src/data/rts/jhc_rts2.c 151
-/*
- * type names
- *
- * sptr_t - a tagged smart pointer, may be a value, may be a pointer to a whnf or lazy location
- * wptr_t - a value guarenteed to be in whnf
- * fptr_t - a pointer to a whnf or a function pointer to something to evaluate, first value in a lazy location.
- * what_t  - the discriminator of a discriminated union
- *
- */
hunk ./src/data/rts/jhc_rts2.c 152
-typedef struct node *  sptr_t;
-typedef struct dnode * wptr_t;
-typedef void *         fptr_t;
-typedef uintptr_t      what_t;
+// we use dummy structs here so the compiler will catch any attempt
+// to use one type in anothers place
+typedef struct sptr * sptr_t;
+typedef struct wptr * wptr_t;
+typedef struct fptr * fptr_t;
+typedef uintptr_t     what_t;
hunk ./src/data/rts/jhc_rts2.c 181
-static int A_UNUSED
+static bool A_UNUSED
hunk ./src/data/rts/jhc_rts2.c 184
-        return ((GETTAG(s) == P_VALUE) || ((GETTAG(s) == P_WHNF) && jhc_malloc_sanity(s,P_WHNF)));
+        return ((GET_PTYPE(s) == P_VALUE) || ((GET_PTYPE(s) == P_WHNF) && jhc_malloc_sanity(s,P_WHNF)));
hunk ./src/data/rts/jhc_rts2.c 187
-static int A_UNUSED
+static bool A_UNUSED
hunk ./src/data/rts/jhc_rts2.c 191
-                return 1;
-        assert(GETTAG(s) == P_LAZY);
-        node_t *ds = (sptr_t)DETAG(s);
+                return true;
+        assert(GET_PTYPE(s) == P_LAZY);
+        node_t *ds = (node_t *)FROM_SPTR(s);
hunk ./src/data/rts/jhc_rts2.c 195
-        if(ISLAZY(ds->head)) {
-                if(ds->head == BLACK_HOLE) return 1;
-                assert(GETTAG(ds->head) == P_FUNC);
-                fptr_t dhead = (fptr_t)DETAG(ds->head);
-                assert(dhead >= &_start && dhead < &_end);
-                return 1;
+        if(IS_LAZY(ds->head)) {
+                if(ds->head == BLACK_HOLE) return true;
+                assert(GET_PTYPE(ds->head) == P_FUNC);
+                fptr_t dhead = FROM_SPTR(ds->head);
+                assert((void *)dhead >= &_start && (void *)dhead < &_end);
+                return true;
hunk ./src/data/rts/jhc_rts2.c 208
-#define jhc_valid_whnf(x) 1
-#define jhc_valid_lazy(x) 1
+#define jhc_valid_whnf(x) true
+#define jhc_valid_lazy(x) true
hunk ./src/data/rts/jhc_rts2.c 226
-        assert(!ISLAZY(s));
+        assert(!IS_LAZY(s));
hunk ./src/data/rts/jhc_rts2.c 234
-        assert(!ISLAZY(s));
+        assert(!IS_LAZY(s));
hunk ./src/data/rts/jhc_rts2.c 244
-        if(ISLAZY(s)) {
-                sptr_t h = (sptr_t)(GETHEAD(DETAG(s)));
-                assert(!ISLAZY(h));
+        if(IS_LAZY(s)) {
+                sptr_t h = (sptr_t)(GETHEAD(FROM_SPTR(s)));
+                assert(!IS_LAZY(h));
hunk ./src/data/rts/jhc_rts2.c 260
-        if(ISLAZY(s)) {
-                assert(GETTAG(s) == P_LAZY);
-                void *ds = (void *)DETAG(s);
+        if(IS_LAZY(s)) {
+                assert(GET_PTYPE(s) == P_LAZY);
+                void *ds = FROM_SPTR(s);
hunk ./src/data/rts/jhc_rts2.c 265
-                if(ISLAZY(h)) {
-                        eval_fn fn = (eval_fn)DETAG(h);
+                if(IS_LAZY(h)) {
+                        eval_fn fn = (eval_fn)FROM_SPTR(h);
hunk ./src/data/rts/jhc_rts2.c 292
-        assert(!ISLAZY(new));
+        assert(!IS_LAZY(new));