Changeset 824

Timestamp:

Aug 3, 2012 2:31:09 AM (12 years ago)

Author:

biz002

Message:

groupings support for pgf

groupings are a way to apply operations on token ranges for producing
or adjusting formatting directives

support for defining groupings and a grouping for 'group' operation

integrate grouping functionality

implement FmtSt FmtSt_cons(FmtSt st, Tseq s) and use it;
this abstracts away from having to deal with grouping related
things when unshifting input

workaround for current Rascal being unhappy with tseqGet;
'tseqGet' simplified and implemented using a 'switch' now;
seemed Rascal was not considering ADT overload alternatives
at all in some cases, and would not match say 'ListTseq'
based on dynamic type

internal pgf grouping types and functions

support for running rascal modules from command-line without
Magnolia compiler instance, using the exact same version of Rascal
as used with Magnolia

all test cases passing again

more test cases

fix import in test cases

Location:

trunk/magnolia-eclipse

Files:

• src/org/magnolialang/pgf/Groupings.rsc (added)
• src/org/magnolialang/pgf/Token.rsc (modified) (7 diffs)
• src/org/magnolialang/pgf/engines/FmtEngine.rsc (modified) (11 diffs)
• src/org/magnolialang/pgf/tests/RunTestSpacer.rsc (modified) (3 diffs)
• src/org/magnolialang/pgf/tests/RunTestSpacerDirect.rsc (copied) (copied from trunk/magnolia-eclipse/src/org/magnolialang/pgf/tests/RunTestSpacer.rsc ) (3 diffs)
• src/org/magnolialang/pgf/tests/TestFmtEngine.rsc (modified) (4 diffs)
• src/org/magnolialang/pgf/tests/TestToken.rsc (added)
• src/org/magnolialang/testutil/RascalPlainRunMain.java (added)
• src/org/magnolialang/testutil/RascalShell.java (modified) (2 diffs)
• src/org/magnolialang/testutil/core.clj (modified) (3 diffs)
• test/org/magnolialang/test/core.clj (modified) (2 diffs)

trunk/magnolia-eclipse/src/org/magnolialang/pgf/Token.rsc

@@ -27 +27 @@
   | Space(str s)
   | Width(num w)
+  | BeginGr(str s) // grouping start
+  | EndGr(str s) // grouping end
   | NilToken()
   | Eof();
@@ -63 +65 @@
   = NilTseq() 
   | ListTseq(list[value] lst)
-  | RestTseq(list[value] lst, int i)
   | LazyTseq(Tseq (value) f, value v);
 
@@ -113 +114 @@
     case NilTseq(): ;
     case ListTseq(l): lst += l;
-    case RestTseq(l, i): lst += tail(l, size(l) - i);
     case list[value] x: lst += x;
     default: lst += [x];
@@ -121 +121 @@
 }
 
-public tuple[Token, Tseq] tseqGet(false) {
-  return <eofTok, nilSeq>;
-}
-
-public tuple[Token, Tseq] tseqGet(Token t) {
-  return <t, nilSeq>;
-}
-
-public tuple[Token, Tseq] tseqGet(str s) {
-  return <Text(s), nilSeq>;
-}
-
-public tuple[Token, Tseq] tseqGet([]) {
-  return <eofTok, nilSeq>;
-}
-
-public tuple[Token, Tseq] tseqGet([x]) {
-  return tseqGet(x);
-}
-
-public tuple[Token, Tseq] tseqGet(list[value] lst, int i) {
+/**
+   With current Rascal this will not work if the choices are
+   implemented using overloading. (See "TestToken.rsc".) Instead we
+   use a 'switch', and specify the 'default' modifier to allow for
+   extending through overloading.
+ */
+public default tuple[Token, Tseq] tseqGet(x) {
+  switch (x) {
+  case Token t: return <t, nilSeq>;
+  case str s: return <Text(s), nilSeq>;
+  case NilTseq(): return <eofTok, nilSeq>;
+  case ListTseq(lst): return tseqGetList(lst);
+  case list[value] lst: return tseqGetList(lst);
+  case LazyTseq(_, _): return tseqGetLazy(x);
+  case false: return <eofTok, nilSeq>;
+  default:
+    throw "attempt to tseqGet from unsupported value <x>";
+  }
+}
+
+public tuple[Token, Tseq] tseqGetList(list[value] lst) {
+  int i = 0;
   int len = size(lst);
   while (i < len) {
@@ -147 +148 @@
     <t, rest> = tseqGet(h);
     if (t != Eof())
+      // 2nd arg of 'tail' is number of elements to include
       return <t, tseq(rest, ListTseq(tail(lst, size(lst) - (i+1))))>;
     i += 1;
@@ -153 +155 @@
 }
 
-public tuple[Token, Tseq] tseqGet(list[value] lst) {
-  return tseqGet(lst, 0);
-}
-
-public tuple[Token, Tseq] tseqGet(s : NilTseq()) {
-  return <eofTok, s>;
-}
-
-public tuple[Token, Tseq] tseqGet(ListTseq(lst)) {
-  return tseqGet(lst);
-}
-
-public tuple[Token, Tseq] tseqGet(RestTseq(lst, i)) {
-  return tseqGet(lst, i);
-}
-
-public tuple[Token, Tseq] tseqGet(s : LazyTseq(_, _)) {
+public tuple[Token, Tseq] tseqGetLazy(s : LazyTseq(_, _)) {
   do {
     Tseq (value) f = s.f;
@@ -188 +174 @@
 }
 
+public bool isEmpty(Tseq ts) {
+  <t, ts> = tseqGet(ts);
+  return (t == Eof());
+}
+
 // deprecated (use emptyTseq)
 public Tseq newTokenStream() {

trunk/magnolia-eclipse/src/org/magnolialang/pgf/engines/FmtEngine.rsc

@@ -64 +64 @@
 
 // --------------------------------------------------
-// formatting algorithm
-// --------------------------------------------------
+// groupings
+// --------------------------------------------------
+
+/*
+  The operations that must be defined for each grouping.
+
+  These Grouping functions:
+
+  begin:: creates fresh state for this grouping;
+          returns state for GrpSt (any value)
+
+  put:: buffers a token within region
+
+  accept:: accepts a token from an inner grouping into this one
+
+  end:: ends this grouping
+
+  eof:: handles an EOF within this grouping
+
+  The way we do this is that we have both an input stream and grouping
+  state in the algorithm. If there is any grouping state then the
+  input is fed into the grouping machinery. Some input may be held in
+  the grouping state before it becomes available to the algorithm
+  proper. Calling 'flush' will cause an error if there's an incomplete
+  grouping.
+*/
+
+data Grouping = 
+    Grouping(
+             str name,
+             value () begin,
+             tuple[value, Tseq] (value, Token) put,
+             tuple[value, Tseq] (value, Token) accept,
+             Tseq (value) end,
+             void (value, str) eof
+             );
+
+public map[str, Grouping] grpMap = ();
+
+public void addGrouping(str name, Grouping g) {
+  grpMap[name] = g;
+}
+
+public Grouping groupingByName(str name) {
+  return grpMap[name];
+}
+
+// tp:: grouping type (Grouping)
+// st:: grouping-specific state (any)
+data Token = GrpSt(Grouping tp, value st) | 
+  NoGrpSt();
 
 // cw:: specified page width (constant)
@@ -75 +124 @@
 // i:: nesting string
 // bt:: backtracking state (if any; can be chained)
+// grp:: grouping state (may be NoGrpSt)
+// grps:: grouping stack
 data FmtSt = FmtSt(int cw, num w, Tseq outDoc, Tseq inDoc, 
-                   int k, LvStack lvStack, str i, FmtSt bt) | 
+                   int k, LvStack lvStack, str i, FmtSt bt,
+                   Token grp, list[Token] grps) | 
   NoFmtSt();
+
+// Emits the specified tokens from a source group to an outer group.
+// The outer group (if any) is assumed to be in the 'grp' field, and
+// the source group need not be in the state any longer (it must be
+// specified as 'sg'). The outer group may further emit tokens
+// forward, all the way up to the top-level (i.e. beyond all the
+// groupings).
+//
+// st:: formatting state (FmtSt)
+// sg:: source group (Token)
+// s:: token stream to receive (Tseq or NilTseq())
+// Returns:: formatting state (FmtSt)
+FmtSt grpEmit(FmtSt st, Token sg, Tseq s) {
+  FmtSt to_top() {
+    inDoc = tseq(s, st.inDoc);
+    return FmtSt(st.cw, st.w, st.outDoc, inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 st.grp, st.grps);
+  }
+
+  FmtSt to_outer(str name, Token tg) {
+    g_type = tg.tp;
+    g_st = tg.st;
+    <n_g_st, r> = g_type.accept(g_st, s, name);
+    n_grp = GrpSt(tg, n_g_st);
+    n_st = FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 n_grp, st.grps);
+    if (r == NilTseq())
+      return n_st;
+    // Since the current grouping produced output, that
+    // belongs to the next grouping, if any. Hence we make
+    // the next grouping the current one, and then prepend
+    // the tokens to be received.
+    st = grpUnshift(n_st);
+    inDoc = tseq(r, st.inDoc);
+    n_st = FmtSt(st.cw, st.w, st.outDoc, inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 st.grp, st.grps);
+    return n_st;
+  }
+
+  if (s == NilTseq())
+    return st;
+
+  Token tg = st.grp;
+  if (tg == NoGrpSt())
+    return to_top();
+  return to_outer(sg.tp.name, tg);
+}
+
+// st:: formatting state (FmtSt)
+// Returns:: formatting state (FmtSt)
+FmtSt grpFlush(FmtSt st) {
+  grp = st.grp;
+  if (grp != NoGrpSt()) {
+    g_type = grp.tp;
+    // Executed for side effects only.
+    // Intended for checking state,
+    // and reporting errors.
+    g_type.eof(grp.st, g_type.name);
+  }
+  return st;
+}
+
+// st:: formatting state (FmtSt)
+// grp:: group to push (Token)
+// Returns:: formatting state (FmtSt)
+FmtSt grpPush(FmtSt st, Token grp) {
+  //println("push <grp> <st.grp> <st.grps>");
+  Token old_grp = st.grp;
+  if (old_grp == NoGrpSt()) {
+    return FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 grp, st.grps);
+  } else {
+    grps = spush(st.grps, old_grp);
+    return FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 grp, grps);
+  }
+}
+
+// st:: formatting state (FmtSt)
+// Returns:: formatting state (FmtSt)
+FmtSt grpPop(FmtSt st) {
+  //println("pop <st.grp> <st.grps>");
+  if (st.grp == NoGrpSt())
+    throw "no grouping to pop";
+  grps = st.grps;
+  if (isEmpty(grps)) {
+    return FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+                 st.k, st.lvStack, st.i, st.bt,
+                 NoGrpSt(), grps);
+  }
+  <grps, grp> = spop(grps);
+  return FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+               st.k, st.lvStack, st.i, st.bt,
+               grp, grps);
+}
+
+// st:: formatting state (FmtSt)
+// Returns:: formatting state (FmtSt)
+FmtSt grpUnshift(FmtSt st) {
+  Token old_grp = st.grp;
+  st = grpPop(st);
+  inDoc = tseq(old_grp, st.inDoc);
+  n_st = FmtSt(st.cw, st.w, st.outDoc, inDoc,
+               st.k, st.lvStack, st.i, st.bt,
+               st.grp, st.grps);
+  return n_st;
+}
+
+// st:: formatting state (FmtSt)
+// h:: Begin token (Token)
+// Returns:: formatting state (FmtSt)
+FmtSt grpBegin(FmtSt st, Token h) {
+  Grouping g_type = groupingByName(h.s);
+  Token inner = GrpSt(g_type, g_type.begin());
+  return grpPush(st, inner);
+}
+
+// st:: formatting state (FmtSt)
+// t:: End token (Token)
+// Returns:: formatting state (FmtSt)
+FmtSt grpEnd(FmtSt st, Token t) {
+  Grouping t_type = groupingByName(t.s);
+  Token grp = st.grp;
+  if (grp == NoGrpSt()) {
+    t_name = t_type.name;
+    ctx = take(5, st.inDoc);
+    throw "close <t_name> grouping without open: before <ctx>";
+  }
+  Grouping g_type = grp.tp;
+  if (g_type != t_type) {
+    t_name = t_type.name;
+    g_name = g_type.name;
+    throw "close <t_name> grouping while <g_name> grouping open";
+  }
+  value g_st = grp.st;
+  Tseq r = g_type.end(g_st);
+  st = grpPop(st);
+  if (r == NilTseq())
+    return st;
+  return grpEmit(st, grp, r);
+}
+
+// st:: formatting state, with open groupings (FmtSt)
+// h:: token belonging in the grouping (Token)
+// Returns:: formatting state (FmtSt)
+FmtSt grpPut(FmtSt st, Token h) {
+  Token grp = st.grp;
+  Grouping g_type = grp.tp;
+  value g_st = grp.st;
+  <n_g_st, r> = g_type.put(g_st, h);
+  n_grp = GrpSt(g_type, n_g_st);
+  n_st = FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+               st.k, st.lvStack, st.i, st.bt,
+               n_grp, st.grps);
+  return grpEmit(grpUnshift(n_st), grp, r);
+}
+
+// --------------------------------------------------
+// formatting algorithm
+// --------------------------------------------------
 
 // w:: page width (integer)
@@ -83 +300 @@
 public FmtSt newFmtSt(int w, Tseq inDoc) {
   return FmtSt(w, w, emptyTseq(), inDoc, 
-               0, [], "", NoFmtSt());
+               0, [], "", NoFmtSt(), NoGrpSt(), []);
 }
 
@@ -94 +311 @@
 // After this it is safe to consume all of 'outDoc'.
 public FmtSt flush(FmtSt st) {
+  //println("flush <st.grp> <st.grps>");
+  st = grpFlush(st);
   return FmtSt(st.cw, st.w,
                st.outDoc, st.inDoc, 
-               st.k, st.lvStack, st.i, NoFmtSt());
-}
-
-// Processes one token of input (if there is any). Note that 'outDoc'
-// cannot externally be considered committed for as long as
-// backtracking is possible. Except if one specifically wants to flush
-// the output.
-// d:: token to process
-// inDoc:: input after it
-// st:: current state
-FmtSt processToken(Token d, Tseq inDoc, FmtSt st) {
+               st.k, st.lvStack, st.i, NoFmtSt(),
+               st.grp, st.grps);
+}
+
+// This function unshifts grouping state out of the way so that
+// prepending input 's' becomes possible. Note that any tokens that
+// have reached the algorithm proper have been past all groupings
+// already, and have nothing to do with groupings.
+//
+// st:: current state (FmtSt)
+// s:: stream to prepend to inDoc (Tseq)
+// Returns:: new state (FmtSt)
+public FmtSt FmtSt_cons(FmtSt st, Tseq s) {
+  Token grp = st.grp;
+  Tseq inDoc = st.inDoc;
+  if (grp == NoGrpSt())
+    return FmtSt(st.cw, st.w,
+                 st.outDoc, tseq(s, inDoc),
+                 st.k, st.lvStack, st.i, st.bt,
+                 grp, st.grps);
+  // outermost grouping will be read first
+  inDoc = tseq(s, reverse(st.grps), grp, inDoc);
+  return FmtSt(st.cw, st.w,
+               st.outDoc, inDoc,
+               st.k, st.lvStack, st.i, st.bt,
+               NoGrpSt(), []);
+}
+
+// Before calling this function ensure that all state (except for the
+// argument token) is consistent and in 'st'. Note that this function
+// does not deal with groupings.
+//
+// Note that 'outDoc' cannot externally be considered committed for as
+// long as backtracking is possible. Except if one specifically wants
+// to flush the output.
+//
+// st:: current state (FmtSt)
+// d:: token to process (Token)
+// Returns:: new state (FmtSt)
+FmtSt processTokenAlgo(FmtSt st, Token d) {
   k = st.k;
   i = st.i;
@@ -112 +360 @@
   case Nest(lv): { 
     <lvStack, i> = margin(st.lvStack, k, i, lv);
-    return FmtSt(st.cw, st.w, st.outDoc, inDoc, k, lvStack, i, st.bt);
+    return FmtSt(st.cw, st.w, st.outDoc, st.inDoc, k, lvStack, i, st.bt,
+                 st.grp, st.grps);
   }
   case Text(s): {
@@ -121 +370 @@
       return st.bt; // backtrack
     else
-      return FmtSt(st.cw, st.w, put(d, st.outDoc), inDoc,
-                   k, st.lvStack, i, st.bt);
+      return FmtSt(st.cw, st.w, put(d, st.outDoc), st.inDoc,
+                   k, st.lvStack, i, st.bt, st.grp, st.grps);
   }
   case Line(): { 
@@ -131 +380 @@
     // right won't fit either, the right choice is still taken if
     // backtracking is not possible.
-    lineDoc = tseq(Line(), Text(i));
-    outDoc = concat(st.outDoc, lineDoc);
-    return FmtSt(st.cw, st.w, outDoc, inDoc,
-                 size(i), st.lvStack, i, NoFmtSt());
+    outDoc = tseq(st.outDoc, Line(), Text(i));
+    return FmtSt(st.cw, st.w, outDoc, st.inDoc,
+                 size(i), st.lvStack, i, NoFmtSt(), st.grp, st.grps);
   }
   case Union(l, r): {
     // Pick left option, leave right for backtracking.
-    r_st = FmtSt(st.cw, st.w, st.outDoc, concat(r, inDoc),
-                 k, st.lvStack, i, st.bt);
-    inDoc = concat(l, inDoc);
-    return FmtSt(st.cw, st.w, st.outDoc, inDoc,
-                 k, st.lvStack, i, r_st);
+    r_st = FmtSt_cons(st, r);
+    l_st = FmtSt(st.cw, st.w, st.outDoc, st.inDoc,
+                 k, st.lvStack, i, r_st, st.grp, st.grps);
+    return FmtSt_cons(l_st, l);
   }
   case Width(w): {
-    return FmtSt(st.cw, w, st.outDoc, inDoc,
-                 k, st.lvStack, i, st.bt);
-  }
-
+    return FmtSt(st.cw, w, st.outDoc, st.inDoc,
+                 k, st.lvStack, i, st.bt, st.grp, st.grps);
+  }
+  case NilToken(): {
+    return st;
+  }
     // Spacer compat. Space(str s);
   case Space(s): {
-    return FmtSt(st.cw, st.w, st.outDoc,
-                 tseq(union(Text(s), Line()), inDoc),
-                 k, st.lvStack, i, st.bt);
+    return FmtSt_cons(st, tseq(union(Text(s), Line())));
   }
   default:
-        throw "FmtEngine::processToken: unknown token: <d>";
-  }
-}
-
-// Processes tokens for as long as there is input.
-FmtSt processTokens(FmtSt st) {
-  Token d;
-  Tseq inDoc;
+        throw "FmtEngine::processTokenAlgo: unknown token: <d>";
+  }
+}
+
+// Processes a token (if there is input). Supports groupings.
+//
+// st:: current state (FmtSt)
+// Returns:: new state (FmtSt)
+FmtSt processToken(FmtSt st) {
+  <d, inDoc> = tseqGet(st.inDoc);
+  if (d == Eof())
+    return st;
+  st = FmtSt(st.cw, st.w, st.outDoc, inDoc,
+             st.k, st.lvStack, st.i, st.bt, st.grp, st.grps);
+  switch (d) {
+  case GrpSt(_, _): return grpPush(st, d);
+  case BeginGr(_): return grpBegin(st, d);
+  case EndGr(_): return grpEnd(st, d);
+  }
+  if (st.grp != NoGrpSt())
+    return grpPut(st, d);
+  return processTokenAlgo(st, d);
+}
+
+// Whether the state has any more data that can be processed (without
+// additional input).
+public bool isPending(FmtSt st) {
+  // Any non-GrpSt tokens are always kept in inDoc, so this is enough.
+  return !isEmpty(st.inDoc);
+}
+
+// Adds a tseq to input.
+public FmtSt write(FmtSt st, Tseq s) {
+  inDoc = tseq(st.inDoc, s);
+  return FmtSt(st.cw, st.w, st.outDoc, inDoc,
+               st.k, st.lvStack, st.i, st.bt, st.grp, st.grps);
+}
+
+// Adds a token to input.
+public FmtSt put(FmtSt st, Token t) {
+  return write(st, tseq(t));
+}
+
+// Drives formatting for as long as there is anything pending.
+public FmtSt drive(FmtSt st) {
   while (true) {
-    inDoc = st.inDoc;
-    <d, inDoc> = tseqGet(inDoc);
-    if (d == Eof())
+    if (!isPending(st))
       return st;
-    st = processToken(d, inDoc, st);
+    //println("before <st>");
+    st = processToken(st);
+    //println("after <st>");
   }
 }
@@ -201 +485 @@
   }
 }
- 
+
+// w: output line width
+// ts: a (complete) input document 
 public str pretty(int w, Tseq ts) {
   FmtSt st = newFmtSt(w, ts);
-  st = processTokens(st);
+  st = flush(drive(st));
   return docToString(st.outDoc);
 }
 
+// Clears output buffer by printing it all out. Printing is done as
+// soon as individual tokens are converted to strings.
 public FmtSt printBuffered(FmtSt st) {
   Tseq ts = st.outDoc;
@@ -214 +502 @@
     if (t == Eof())
       return FmtSt(st.cw, st.w, ts, st.inDoc,
-                   st.k, st.lvStack, st.i, st.bt);
+                   st.k, st.lvStack, st.i, st.bt,
+                   st.grp, st.grps);
     print(tokenToString(t));
   }
@@ -225 +514 @@
   while (true) {
     if (st.bt == NoFmtSt())
+      // cannot backtrack so safe to print outDoc
       st = printBuffered(st);
-    <d, inDoc> = tseqGet(st.inDoc);
-    if (d == Eof())
+    if (!isPending(st))
       return st;
-    st = processToken(d, inDoc, st);
+    //println("before <st>");
+    st = processToken(st);
+    //println("after <st>");
   }
 }
@@ -278 +569 @@
   return group(tseq(x));
 }
-
-// --------------------------------------------------
-// 
-// --------------------------------------------------
-
-public void main(list[str] args) {
-}

trunk/magnolia-eclipse/src/org/magnolialang/pgf/tests/RunTestSpacer.rsc

@@ -18 +18 @@
 
 public DecisionTable binOpTable = makeTable((
-    //<"BINOP","*"> : Insert(Text("xxx")),
     <"BINOP","*"> : Sequence([Insert(Text("x")), Insert(Text("y")), Insert(Text("z"))]),
     <"*","*"> : Nothing()
@@ -24 +23 @@
 
 void printFormatted(Tseq ts) {
-  prettyPrintFlush(newFmtSt(80, ts));
+  FmtSt st = newFmtSt(80, ts);
+  prettyPrintFlush(st);
 }
 
@@ -55 +55 @@
 }
 
-public void main(list[str] args) {
+public void runRunTestSpacer() {
   for (d <- exprTestData)
     runTest(d);
 }
+
+public void main(list[str] args) {
+  runRunTestSpacer();
+}

trunk/magnolia-eclipse/src/org/magnolialang/pgf/tests/RunTestSpacerDirect.rsc

@@ -1 +1 @@
 // run with rascal-src
-module org::magnolialang::pgf::tests::RunTestSpacer
+module org::magnolialang::pgf::tests::RunTestSpacerDirect
 
-import org::magnolialang::pgf::tests::ExprTestData; // generated
-
+import org::magnolialang::pgf::PrettyParseTree;
+import org::magnolialang::pgf::TableBuilder;
+import org::magnolialang::pgf::Token;
 import org::magnolialang::pgf::engines::FmtEngine;
 import org::magnolialang::pgf::engines::Spacer;
-import org::magnolialang::pgf::TableBuilder;
-import org::magnolialang::pgf::Token;
+import org::magnolialang::pgf::tests::SimpleExprLang;
 
 import IO;
+import List;
 
 // not visible from org::magnolialang::pgf::engines::Spacer !!
@@ -18 +19 @@
 
 public DecisionTable binOpTable = makeTable((
-    //<"BINOP","*"> : Insert(Text("xxx")),
     <"BINOP","*"> : Sequence([Insert(Text("x")), Insert(Text("y")), Insert(Text("z"))]),
     <"*","*"> : Nothing()
@@ -55 +55 @@
 }
 
-public void main(list[str] args) {
+list[Expr] progs = 
+  [
+   (Expr)`abc + cde`,
+   (Expr)`abc + cde + def`,
+   (Expr)`!a-b`,
+   (Expr)`f(abc, cde)`,
+   (Expr)`a * b + c / d`
+  ];
+
+public void runRunTestSpacerDirect() {
+  Tseq f(Expr e) {
+    return prettyParseTree(emptyTseq(), e);
+  }
+  list[Tseq] exprTestData = mapper(progs, f);
   for (d <- exprTestData)
     runTest(d);
 }
+
+public void main(list[str] args) {
+  runRunTestSpacerDirect();
+}

trunk/magnolia-eclipse/src/org/magnolialang/pgf/tests/TestFmtEngine.rsc

@@ -5 +5 @@
 import org::magnolialang::pgf::FmtUtil;
 import org::magnolialang::pgf::FmtUtilC;
+import org::magnolialang::pgf::Groupings;
 import org::magnolialang::pgf::Token;
 
@@ -65 +66 @@
 list[tuple[str, Tseq]] d_lst = 
   [
+   <"grouped (with groupings)", tseq(group_, d1, _group)>,
    <"many indented lines", cat("margin", indent(2), br, "first", indent(2), br, "second", br, "third", br, "fourth", dedent, dedent, br, "margin")>,
    <"empty block of declarations", cat("class X ", declBlock([]), ";")>,
@@ -94 +96 @@
   ];
 
-public void main(list[str] args) {
+public void runTestFmtEngine() {
+  //println(groupingByName("group"));
   //println(fillWords(lorem_ipsum_paragraph)); return;
   if (false) {
@@ -108 +111 @@
     testIncremental();
 }
+
+public void main(list[str] args) {
+  runTestFmtEngine();
+}

trunk/magnolia-eclipse/src/org/magnolialang/testutil/RascalShell.java

@@ -168 +168 @@
         // ---------------------------------------------------------------------------------------
 
+        public static int runModule(Evaluator evaluator, IValueFactory vf, TypeFactory tf, String modName, Collection<String> args) {
+                IListWriter w = vf.listWriter(tf.stringType());
+                for(String s : args) {
+                        w.append(vf.string(s));
+                }
+                try {
+                        evaluator.doImport(null, modName);
+                        IValue v = evaluator.call(null, "main", w.done());
+                        if(v != null) {
+                                System.out.println(v.toString());
+                        }
+                        return 0;
+                }
+                catch(ParseError pe) {
+                        URI uri = pe.getLocation();
+                        System.err.println("Parse error in " + uri + " from <" + (pe.getBeginLine() + 1) + "," + pe.getBeginColumn() + "> to <" + (pe.getEndLine() + 1) + "," + pe.getEndColumn() + ">");
+                }
+                catch(StaticError e) {
+                        System.err.println("Static Error: " + e.getMessage());
+                        e.printStackTrace(); // for debugging only
+                }
+                catch(Throw e) {
+                        System.err.println("Uncaught Rascal Exception: " + e.getMessage());
+                        String trace = e.getTrace();
+                        if(trace != null) {
+                                System.err.println(trace);
+                        }
+                }
+                catch(ImplementationError e) {
+                        e.printStackTrace();
+                        System.err.println("ImplementationError: " + e.getMessage());
+                }
+                catch(Throwable e) {
+                        System.err.println("Unexpected exception (generic Throwable): " + e.getMessage());
+                        System.err.println(evaluator.getStackTrace());
+                }
+                return 1;
+        }
+
+
         /**
          * Loads specified Rascal module in the evaluation context of the specified
@@ -185 +225 @@
                 IValueFactory vf = compiler.getVf();
                 TypeFactory tf = compiler.getTf();
-                IListWriter w = vf.listWriter(tf.stringType());
-                for(String s : args) {
-                        w.append(vf.string(s));
-                }
-                try {
-                        evaluator.doImport(null, modName);
-                        IValue v = evaluator.call(null, "main", w.done());
-                        if(v != null) {
-                                System.out.println(v.toString());
-                        }
-                        return 0;
-                }
-                catch(ParseError pe) {
-                        URI uri = pe.getLocation();
-                        System.err.println("Parse error in " + uri + " from <" + (pe.getBeginLine() + 1) + "," + pe.getBeginColumn() + "> to <" + (pe.getEndLine() + 1) + "," + pe.getEndColumn() + ">");
-                }
-                catch(StaticError e) {
-                        System.err.println("Static Error: " + e.getMessage());
-                        e.printStackTrace(); // for debugging only
-                }
-                catch(Throw e) {
-                        System.err.println("Uncaught Rascal Exception: " + e.getMessage());
-                        String trace = e.getTrace();
-                        if(trace != null) {
-                                System.err.println(trace);
-                        }
-                }
-                catch(ImplementationError e) {
-                        e.printStackTrace();
-                        System.err.println("ImplementationError: " + e.getMessage());
-                }
-                catch(Throwable e) {
-                        System.err.println("Unexpected exception (generic Throwable): " + e.getMessage());
-                        System.err.println(evaluator.getStackTrace());
-                }
-                return 1;
-        }
+                return runModule(evaluator, vf, tf, modName, args);
+        }
+
 }

trunk/magnolia-eclipse/src/org/magnolialang/testutil/core.clj

@@ -85 +85 @@
   (.eval rsc-evaluator s))
 
+;; Behaves better upon VoidResult.
 (defn reval-str [s]
   (rtostring (reval s)))
@@ -92 +93 @@
   (.evalMore rsc-evaluator s))
 
+(defn reval-more-str [s]
+  (rtostring (.evalMore rsc-evaluator s)))
+
 ;; E.g. (rimport "org::magnolialang::tasks::Manager")
 (defn rimport [s]
@@ -101 +105 @@
 (defn rcall [s & as]
   (rapply s as))
+
+;; This produces VoidResult, which
+;; org.rascalmpl.interpreter.result.Result.toString dislikes.
+;; E.g. (rtostring (make-rvoid))
+(defn make-rvoid []
+  (reval-more ";"))
 
 ;; For reloading Rascal modules individually.

trunk/magnolia-eclipse/test/org/magnolialang/test/core.clj

@@ -1 +1 @@
 (ns org.magnolialang.test.core
-  (:use [org.magnolialang.clojure.core])
+  (:use [org.magnolialang.testutil.core])
   (:use [clojure.test]))
 
@@ -23 +23 @@
 
 (deftest test-view-flattened-module
-  (view-flattened-module "basic.Basic"))
+  (view-flattened-module "basic.Basic")
+  (view-flattened-module "basic.Integer")
+  (view-flattened-module "errors.ConstantOverloading")
+  (view-flattened-module "mutification.Integer")
+  (view-flattened-module "mutification.Mutification")
+  (view-flattened-module "mutification.Mutification2")
+  (view-flattened-module "override.Override")
+  (view-flattened-module "scoping.Scoping")
+  (view-flattened-module "toplevel.Operators")
+  (view-flattened-module "toplevel.Parens")
+  (view-flattened-module "toplevel.Predicates")
+  (view-flattened-module "toplevel.Renaming")
+  (view-flattened-module "typecheck.OnDefines")
+  (view-flattened-module "typecheck.Requires")
+  (view-flattened-module "typecheck.TypeDecl"))
+
+(deftest test-rreload
+  (rimport "org::magnolialang::pgf::tests::TestFmtEngine")
+  (rreload "org::magnolialang::pgf::tests::TestFmtEngine"))
+
+(deftest test-spacer
+  (rimport "org::magnolialang::pgf::tests::RunTestSpacerDirect")
+  (reval-str "runRunTestSpacerDirect()"))
+
+(deftest test-fmt-engine
+  (rimport "org::magnolialang::pgf::tests::TestFmtEngine")
+  (reval-more-str "runTestFmtEngine();")
+  )
+
+(deftest test-token
+  (rimport "org::magnolialang::pgf::tests::TestToken")
+  (reval-str "runTestToken()"))