001    // Copyright (c) 2001 Hursh Jain (http://www.mollypages.org) 
002    // The Molly framework is freely distributable under the terms of an
003    // MIT-style license. For details, see the molly pages web site at:
004    // http://www.mollypages.org/. Use, modify, have fun !
005    
006    package fc.web.page;
007    
008    import java.io.*;
009    import java.util.*;
010    import java.util.regex.*;
011    import fc.io.*;
012    import fc.util.*;
013    
014    /*
015    NOTES
016    
017    Code blocks of the form 
018     [...] 
019    cause problems with java arrays
020    
021    String[] or foo[4] etc.., craps out. So we need to use 
022     [[...]] 
023    for the molly code blocks
024    
025    1. If you are hacking this file, start with parseText()
026    
027    2. Turn the dbg flag to true to see how the parser works
028    
029    3. Keep in mind that the order of switch'es in a case statement in various
030    methods is not always arbitrary (the order matters in this sort
031    of recursive descent parsing)
032    
033    4. Read www.mollypages.org/page/grammar/index.mp for a intro
034    to parsing
035    
036    5. This parser as shipped has a set of regression tests in the
037    fc/web/page/test directory. These consist of a bunch of *.mp
038    files and corresponding *.java files, each of which is known 
039    to be generated properly. If you change stuff around, run these
040    regression tests again by invoking "java fc.web.page.PageParserTest"
041    Note, if you change things such that the .java output of the parser
042    is different, then the tests will fail (since the new .java files
043    of your parser will be different to the test ones shipped in
044    fc/web/page/test. In this case, once you know that your parser works
045    as you like it, then you should create a new baseline for your parser
046    by invoking "java fc.web.page.PageParserTest -generateExpected" and
047    then you can use *that* as the new baseline for further changes in
048    your parser (you may have to modify the *.mp files in /fc/web/page/test
049    to use your new page syntax).
050    
051    6. 
052    When including files, previous versions of the parser constructed a new IncludeFile element
053    which would be invoked when the page was written out. When invoked (via render), that element
054    would creaet a new PageParser and set includeMode = true on that new parser. This new parser 
055    would then parse/write out the subtree of the included file in-line.
056    
057    This inline processing had issues since the included file could not contain import statements,
058    declarations, etc (since those had already been written out by the parent/top level parser).
059    Another hack was to pass the child parser the parent/top level object and give access to the 
060    top level parse root to the child parser (the child parser would have to be invoked immediaately
061    anyway). Also, since inner classes for parse elements are non-static, the separate parser would
062    create a parse tree, and although it would add those classes to the top most parse tree, the 
063    classes themselves (when trying to write) would refer to the separate output stream of the child
064    class (the output stream would also have to be set to the parent class). It was doaable but gets
065    un-neccessarily complex.
066    
067    The only benefit to a seperate parser being able to print start/end sections:
068    >> start include file
069     [..invoke child parser]
070    >> end include file
071    
072    In the current/cleaner approach, I simply insert the included file into the character stream.
073    But there isn't any easy way to track when that stream finishes and the original content starts 
074    again. So we get:
075    
076    >> start include file
077    [..include into original stream and continue parsing]
078    -- no end include file line --
079    
080    */
081    
082    /**
083    Parses a page and writes out the corresponding java file to the specified
084    output. The parser and scanner is combined into one class here for
085    simplicity (a seperate scanner is overkill for a simple LL(1) grammar
086    such as molly pages).
087    
088    @author hursh jain
089    */
090    public final class PageParser
091    {
092    private static final boolean dbg    = false;
093    private static final int     EOF    = -1;
094    private              int     dbgtab = 0;
095    
096    String          classname;
097    String          packagename = Page.PACKAGE_NAME;
098    PageReader      in;
099    PrintWriter     out;
100    Log             log;
101    File            inputFile;
102    File            outputFile;
103    File            contextRoot;
104    boolean         includeMode = false;
105    String          src_encoding;
106    
107    //Read data
108    //we use these since stringbuffer/builders do not have a clear/reset function
109    CharArrayWriter buf = new CharArrayWriter(4096);
110    CharArrayWriter wsbuf = new CharArrayWriter(32);  // ^(whitespace)* 
111    int c = EOF;
112    
113    //PageData
114    List  decl       = new ArrayList();     //declarations
115    List  inc_decl       = new ArrayList();     //external included declarations
116    List  imps           = new ArrayList();     //imports
117    List  tree           = new ArrayList();     //code, exp, text etc.
118    Map   directives     = new HashMap();       //page options
119    Set   circularityTrack = new HashSet();   //track pages already included to stop circular refs
120    
121    /** 
122    The  name ("mimetype") of the[@ mimetype=....] directive. The value of
123    <tt>none</tt> or an empty string will turn off writing any mimetype
124    entirely (the user can then write a mimetype via the {@link
125    javax.servlet.ServletResponse.setContentType} method manually).
126    <p>
127    Note, from {@link
128    javax.servlet.ServletResponse.setContentType ServletResponse}
129    <pre>
130    Note that the character encoding cannot be communicated via HTTP headers
131    if the servlet does not specify a content type; however, it is still used
132    to encode text written via the servlet response's writer.
133    </pre>
134    */
135    public static String d_mimetype = "mimetype";
136    
137    /*
138    this value (or an empty string) for mimetype means no mimetype
139    will be specified (not even the default mimetype)
140    */
141    public static String mimetype_none = "none";
142    
143    /** 
144    The name ("encoding") of the [page encoding=....] directive. 
145    */
146    public static String d_encoding = "encoding";
147    
148    /** 
149    The name ("src-encoding") of the [page src-encoding=....] directive. 
150    */
151    public static String d_src_encoding = "src-encoding";
152    
153    /** The name ("buffersize") of the [page buffersize=....] directive */
154    public static String d_buffersize = "buffersize";
155    
156    /** The name ("out") of the [page out=....] directive */
157    public static String d_out = "out";
158    /** A value ("outputstream") of the [page out=outputstream] directive */
159    public static String d_out_stream1 = "outputstream";
160    /** A value ("outputstream") of the [page out=stream] directive */
161    public static String d_out_stream2 = "stream";
162    /** A value ("writer") of the [page out=writer] directive */
163    public static String d_out_writer = "writer";
164    /** The name of the  ("remove-initial-whitespace") directive */
165    public static String d_remove_initial_emptylines = "remove-initial-emptylines";
166    /** The name of the  ("remove-all-emptylines") directive */
167    public static String d_remove_all_emptylines = "remove-all-emptylines";
168    
169    /* 
170    This constructor for internal use.
171    
172    The parser can be invoked recursively to parse included files as
173    well..that's what the includeMode() does (and this construtor is invoked
174    when including). When including, we already have a output writer
175    created, we use that writer (instead of creating a new one based on
176    src_encoding as we do for in normal page parsing mode).
177    */
178    private PageParser(
179     File contextRoot, File input, PrintWriter outputWriter, String classname, Log log) 
180    throws IOException
181      {
182      this.contextRoot = contextRoot;
183      this.inputFile = input; 
184      this.in  = new PageReader(input);
185      this.out = outputWriter;
186      this.classname = classname;
187      this.log = log;
188    
189      circularityTrack.add(input.getAbsolutePath());
190      }
191    
192    /**
193    Creates a new page parser that will use the default log obtained by
194    {@link Log#getDefault}
195    
196    @param  contextRoot absolute path to the webapp context root directory
197    @param  input   absolute path to the input page file
198    @param  input   absolute path to the output file (to be written to).
199    @param  classname classname to give to the generated java class.
200    */
201    public PageParser(File contextRoot, File input, File output, String classname) 
202    throws IOException
203      {
204      this(contextRoot, input, output, classname, Log.getDefault());
205      }
206    
207    /**
208    Creates a new page parser.
209    
210    @param  contextRoot absolute path to the webapp context root directory
211    @param  input   absolute path to the input page file
212    @param  output    absolute path to the output file (to be written to).
213    @param  classname classname to give to the generated java class.
214    @log  log     destination for internal logging output.
215    */
216    public PageParser(
217      File contextRoot, File input, File output, String classname, Log log) 
218    throws IOException
219      {
220      this.contextRoot = contextRoot;
221      this.inputFile = input; 
222      this.in  = new PageReader(input);
223      this.outputFile = output;
224      this.classname = classname;
225      this.log = log;
226    
227      circularityTrack.add(input.getAbsolutePath());
228      }
229    
230    void append(final int c)
231      {
232      Argcheck.istrue(c >= 0, "Internal error: recieved c=" + c);
233      buf.append((char)c);
234      }
235    
236    void append(final char c)
237      {
238      buf.append(c);
239      }
240    
241    void append(final String str)
242      {
243      buf.append(str);
244      }
245    
246    /* not used anymore */
247    PageParser includeMode()
248      {
249      includeMode = true;
250      return this;
251      }
252    
253    /**
254    Parses the page. If the parse is successful, the java source will be
255    generated.
256    
257    @throws IOException   a parse failure occurred. The java source file
258                may or may not be properly generated or written
259                in this case.
260    */
261    public void parse() throws IOException
262      {
263      parseText();  
264      if (! includeMode)  {
265        writePage();
266        out.close();
267        }
268      else{
269        out.flush();
270        }
271      in.close();
272      }
273    
274    //util method for use in the case '[' branch of parseText below.
275    private Text newTextNode()
276      {
277      Text text = new Text(buf);
278      tree.add(text);
279      buf.reset();
280      return text;
281      }
282      
283    void parseText() throws IOException
284      {
285      if (dbg) dbgenter(); 
286          
287      while (true)
288        { 
289        c = in.read();
290        
291        if (c == EOF) {
292          tree.add(new Text(buf));
293          buf.reset();
294          break;
295          }
296          
297        switch (c)
298          { 
299          //Escape start tags
300          case '\\':
301            /*  we don't need to do this: previously, expressions
302            were [...] but now they are [=...], previously we needed
303            to escape \[[ entirely (since if we escaped \[ the second
304            [ would start an expression
305            */
306            /*        
307            if (in.match("[["))  
308              append("[[");
309            */
310            //escape only \[... otherwise leave \ alone
311            if (in.match("["))
312              append("[");
313            else
314              append(c);
315            break;
316    
317          case '[':
318            /* suppose we have
319            \[[
320            escape handling above will capture \[
321            then the second '[' drops down here. Good so far.
322            But we must not create a new text object here by
323            default...only if we see another [[ or [= or [include or
324            whatever. 
325            */
326            /*
327            But creating a text object at the top is easier
328            then repeating this code at every if..else branch below
329            but this creates superfluous line breaks.
330            
331            hello[haha]world
332            -->prints as-->
333            hello  (text node 1)
334            [haha] (text node 2)
335            world  (text node 3)
336            --> we want
337            hello[haha]world (text node 1)
338            */
339              
340            if (in.match('[')) { 
341              newTextNode();
342              parseCode(); 
343              }
344            else if (in.match('=')) {
345              Text text = newTextNode();
346              parseExpression(text);
347              }
348            else if (in.match('!')) {
349              newTextNode();
350              parseDeclaration();
351              }
352            else if (in.match("/*")) {
353              newTextNode();
354              parseComment(); 
355              }
356            else if (in.matchIgnoreCase("page")) {
357              newTextNode();
358              parseDirective();
359              }
360            //longest match: "include-file" etc., last: "include"
361            else if (in.matchIgnoreCase("include-file")) {
362              newTextNode();
363              parseIncludeFile();
364              }
365            else if (in.matchIgnoreCase("include-decl")) {
366              newTextNode();
367              parseIncludeDecl();
368              }
369            else if (in.matchIgnoreCase("include")) {
370              newTextNode();
371              parseInclude();
372              }
373            else if (in.matchIgnoreCase("forward")) {
374              newTextNode();
375              parseForward();
376              }
377            else if (in.matchIgnoreCase("import")) {
378              newTextNode();
379              parseImport();
380              }
381            else  {
382              //System.out.println("c1=" + (char)c);
383              append(c);
384              }
385            break;  
386      
387          default:
388            //System.out.println("c2=" + (char)c);
389            append(c);
390            
391          } //switch    
392        } //while
393        
394      if (dbg) dbgexit(); 
395      }
396      
397    void parseCode() throws IOException
398      {
399      if (dbg) dbgenter(); 
400    
401      int startline = in.getLine();
402      int startcol = in.getCol();
403      
404      while (true)
405        {
406        c = in.read();  
407      
408        switch (c) /* the order of case tags is important. */
409          {
410          case EOF:
411            unclosed("code", startline, startcol);
412            if (dbg) dbgexit(); 
413            return;
414    
415          case '/':   //Top level:  // and /* comments
416            append(c);
417            c = in.read();
418            append(c);
419            if (c == '/') 
420              appendCodeSlashComment();
421            else if (c == '*') 
422              appendCodeStarComment();
423              break;        
424        
425          case '"':     //strings outside of any comment
426            append(c);
427            appendCodeString();  
428            break;
429            
430          case '\'':
431            append(c);
432            appendCodeCharLiteral();
433            break;
434            
435          case ']':
436            if (in.match(']')) {
437              tree.add(new Code(buf));
438              buf.reset();
439              if (dbg) dbgexit(); 
440              return;
441              }
442            else {
443              append(c);
444              }
445            break;
446          
447          /* 
448          a hash by itself on a line starts a hash section.
449          whitespace before the # on that line is used as an
450          printing 'out' statements for that hash.
451          
452          for (int n = 0; n < ; n++) {
453          ....# foo #
454          | }
455          |=> 4 spaces 
456          so nice if generated code looked like:
457          
458          for (int n = 0; n < ; n++) {
459              out.print(" foo ");
460              }
461          */
462          case '\n':
463          case '\r':
464            append(c);       //the \n or \r just read
465            readToFirstNonWS();  //won't read past more newlines 
466            //is '#' is first non-ws on this line ?
467            c = in.read();
468            if (c == '#') {           
469              tree.add(new Code(buf));
470              buf.reset();
471              //whitespace provides indentation offset
472              parseHash(wsbuf.toString()); 
473              }
474            else{
475              append(wsbuf.toString());  //wsbuf contains codetext
476              //let other cases also handle first non-ws or EOF
477              in.unread();    
478              }
479            break;
480          
481          /* in this case, hash does not start on a new line, like:
482             for (...) { #
483          */
484          case '#':
485            tree.add(new Code(buf));
486            buf.reset();
487            parseHash(null);
488            break;  
489          
490          default:
491            append(c);
492          } //switch    
493        } //while
494      }
495    
496    void parseHash(String offset) throws IOException
497      {
498      if (dbg) dbgenter(); 
499    
500      int startline = in.getLine();
501      int startcol = in.getCol();
502    
503      while (true)
504        {
505        c = in.read();  
506      
507        switch (c)
508          {
509          case EOF: 
510            unclosed("hash", startline, startcol);
511            if (dbg) dbgexit(); 
512            return;
513    
514          //special case: very common and would be a drag to escape
515          //this every time:
516          //  # <table bgcolor="#ffffff">....   #
517          //Now, all of:
518          //  bgcolor="#xxx"  
519          //  bgcolor='#xxx'
520          //  bgcolor="\#xxx" 
521          //will work the same and give: bgcolor="#xxx"
522          //1)
523          //However to get a:
524          //  bgcolor=#xxx    (no quoted around #xxx)
525          //we still have to say:
526          //  bgcolor=\#xxx   
527          //2)
528          //Of course, since we special case this, then:
529          //  #"bar"#
530          // that ending # is lost and we end up with
531          //  #"bar"  with no closing hash
532          //So we need to make sure that we write:
533          //  #"bar" #
534          // instead
535    
536          case '\'':
537          case '"':
538            append(c);
539            if (in.match('#')) 
540              append('#');
541            break;
542            
543          case '\\':
544            if (in.match('[')) 
545              append('[');      
546            else if (in.match('#'))
547              append('#');
548            else
549              append(c);
550            break;
551            
552          case '[':
553            if (in.match('=')) {
554              Hash hash = new Hash(offset, buf);
555              tree.add(hash);
556              buf.reset();
557              parseExpression(hash);
558              }
559            else{
560              append(c);
561              }
562            break;
563    
564          /*
565          this case is not needed but is a bit of a optimization
566          for (int n = 0; n < 1; n++) {
567            #
568            foo
569          ....#...NL
570            }
571          avoids printing the dots (spaces) and NL in this case
572          (the newline after foo is still printed)
573          */
574          case '\n':
575          case '\r':
576            append(c);
577            readToFirstNonWS(); 
578            c = in.read();
579            //'#' is first non-ws on the line
580            if (c == '#') {
581              tree.add(new Hash(offset, buf));
582              buf.reset();
583              //skipIfWhitespaceToEnd();
584              if (dbg) dbgexit(); 
585              return;
586              }
587            else {
588              append(wsbuf.toString());
589              in.unread(); //let other cases also handle first non-ws   
590              }
591            break;
592    
593          case '#':
594            tree.add(new Hash(offset, buf));  
595            //skipIfWhitespaceToEnd();
596            buf.reset();
597            if (dbg) dbgexit(); 
598            return;
599            
600          default:
601            append(c);
602          }  //switch 
603        } //while
604      }
605    
606    /**
607    [page <<<FOO]
608    ...as-is..no parse, no interpolation..
609    FOO
610    */
611    void parseHeredoc(StringBuilder directives_buf) throws IOException
612      {
613      if (dbg) dbgenter(); 
614    
615      int startline = in.getLine();
616      int startcol = in.getCol();
617          
618      int i = directives_buf.indexOf("<<<"); /* "<<<".length = 3 */
619      CharSequence subseq = directives_buf.substring(
620                i+3, 
621                /*directives_buf does not have a ending ']' */
622                directives_buf.length() 
623                );
624        
625      final String      heredoc     = subseq.toString().trim();
626      final int         heredoc_len = heredoc.length();
627      final CharArrayWriter heredoc_buf = new CharArrayWriter(2048);
628    
629      /* 
630      the ending heredoc after newline speeds things up a bit
631      which is why is traditionally used i guess, otherwise
632      we have to try a full match every first match. this 
633      implementation doesn't care where the ending heredoc
634      appears (can be anywhere)...simplifies the implementation.
635      */
636      
637      while (true)
638        { 
639        c = in.read();
640        
641        if (c == EOF) {
642          unclosed("heredoc: <<<"+heredoc, startline, startcol);
643          break;
644          }
645          
646        if (c == heredoc.charAt(0))
647          {
648          boolean matched = true;
649          if (heredoc_len > 1) {
650            matched = in.match(heredoc.substring(1));
651            }
652          if (matched) {  
653            tree.add(new Heredoc(heredoc_buf));
654            break;
655            }
656          }
657        
658        //default action
659        heredoc_buf.append((char)c);  
660        } //while
661        
662      if (dbg) dbgexit(); 
663      }
664    
665    /*
666    Text is the parent node for the expression. A new expression is parsed,
667    created and added to the text object by this method
668    */
669    void parseExpression(Element parent) throws IOException
670      {
671      if (dbg) dbgenter(); 
672    
673      int startline = in.getLine();
674      int startcol = in.getCol();
675    
676      while (true)
677        {
678        c = in.read();      
679      
680        switch (c)
681          {
682          case EOF:
683            unclosed("expression", startline, startcol);
684            if (dbg) dbgexit(); 
685            return;
686    
687          case '\\':
688            if (in.match(']')) 
689              append(']');    
690            else
691              append(c);
692            break;
693    
694          case ']':
695            if (buf.toString().trim().length() == 0)
696              error("Empty expression not allowed", startline, startcol);
697            parent.addExp(new Exp(buf));
698            buf.reset();  
699            if (dbg) dbgexit(); 
700            return;
701            
702          default:
703            append(c);
704          }
705        }
706      }
707    
708    void parseComment() throws IOException
709      {
710      if (dbg) dbgenter(); 
711    
712      int startline = in.getLine();
713      int startcol = in.getCol();
714    
715      while (true)
716        {
717        c = in.read();      
718      
719        switch (c)
720          {
721          case EOF:
722            unclosed("comment", startline, startcol);
723            if (dbg) dbgexit(); 
724            return;
725            
726          case '*':
727            if (in.match("/]"))
728              {
729              tree.add(new Comment(buf));
730              buf.reset();  
731              if (dbg) dbgexit(); 
732              return;
733              }
734            else
735              append(c);  
736            break;
737          
738          default:
739            append(c);
740          }
741        }
742      }
743    
744    void parseDeclaration() throws IOException
745      {
746      if (dbg) dbgenter(); 
747      int startline = in.getLine();
748      int startcol = in.getCol();
749    
750      while (true)
751        {
752        c = in.read();      
753      
754        switch (c)
755          {
756          case EOF:
757            unclosed("declaration", startline, startcol);
758            if (dbg) dbgexit(); 
759            return;
760          
761          case '!':
762            if (in.match(']')) {
763              decl.add(new Decl(buf));
764              buf.reset();  
765              if (dbg) dbgexit(); 
766              return;
767              }
768            else{
769              append(c);
770              }
771            break;
772    
773          //top level // and /* comments, ']' (close decl tag)
774          //is ignored within them
775          case '/':   
776            append(c);
777            c = in.read();
778            append(c);
779            if (c == '/') 
780              appendCodeSlashComment();
781            else if (c == '*') 
782              appendCodeStarComment();
783              break;        
784        
785          //close tags are ignored within them
786          case '"':     //strings outside of any comment
787            append(c);
788            appendCodeString();  
789            break;
790            
791          case '\'':
792            append(c);
793            appendCodeCharLiteral();
794            break;
795                
796          default:
797            append(c);
798          }
799        }
800    
801      }
802    
803    void parseDirective() throws IOException
804      {
805      if (dbg) dbgenter(); 
806    
807      int startline = in.getLine();
808      int startcol = in.getCol();
809    
810      StringBuilder directives_buf = new StringBuilder(1024);
811    
812      while (true)
813        {
814        c = in.read();      
815      
816        switch (c)
817          {
818          case EOF:
819            unclosed("directive", startline, startcol);
820            if (dbg) dbgexit(); 
821            return;
822            
823          case ']':
824            if (directives_buf.indexOf("<<<") >= 0)  {
825              parseHeredoc(directives_buf); 
826              }
827            else{/* other directives used at page-generation time */
828              addDirectives(directives_buf);
829              }
830              
831            if (dbg) dbgexit(); 
832            return;
833          
834          default:
835            directives_buf.append((char)c);
836          }
837        }
838    
839      }
840    
841    //[a-zA-Z_\-0-9] == ( \w | - )
842    static final Pattern directive_pat = Pattern.compile(
843      //foo = "bar baz" (embd. spaces)
844      "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*\"((?:.|\r|\n)+?)\""  
845      + "|"
846      //foo = "bar$@#$" (no spaces) OR foo = bar (quotes optional)
847      + "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*(\\S+)" 
848      );
849      
850        
851    void addDirectives(StringBuilder directives_buf) throws ParseException
852      {
853      if (dbg) {
854        dbgenter(); 
855        System.out.println("-------directives section--------");
856        System.out.println(directives_buf.toString());
857        System.out.println("-------end directives-------");
858        }
859      
860      String name, value;
861      try {
862        Matcher m = directive_pat.matcher(directives_buf);
863        while (m.find()) 
864          {
865          if (dbg) System.out.println(">>>>[0]->" + m.group() 
866            + "; [1]->" + m.group(1)  
867            + " [2]->" + m.group(2)  
868            + " [3]->" + m.group(3)  
869            + " [4]->" + m.group(4));
870            
871          name = m.group(1) != null ? m.group(1).toLowerCase() :
872                        m.group(3).toLowerCase();
873          value = m.group(2) != null ? m.group(2).toLowerCase() :
874                         m.group(4).toLowerCase();
875    
876          if (name.equals(d_buffersize)) 
877            {
878            //can throw parse exception
879            directives.put(name, 
880              IOUtil.stringToFileSize(value.replace("\"|'",""))); 
881            }
882          else if (name.equals(d_encoding)) {
883            directives.put(name, value.replace("\"|'",""));       
884            }
885          else if (name.equals(d_src_encoding)) {
886            directives.put(name, value.replace("\"|'",""));       
887            } 
888          else if (name.equals(d_mimetype)) {
889            directives.put(name, value.replace("\"|'",""));       
890            }
891          else if (name.equals(d_out)) {
892            directives.put(name, value.replace("\"|'",""));       
893            } 
894          else if (name.equals(d_remove_initial_emptylines)) {
895            directives.put(name, value.replace("\"|'",""));       
896            } 
897          else if (name.equals(d_remove_all_emptylines)) {
898            directives.put(name, value.replace("\"|'",""));       
899            } 
900          //else if .... other directives here as needed....
901          else 
902            throw new Exception("Do not understand directive: " + m.group());
903          }
904        if (dbg) System.out.println("Added directives: " + directives);
905        }
906      catch (Exception e) {
907        throw new ParseException("File: " + inputFile.getAbsolutePath() 
908                      + ";\n" + e.toString());
909        }
910    
911      if (dbg) dbgexit(); 
912      }
913    
914    void parseIncludeFile() throws IOException
915      {
916      if (dbg) dbgenter(); 
917    
918      int startline = in.getLine();
919      int startcol = in.getCol();
920      String option = null;
921      
922      while (true)
923        {
924        c = in.read();      
925      
926        switch (c)
927          {
928          case EOF:
929            unclosed("include-file", startline, startcol);
930            if (dbg) dbgexit(); 
931            return;
932            
933          case '[':
934            if (in.match('=')) {
935      //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\"
936      //in [include-file... section starting at:", startline, startcol);
937      //instead of warn, we will error out. failing early is better.
938      //this does preclude having '[=' in the file name, but it's a good
939      //tradeoff
940              error("Expressions cannot exist in file includes. The offending static-include section starts at:", startline, startcol);
941              }
942            append(c);
943            break;
944          
945          case ']':
946            includeFile(buf, option); /* not added in the tree, just included in the stream */
947            buf.reset();  
948            if (dbg) dbgexit(); 
949            return;
950          
951          case 'o':
952            if (! in.match("ption"))
953              append(c);
954            else{
955              skipWS();
956              if (! in.match("=")) {
957                error("bad option parameter in file include: ", startline, startcol);
958                }
959              skipWS();
960              
961              int c2;
962              StringBuilder optionbuf = new StringBuilder();
963              while (true) {
964                c2 = in.read();
965                if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
966                  in.unread();
967                  break;
968                  }
969                optionbuf.append((char)c2);
970                }
971              
972              option = optionbuf.toString();
973              //System.out.println(option);
974              } //else
975            break;
976      
977          default:
978            append(c);
979          }
980        }
981      }
982    
983    void parseIncludeDecl() throws IOException
984      {
985      if (dbg) dbgenter(); 
986    
987      int startline = in.getLine();
988      int startcol = in.getCol();
989      String option = null;
990      
991      while (true)
992        {
993        c = in.read();      
994      
995        switch (c)
996          {
997          case EOF:
998            unclosed("include-decl", startline, startcol);
999            if (dbg) dbgexit(); 
1000            return;
1001            
1002          case '[':
1003            if (in.match('=')) {
1004        //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\" in [include-static... section starting at:", startline, startcol);
1005        //we will throw an exception. failing early is better. this
1006        //does preclude having '[=' in the file name, but it's a good tradeoff
1007              error("Expressions cannot exist in include-decl. The offending static-include section starts at:", startline, startcol);
1008              }
1009            append(c);
1010            break;
1011          
1012          case ']':
1013            IncludeDecl i = new IncludeDecl(buf);
1014            if (option != null)
1015              i.setOption(option);
1016            inc_decl.add(i);
1017            buf.reset();  
1018            if (dbg) dbgexit(); 
1019            return;
1020          
1021          case 'o':
1022            if (! in.match("ption"))
1023              append(c);
1024            else{
1025              skipWS();
1026              if (! in.match("=")) {
1027                error("bad option parameter in include-code: ", startline, startcol);
1028                }
1029              skipWS();
1030              
1031              int c2;
1032              StringBuilder optionbuf = new StringBuilder();
1033              while (true) {
1034                c2 = in.read();
1035                if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
1036                  in.unread();
1037                  break;
1038                  }
1039                optionbuf.append((char)c2);
1040                }
1041              
1042              option = optionbuf.toString();
1043              //System.out.println(option);
1044              } //else
1045            break;
1046      
1047          default:
1048            append(c);
1049          }
1050        }
1051      }
1052    
1053    //the filename/url can be optionally double quoted. leading/trailing
1054    //double quotes (if any) are ignored when an include is rendered...
1055    //this way there isn't any additional parsing needed here...I could
1056    //ignore the optional quote here (and that's the formal proper way) 
1057    //and then not move the ignore quote logic into the render() method but
1058    //this way is good too...and simpler..
1059    //same goes for the other parseIncludeXX/ForwardXX functions.
1060    void parseInclude() throws IOException
1061      {
1062      if (dbg) dbgenter(); 
1063    
1064      int startline = in.getLine();
1065      int startcol = in.getCol();
1066      Include include = new Include();
1067      while (true)
1068        {
1069        c = in.read();      
1070      
1071        switch (c)
1072          {
1073          case EOF:
1074            unclosed("include", startline, startcol);
1075            if (dbg) dbgexit(); 
1076            return;
1077            
1078          case '[':
1079            if (in.match('=')) {
1080              include.add(buf);
1081              buf.reset();
1082              parseExpression(include);
1083              }
1084            else{
1085              append(c);
1086              }
1087            break;
1088          
1089          case ']':
1090            include.add(buf);
1091            tree.add(include);
1092            buf.reset();  
1093            if (dbg) dbgexit(); 
1094            return;
1095          
1096          default:
1097            append(c);
1098          }
1099        }
1100      }
1101    
1102    void parseForward() throws IOException
1103      {
1104      if (dbg) dbgenter(); 
1105    
1106      int startline = in.getLine();
1107      int startcol = in.getCol();
1108    
1109      Forward forward = new Forward();
1110      while (true)
1111        {
1112        c = in.read();      
1113      
1114        switch (c)
1115          {
1116          case EOF:
1117            unclosed("forward", startline, startcol);
1118            if (dbg) dbgexit(); 
1119            return;
1120            
1121          case '[':
1122            if (in.match('=')) {
1123              forward.add(buf);
1124              buf.reset();
1125              parseExpression(forward);
1126              }
1127            else{
1128              append(c);
1129              }
1130            break;
1131          
1132          case ']':
1133            forward.add(buf);
1134            tree.add(forward);
1135            buf.reset();  
1136            if (dbg) dbgexit(); 
1137            return;
1138          
1139          default:
1140            append(c);
1141          }
1142        }
1143      }
1144    
1145    //we need to parse imports seperately because they go outside
1146    //a class declaration (and [!...!] goes inside a class)
1147    //import XXX.*;
1148    //class YYY {
1149    //[!....stuff from here ....!]
1150    //...
1151    void parseImport() throws IOException
1152      {
1153      if (dbg) dbgenter(); 
1154    
1155      int startline = in.getLine();
1156      int startcol = in.getCol();
1157    
1158      while (true)
1159        {
1160        c = in.read();      
1161      
1162        switch (c)
1163          {
1164          case EOF:
1165            unclosed("import", startline, startcol);
1166            if (dbg) dbgexit(); 
1167            return;
1168          
1169          case '\n':
1170            imps.add(new Import(buf));
1171            buf.reset();
1172            break;
1173            
1174          case ']':
1175            imps.add(new Import(buf));
1176            buf.reset();  
1177            if (dbg) dbgexit(); 
1178            return;
1179          
1180          default:
1181            append(c);
1182          }
1183        }
1184      }
1185    
1186    /*
1187    Called when // was read at the top level inside a code block. Appends
1188    the contents of a // comment to the buffer (not including the trailing
1189    newline)
1190    */
1191    void appendCodeSlashComment() throws IOException
1192      {
1193      if (dbg) dbgenter();
1194      
1195      while (true) 
1196        {
1197        c = in.read();
1198        
1199        if (c == EOF)
1200          break;
1201      
1202        //do not append \r, \r\n, or \n, that finishes the // comment
1203        //we need that newline to figure out if the next line is a hash
1204        //line
1205        if (c == '\r') {
1206          in.unread();
1207          break;
1208          }
1209        
1210        if (c == '\n') {
1211          in.unread();
1212          break;  
1213          }
1214    
1215        append(c);
1216        }
1217      
1218      if (dbg) dbgread("CodeSLASHComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1219      if (dbg) dbgexit();
1220      }
1221    
1222    /*
1223    Called when /* was read at the top level inside a code block. Appends
1224    the contents of a /*comment to the buffer. (not including any trailing
1225    newline or spaces)
1226    */
1227    void appendCodeStarComment() throws IOException
1228      {
1229      if (dbg) dbgenter(); 
1230      
1231      while (true) 
1232        {
1233        c = in.read();  
1234    
1235        if (c == EOF)
1236          break;
1237      
1238        append(c);
1239        
1240        if (c == '*') 
1241          {
1242          if (in.match('/')) {
1243            append('/');
1244            break;
1245            }
1246          }
1247        }
1248    
1249      if (dbg) dbgread("CodeSTARComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1250      if (dbg) dbgexit(); 
1251      }
1252    
1253    /*
1254    Called (outside of any comments in the code block) when: 
1255    --> parseCode()
1256         ... "
1257             ^ (we are here)
1258    */
1259    void appendCodeString() throws IOException
1260      {
1261      if (dbg) dbgenter(); 
1262    
1263      int startline = in.getLine();
1264      int startcol = in.getCol();
1265    
1266      while (true) 
1267        {
1268        c = in.read();
1269      
1270        if (c == EOF || c == '\r' || c == '\n')
1271          unclosed("string literal", startline, startcol);
1272      
1273        append(c);
1274      
1275        if (c == '\\') {
1276          c = in.read();
1277          if (c == EOF)
1278            unclosed("string literal", startline, startcol);
1279          else {
1280            append(c);
1281            continue;   //so \" does not hit the if below and break
1282            }
1283          }
1284        
1285        if (c == '"')
1286          break;
1287        }
1288    
1289      if (dbg) dbgread("appendCodeString Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1290      if (dbg) dbgexit(); 
1291      }
1292    
1293    
1294    /*
1295    Called (outside of any comments in the code block) when: 
1296    --> parseCode()
1297         ... '
1298             ^ (we are here)
1299    */
1300    void appendCodeCharLiteral() throws IOException
1301      {
1302      if (dbg) dbgenter(); 
1303    
1304      int startline = in.getLine();
1305      int startcol = in.getCol();
1306    
1307      while (true) 
1308        {
1309        c = in.read();
1310      
1311        if (c == EOF || c == '\r' || c == '\n')
1312          unclosed("char literal", startline, startcol);
1313      
1314        append(c);
1315      
1316        if (c == '\\') {
1317          c = in.read();
1318          if (c == EOF)
1319            unclosed("char literal", startline, startcol);
1320          else {
1321            append(c);
1322            continue;   //so \' does not hit the if below and break
1323            }
1324          }
1325        
1326        if (c == '\'')
1327          break;
1328        }
1329    
1330      if (dbg) dbgread("appendCodeCharLiteral Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1331      if (dbg) dbgexit(); 
1332      }
1333    
1334    
1335    /*
1336    Reads from the current position till the first nonwhitespace char, EOF or
1337    newline is encountered. Reads are into the whitespace buffer. does not
1338    consume the character past the non-whitespace character and does
1339    NOT read multiple lines of whitespace.
1340    */
1341    void readToFirstNonWS() throws IOException 
1342      {
1343      wsbuf.reset();
1344    
1345      while (true)
1346        {
1347        c = in.read();
1348      
1349        if (c == '\r' || c == '\n')
1350          break;
1351          
1352        if (c == EOF || ! Character.isWhitespace(c))
1353          break;
1354      
1355        wsbuf.append((char)c);
1356        }
1357        
1358      in.unread();
1359      }
1360    
1361    //skip till end of whitespace or EOF. does not consume any chars past 
1362    //the whitespace.
1363    void skipWS() throws IOException
1364      {
1365      int c2 = EOF;
1366      while (true) {
1367        c2 = in.read();
1368        if (c2 == EOF || ! Character.isWhitespace(c2)) {
1369          in.unread();
1370          break;
1371          }
1372        } 
1373      }
1374      
1375    //skips to the end of line if the rest of the line is (from the current
1376    //position), all whitespace till the end. otherwise, does not change 
1377    //current position. consumes trailing newlines (if present) when reading 
1378    //whitespace.
1379    void skipIfWhitespaceToEnd() throws IOException
1380      {
1381      int count = 0;
1382      
1383      while (true) 
1384        {
1385        c = in.read();
1386          count++;
1387    
1388        if (c == '\r') {
1389          in.match('\n');
1390          return;
1391          }
1392          
1393        if (c == '\n' || c == EOF)
1394          return;
1395          
1396        if (! Character.isWhitespace(c))
1397          break;
1398          }
1399    
1400      in.unread(count);
1401      }
1402    
1403    //not used anymore but left here for potential future use. does not
1404    //consume the newline (if present)
1405    void skipToLineEnd() throws IOException 
1406      {
1407        while (true) 
1408          {
1409          int c = in.read();
1410          if (c == EOF) {
1411            in.unread();
1412          break;
1413            }
1414          if (c == '\n' || c == '\r') { 
1415            in.unread();
1416            break;
1417            }
1418          }
1419        }
1420    
1421    String quote(final char c) 
1422      {
1423        switch (c)
1424          {
1425          case '\r':
1426                return "\\r";
1427                
1428          case '\n':
1429                return "\\n";
1430     
1431        case '\"':
1432          //can also say: new String(new char[] {'\', '"'})
1433                return "\\\"";    //--> \"
1434     
1435        case '\\':
1436                return "\\\\";
1437        
1438          default:
1439            return String.valueOf(c);
1440          }
1441        }
1442    
1443    //======= util and debug methods ==========================
1444    String methodName(int framenum)
1445      {
1446      StackTraceElement ste[] = new Exception().getStackTrace();
1447      //get method that called us, we are ste[0]
1448      StackTraceElement st = ste[framenum];
1449      String file = st.getFileName();
1450      int line = st.getLineNumber();
1451      String method = st.getMethodName();
1452      String threadname = Thread.currentThread().getName();
1453      return method + "()";   
1454      }
1455    
1456    void dbgenter() {
1457      System.out.format("%s-->%s\n", StringUtil.repeat('\t', dbgtab++), methodName(2));
1458      }
1459      
1460    void dbgexit() {
1461      System.out.format("%s<--%s\n", StringUtil.repeat('\t', --dbgtab), methodName(2));
1462      }
1463    
1464    void dbgread(String str) {
1465      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1466      }
1467    
1468    void dbgread(String str, List list) {
1469      System.out.format("%s %s: ", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1470      for (int n = 0; n < list.size(); n++) {
1471        System.out.print( StringUtil.viewableAscii( (String)list.get(n) ) );
1472        }
1473      System.out.println("");
1474      }
1475    
1476    void dbgread(char c) {
1477      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(c));
1478      }
1479    
1480    void dbgread(CharArrayWriter buf) {
1481      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(buf.toString()));
1482      }
1483    
1484    void unclosed(String blockname, int startline, int startcol) throws IOException
1485      {
1486      throw new IOException(blockname + " tag not closed.\nThis tag was possibly opened in: \nFile:"
1487        + inputFile + ", line:" 
1488        + startline + " column:" + startcol +
1489        ".\nCurrent line:" + in.getLine() + " column:" + in.getCol());  
1490      }
1491    
1492    void error(String msg, int line, int col) throws IOException
1493      {
1494      throw new IOException("Error in File:" + inputFile + " Line:" + line + " Col:" + col + " " + msg);  
1495      }
1496    
1497    void error(String msg) throws IOException
1498      {
1499      throw new IOException("Error in File:" + inputFile + " " + msg);  
1500      }
1501    
1502    //============== Non Parsing methods ================================
1503    void o(Object str) {
1504      out.print(str);
1505      }
1506    
1507    void ol(Object str) {
1508      out.println(str); 
1509      }
1510    
1511    void ol() {
1512      out.println();
1513      }
1514      
1515    /**
1516    Returns the src_encoding directive (if any) defined in this page.
1517    */
1518    String getSourceEncoding() {
1519      return src_encoding;
1520      }
1521    
1522    /* 
1523    include an external file whose contents will be rendered as part of the page.
1524    */ 
1525    void includeFile(CharArrayWriter buf, String option) throws IOException
1526      {
1527      String str;
1528      
1529      if (dbg) dbgread("<new INCLUDE-FILE> "); 
1530      str = removeLeadingTrailingQuote(buf.toString().trim());
1531      
1532      File includeFile = null;
1533      File parentDir = inputFile.getParentFile();
1534      if (parentDir == null) {
1535        parentDir = new File(".");
1536        }
1537    
1538      if (str.startsWith("/"))
1539        includeFile = new File(contextRoot, str);
1540      else
1541        includeFile = new File(parentDir, str);
1542            
1543      //System.out.println(">>>>>>>>>> f="+f +";root="+contextRoot);
1544          
1545      if (! includeFile.exists()) {
1546        throw new IOException("Include file does not exist: " + includeFile.getCanonicalPath());
1547        }
1548    
1549      if (circularityTrack.contains(includeFile.getAbsolutePath())) {
1550         throw new IOException("Circularity detected when including: " + includeFile.getCanonicalPath() + "\nAlready included the following files: " + circularityTrack);
1551        }
1552    
1553      tree.add(new MollyComment(
1554        "//>>>START INCLUDE from: " + includeFile.getAbsolutePath()));
1555        
1556      /*
1557        PageParser pp = new PageParser(contextRoot, includeFile, out, classname, log);
1558        pp.includeMode().parse();  //writes to out
1559      */
1560      
1561      in.insertIntoStream(includeFile);
1562    
1563      /* this is printed immediately before the inserted contents can be processed, so don't add this */
1564      /*
1565      tree.add(new MollyComment(
1566        "//>>>END INCLUDE from: " + includeFile.getAbsolutePath()));
1567      */
1568      
1569      circularityTrack.add(includeFile.getAbsolutePath());
1570      }
1571    
1572      
1573    void writePage() throws IOException
1574      { 
1575      if (! includeMode)
1576        {
1577        if (directives.containsKey(d_src_encoding)) {
1578          this.src_encoding = (String) directives.get(d_src_encoding);
1579          this.src_encoding = removeLeadingTrailingQuote(this.src_encoding);
1580          }
1581      
1582        //create a appropriate PrintWriter based on either the default
1583        //jave encoding or the page specified java encoding
1584        //the java source file will be written out in this encoding
1585      
1586        FileOutputStream  fout = new FileOutputStream(outputFile);
1587        OutputStreamWriter  fw   = (src_encoding != null) ?
1588            new OutputStreamWriter(fout, src_encoding) :
1589            new OutputStreamWriter(fout);
1590            
1591        out = new PrintWriter(new BufferedWriter(fw));
1592        }
1593        
1594      if (! includeMode) 
1595        {
1596        writePackage();
1597        writeImports();
1598        
1599        o ("public class ");
1600        o (classname);
1601        ol(" extends fc.web.page.PageImpl");
1602        ol("{");
1603        }
1604    
1605      writeFields();
1606    
1607      if (! includeMode) {
1608        writeConstructor();
1609        }
1610        
1611      writeMethods();
1612      
1613      if (! includeMode) {
1614        ol("}");
1615        }
1616      }
1617    
1618    void writePackage()
1619      {
1620      o ("package ");
1621      o (packagename);
1622      ol(";");
1623      ol();
1624      }
1625      
1626    void writeImports() throws IOException
1627      {
1628      ol("import javax.servlet.*;");
1629      ol("import javax.servlet.http.*;");
1630      ol("import java.io.*;");
1631      ol("import java.util.*;");
1632      //write this in case (very rare) that a page overrides the 
1633      //Page.init()/destory methods [we need pageservlet for init(..)]
1634      ol("import fc.web.page.PageServlet;");
1635      for (int n = 0; n < imps.size(); n++) {
1636        ((Element)imps.get(n)).render();
1637        ol();
1638        }
1639      ol();
1640      }
1641    
1642    void writeFields()
1643      {
1644      }
1645    
1646    void writeConstructor()
1647      {
1648      }
1649    
1650    void writeMethods() throws IOException
1651      {
1652      writeDeclaredMethods();
1653      writeIncludedMethods();
1654      writeRenderMethod();
1655      }
1656      
1657    void writeDeclaredMethods() throws IOException
1658      {
1659      for (int n = 0; n < decl.size(); n++) {
1660        ((Element)decl.get(n)).render();
1661        }
1662      
1663      if (decl.size() > 0)
1664        ol();
1665      }
1666    
1667    void writeIncludedMethods() throws IOException
1668      {
1669      for (int n = 0; n < inc_decl.size(); n++) {
1670        ((Element)inc_decl.get(n)).render();
1671        }
1672        
1673      if (inc_decl.size() > 0)
1674        ol();
1675      }
1676    
1677    void writeRenderMethod() throws IOException
1678      {
1679      if  (! includeMode) {
1680        writeRenderTop();
1681        }
1682      
1683      /* remove leading emptylines if directed */
1684      if (directives.containsKey(d_remove_initial_emptylines)) 
1685        {
1686        if (dbg) System.out.println("[d_remove_initial_emptylines] directive found, removing leading whitepace");
1687        boolean white_space = true;
1688        
1689        //have to use iterator when removing while transversing
1690        Iterator it = tree.iterator();
1691        while (it.hasNext()) 
1692          {
1693          Element e = (Element) it.next();
1694          if (e instanceof Text) 
1695            {
1696            Text t = (Text) e;
1697            if (t.isOnlyWhitespace()) {
1698              it.remove();
1699              }
1700            }
1701          else{
1702            if (! (e instanceof Comment || e instanceof Decl || e instanceof MollyComment)) {
1703              //the initial whitespace mode is not applicable since some other declaration seen
1704              break;
1705              }
1706            }
1707          }
1708        }
1709    
1710      /* remove all empty lines if directed */
1711      if (directives.containsKey(d_remove_all_emptylines)) 
1712        {
1713        if (dbg) System.out.println("[d_remove_all_emptylines] directive found, removing leading whitepace");
1714        boolean white_space = true;
1715        
1716        //have to use iterator when removing while transversing
1717        Iterator it = tree.iterator();
1718        while (it.hasNext()) 
1719          {
1720          Element e = (Element) it.next();
1721          if (e instanceof Text) 
1722            {
1723            Text t = (Text) e;
1724            if (t.isOnlyWhitespace()) {
1725              it.remove();
1726              }
1727            }
1728          }
1729        }
1730    
1731      
1732      for (int n = 0; n < tree.size(); n++) {
1733        ((Element)tree.get(n)).render();
1734        }
1735        
1736      if (! includeMode) {
1737        writeRenderBottom();
1738        }
1739          
1740      }
1741      
1742    void writeRenderTop() throws IOException
1743      {
1744      ol("public void render(HttpServletRequest req, HttpServletResponse res) throws Exception");
1745      ol("\t{");
1746        ol("  /* for people used to typing 'request/response' */");
1747      ol("  final HttpServletRequest  request = req;");
1748      ol("  final HttpServletResponse response = res;");
1749      ol();
1750      //mime+charset
1751      String content_type = "";
1752      if (directives.containsKey(d_mimetype)) 
1753        {
1754        String mtype = (String) directives.get(d_mimetype);
1755        if (!  (mtype.equals("") || mtype.equals(mimetype_none)) ) 
1756          {
1757          mtype = removeLeadingTrailingQuote(mtype);
1758          content_type += mtype;
1759          }
1760        } 
1761      else{
1762        content_type += Page.DEFAULT_MIME_TYPE;
1763        }
1764    
1765        
1766      if (directives.containsKey(d_encoding)) {
1767        String encoding = (String) directives.get(d_encoding);
1768        encoding = removeLeadingTrailingQuote(encoding);
1769        /*an empty encoding means that the encoding is specified in the
1770        html header*/
1771        if (! encoding.trim().equals("")) { 
1772          content_type += "; charset=";
1773          content_type += encoding; 
1774          }
1775        }
1776      else{
1777        content_type += "; charset=";
1778        content_type += Page.DEFAULT_ENCODING;
1779        }
1780    
1781      o ("  res.setContentType(\""); o (content_type); ol("\");");
1782    
1783      //buffer
1784      if (directives.containsKey(d_buffersize)) {
1785        o ("  res.setBufferSize(");
1786        o (directives.get(d_buffersize));
1787        ol(");");
1788        }
1789        
1790      //stream or writer
1791      boolean stream = false;
1792      if (directives.containsKey(d_out)) 
1793        {
1794        String stream_type = ((String) directives.get(d_out))
1795                        .toLowerCase().intern();
1796    
1797        if (stream_type == d_out_stream1 || stream_type == d_out_stream2) {
1798          stream = true;
1799          }
1800        else if (stream_type == d_out_writer) {
1801          stream = false;
1802          }
1803        else{
1804          error("Did not understand directive [directive name=out, value=" + stream_type + "]. Choose between (" +  d_out_stream1 + ") and (" + d_out_writer + ")");
1805          }
1806        }
1807        
1808      if (stream)
1809        ol("  ServletOutputStream out = res.getOutputStream();");
1810      else
1811        ol("  PrintWriter out = res.getWriter();");
1812    
1813      }
1814    
1815    void writeRenderBottom() throws IOException
1816      {
1817      ol();
1818      ol("\t} //~render end");
1819      }
1820    
1821    
1822    /*
1823    int tabcount = 1;
1824    String tab = "\t";
1825    void tabInc() {
1826      tab = StringUtil.repeat('\t', ++tabcount);
1827      }
1828    void tabDec() {
1829      tab = StringUtil.repeat('\t', --tabcount);
1830      }
1831    */
1832    
1833    abstract class Element {
1834      abstract void render() throws IOException;
1835      //text, include etc., implement this as needed. 
1836      void addExp(Exp e) {  
1837        throw new RuntimeException("Internal error: not implemented by this object"); 
1838        }
1839      }
1840        
1841    //this should NOT be added to the tree directly but added to Text or Hash
1842    //via the addExp() method. This is because exps must be printed inline
1843    class Exp extends Element
1844      {
1845      String str;
1846      
1847      Exp(CharArrayWriter buf) {
1848        this.str = buf.toString();
1849        if (dbg) dbgread("<new EXP> "+ str); 
1850        }
1851    
1852      void render() {
1853        o("out.print  (");
1854        o(str);
1855        ol(");");
1856        }
1857        
1858      public String toString() {
1859        return "Exp: [" + str + "]";
1860        }
1861      }
1862      
1863    class Text extends Element
1864      {
1865      String  offset_space;
1866      final   List list = new ArrayList();
1867    
1868      //each text section is parsed by a text node. Within EACH text
1869      //node, we split it's contained text into separate lines and
1870      //generate code to print each line with a "out.println(...)"
1871      //statement. This maintains the same source order as the molly
1872      //page. If we munge together everything and print all of it's
1873      //contents with just one out.println(...)" statement, we would
1874      //get one large line with embedded \n and that would make
1875      //things more difficult to co-relate with the source file.
1876    
1877      Text(final String offset, final CharArrayWriter b) 
1878        {
1879        if (offset == null)
1880          offset_space = "\t";
1881        else
1882          offset_space = "\t" + offset;
1883      
1884        final char[] buf = b.toCharArray();
1885    
1886        boolean prevWasCR = false;
1887        //jdk default is 32. we say 256. not too large, maybe
1888        //less cache pressure. not too important, gets resized
1889        //as needed anyway.
1890        final CharArrayWriter tmp = new CharArrayWriter(256);
1891        
1892        for (int i=0, j=1; i < buf.length; i++, j++) 
1893          {
1894          char c = buf[i];   
1895          if (j == buf.length) {
1896            tmp.append(quote(c));
1897            list.add(tmp.toString());
1898            tmp.reset();
1899            }
1900          else if (c == '\n') {
1901            tmp.append(quote(c));
1902            if (! prevWasCR) {
1903              list.add(tmp.toString());
1904              tmp.reset();
1905              }
1906            }
1907          else if (c == '\r') {
1908            tmp.append(quote(c));
1909            list.add(tmp.toString());
1910            tmp.reset();
1911            prevWasCR = true;
1912            }
1913          else{
1914            tmp.append(quote(c));
1915            prevWasCR = false;
1916            }
1917          }
1918    
1919        if (dbg) {
1920          String classname = getClass().getName();
1921          dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1922          }
1923        }
1924    
1925      Text(CharArrayWriter b) 
1926        {
1927        this(null, b);
1928        }
1929        
1930      void addExp(Exp e)
1931        {
1932        list.add(e);
1933        }
1934    
1935      void render() 
1936        {
1937        for (int i=0; i<list.size(); i++) 
1938          {
1939          Object obj = list.get(i); //can be String or Exp
1940          if (obj instanceof Exp) {
1941            o(offset_space);
1942            ((Exp)obj).render();
1943            }
1944          else{
1945            o(offset_space);
1946            o("out.print  (\"");
1947            o(obj);
1948            ol("\");"); 
1949            }
1950          }
1951        } //render
1952    
1953      //a newline is actuall '\' and '\n' since it's fed to out (..)
1954      //to check for whitepace we need to check for '\', 'n', etc
1955      boolean isOnlyWhitespace() 
1956        {
1957        for (int i = 0; i < list.size(); i++) 
1958          {
1959          Object obj = list.get(i); //can be String or Exp
1960          if (obj instanceof Exp) {
1961            return false;
1962            }
1963          else{
1964            String s = (String) obj;
1965            //dont even ask my why \\\\, fucking ridiculous
1966            if (! s.matches("^(\\\\n|\\\\r|\\\\t| )*$")) {
1967              return false;
1968              }
1969            }
1970          }
1971          
1972        return true;
1973        }
1974        
1975      public String toString() {
1976        StringBuilder buf = new StringBuilder();
1977        buf.append("Text: [");
1978        for (int n = 0; n < list.size(); n++) {
1979          buf.append(StringUtil.viewableAscii(String.valueOf(list.get(n))));
1980          }
1981        buf.append("]");
1982        return buf.toString();
1983        }
1984      
1985      }
1986    
1987    class Hash extends Text
1988      {
1989      Hash(final String offset, final CharArrayWriter b) 
1990        {
1991        super(offset, b);
1992        }
1993    
1994      //same as super.render() except for j == list.size() o/ol() below
1995      void render() 
1996        {
1997        for (int i=0, j=1; i<list.size(); i++, j++) 
1998          {
1999          Object obj = list.get(i); //can be String or Exp
2000          if (obj instanceof Exp) {
2001            o(offset_space);
2002            ((Exp)obj).render();
2003            }
2004          else{
2005            o(offset_space);
2006            o("out.print  (\"");
2007            o(obj);
2008            
2009            if (j == list.size()) 
2010              o ("\");");
2011            else
2012              ol("\");"); 
2013            }
2014          }
2015        } //render
2016    
2017      public String toString() {
2018        return "Hash: " + list;
2019        }
2020      }
2021    
2022    class Heredoc extends Text
2023      {
2024      Heredoc(final CharArrayWriter buf) 
2025        {
2026        super(null, buf);
2027        }
2028    
2029      //override, exp cannot be added to heredoc sections
2030      void addExp(Exp e)
2031        {
2032        throw new IllegalStateException("Internal implementation error: this method should not be called for a Heredoc object");
2033        }
2034        
2035      void render() 
2036        {
2037        for (int i=0, j=1; i<list.size(); i++, j++) 
2038          {
2039          Object obj = list.get(i); 
2040          o(offset_space);
2041          o("out.print  (\"");
2042          o(obj);
2043          ol("\");"); 
2044          }
2045        } //render
2046    
2047      public String toString() {
2048        return "Heredoc: " + list;
2049        }
2050    
2051      }
2052    
2053    
2054    class Code extends Element
2055      {
2056      List list = new ArrayList();
2057      
2058      Code(CharArrayWriter b) 
2059        {
2060        //we split the code section into separate lines and 
2061        //print each line with a out.print(...). This maintains
2062        //the same source order as the molly page. If we munge together
2063        //everything, we would get one large line with embedded \n
2064        //and that would make things more difficult to co-relate.
2065        final char[] buf = b.toCharArray();
2066        CharArrayWriter tmp = new CharArrayWriter();
2067        for (int i=0, j=1; i < buf.length; i++, j++) {
2068          char c = buf[i];   
2069          if (j == buf.length) { //end of buffer
2070            tmp.append(c);
2071            list.add(tmp.toString());
2072            tmp.reset();
2073            }
2074          else if (c == '\n') {
2075            tmp.append(c);
2076            list.add(tmp.toString());
2077            tmp.reset();
2078            }
2079          else
2080            tmp.append(c);
2081          }
2082        if (dbg) {
2083          String classname = getClass().getName();
2084          dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
2085          }
2086        }
2087    
2088      void render() {
2089        for (int i = 0; i < list.size(); i++) {
2090          o('\t');
2091          o(list.get(i));
2092          }
2093        }
2094        
2095      public String toString() {
2096        return "Code: " + list;
2097        }
2098      }
2099    
2100    class Comment extends Element
2101      {
2102      String str;
2103      
2104      Comment(CharArrayWriter buf) {
2105        this.str = buf.toString();
2106        if (dbg) dbgread("<new COMMENT> "+ str); 
2107        }
2108    
2109      void render() {
2110        //we don't print commented sections
2111        }
2112    
2113      public String toString() {
2114        return "Comment: [" + str + "]";
2115        }
2116      }
2117    
2118    class Decl extends Code
2119      {
2120      Decl(CharArrayWriter buf) {
2121        super(buf);
2122        }
2123    
2124      void render() {
2125        for (int i = 0; i < list.size(); i++) {
2126          o (list.get(i));
2127          }
2128        }
2129      }
2130    
2131    /* base class for Forward and Include */
2132    class ForwardIncludeElement extends Element
2133      {
2134      List    parts = new ArrayList();
2135      boolean useBuf = false;
2136      
2137      // the following is for includes with expressions 
2138      // [include foo[=i].html]  
2139      // i could be 1,2,3.. the parser adds the xpression [=i] to this
2140      // object if it's present via the addExp method below
2141      void add(CharArrayWriter buf)
2142        {
2143        parts.add(buf.toString().trim());
2144        if (parts.size() > 1) {
2145          useBuf = true;
2146          }
2147        }
2148    
2149      void addExp(Exp e)
2150        {
2151        parts.add(e);
2152        useBuf = true;
2153        }
2154    
2155      void render() throws IOException
2156        {
2157        if (parts.size() == 0) {
2158          //log.warn("possible internal error, parts.size()==0 in Forward");
2159          return;
2160          }
2161    
2162        ol("\t{ //this code block gives 'rd' its own namespace");
2163      
2164        if (! useBuf) {
2165          o ("\tfinal RequestDispatcher rd = req.getRequestDispatcher(\"");
2166          //only 1 string
2167          o (removeLeadingTrailingQuote(parts.get(0).toString())); 
2168          ol("\");");
2169          }
2170        else{
2171          ol("\tfinal StringBuilder buf = new StringBuilder();");
2172          for (int n = 0; n < parts.size(); n++) {
2173            Object obj = parts.get(n);
2174            if ( n == 0 || (n + 1) == parts.size() ) {
2175              obj = removeLeadingTrailingQuote(obj.toString());
2176              }
2177            if (obj instanceof String) {
2178              o ("\tbuf.append(\"");
2179              o (obj);
2180              ol("\");");
2181              }
2182            else{
2183              o ("\tbuf.append(");
2184              o ( ((Exp)obj).str );
2185              ol(");");
2186              }
2187            } //for
2188          ol("\tfinal RequestDispatcher rd = req.getRequestDispatcher(buf.toString());");
2189          } //else
2190        }
2191    
2192    
2193      public String toString() {
2194        return "Forward: " + parts;
2195        }
2196      }
2197    
2198    /* a request dispatcher based include. */
2199    class Include extends ForwardIncludeElement
2200      {
2201      Include() {
2202        if (dbg) dbgread("<new INCLUDE> "); 
2203        }
2204        
2205      void render() throws IOException
2206        {
2207        super.render();
2208        ol("\trd.include(req, res);");
2209        ol("\t}   //end rd block");
2210        }
2211    
2212      /* uses parent toString */
2213      }
2214    
2215    /* a request dispatcher based forward */
2216    class Forward extends ForwardIncludeElement
2217      {
2218      Forward() {
2219        if (dbg) dbgread("<new FORWARD>"); 
2220        }
2221    
2222      void render() throws IOException
2223        {
2224        super.render();
2225        ol("\t//WARNING: any uncommitted page content before this forward will be discarded.");
2226        ol("\t//If the response has already been committed an exception will be thrown. ");
2227    
2228        ol("\trd.forward(req, res);");
2229    
2230        ol("\t//NOTE: You should 'return' right after this line. There should be no content in your ");
2231        ol("\t//page after the forward statement");
2232        ol("\t}   //end rd block");
2233        }
2234    
2235      /* uses parent toString */
2236      }
2237    
2238    
2239    /* a molly mechanism to include an external file containing code and method
2240       declarations. These are typically commom utility methods and global
2241       vars. The included file is not parsed by the molly parser... the contents
2242       are treated as if they were written directly inside a [!....!] block.
2243    */ 
2244    class IncludeDecl extends Element
2245      {
2246      String str;
2247      String opt;
2248      
2249      IncludeDecl(CharArrayWriter buf) {
2250        if (dbg) dbgread("<new INCLUDE-DECL> "); 
2251        str = removeLeadingTrailingQuote(buf.toString().trim());
2252        }
2253      
2254      void setOption(String opt) {
2255        this.opt = opt;
2256        }
2257      
2258      void render() throws IOException
2259        {
2260        File f = null;
2261        File parentDir = inputFile.getParentFile();
2262        if (parentDir == null) {
2263          parentDir = new File(".");
2264          }
2265    
2266        final int strlen = str.length();
2267        
2268        if (str.startsWith("\"") || str.startsWith("'")) 
2269          {
2270          if (strlen == 1) //just " or ' 
2271            throw new IOException("Bad include file name: " + str);
2272            
2273          str = str.substring(1, strlen);
2274          }
2275    
2276        if (str.endsWith("\"") || str.endsWith("'")) 
2277          {
2278          if (strlen == 1) //just " or ' 
2279            throw new IOException("Bad include file name: " + str);
2280            
2281          str = str.substring(0, strlen-1);
2282          }
2283    
2284        if (str.startsWith("/"))
2285          f = new File(contextRoot, str);
2286        else
2287          f = new File(parentDir, str);
2288        
2289        /* f = new File(parentDir, str); */
2290        
2291        if (! f.exists()) {
2292          throw new IOException("Include file does not exist: " + f.getCanonicalPath());
2293          }
2294    
2295        o("//>>>START INCLUDE DECLARTIONS from: ");
2296        o(f.getAbsolutePath());
2297        ol();
2298            
2299        o(IOUtil.inputStreamToString(new FileInputStream(f)));
2300      
2301        o("//>>>END INCLUDE DECLARATIONS from: ");
2302        o(f.getAbsolutePath());
2303        ol();
2304        
2305        //circularities are tricky, later
2306        //includeMap.put(pageloc, f.getCanonicalPath());
2307        }
2308    
2309      public String toString() {
2310        return "IncludeDecl: [" + str + "; options: " + opt + "]";
2311        }
2312      }
2313    
2314    class Import extends Code
2315      {
2316      Import(CharArrayWriter buf) {
2317        super(buf);
2318        }
2319    
2320      void render() {
2321        for (int i = 0; i < list.size(); i++) {
2322          o (list.get(i));
2323          }
2324        }
2325      }
2326    
2327    class MollyComment extends Element
2328      {
2329      String str;
2330      
2331      MollyComment(String str) {
2332        this.str = str;
2333        if (dbg) dbgread("<new MollyComment> "+ str); 
2334        }
2335    
2336      void render() {
2337        ol(str);
2338        }
2339        
2340      public String toString() {
2341        return "MollyComment: [" + str + "]";
2342        }
2343      }
2344      
2345    /**
2346    removes starting and trailing single/double quotes. used by the
2347    include/forward render methods only, NOT used while parsing.
2348    */
2349    private static String removeLeadingTrailingQuote(String str)
2350      {
2351      if (str == null)
2352        return str;
2353    
2354      if ( str.startsWith("\"") || str.startsWith("'") )  {
2355        str = str.substring(1, str.length());
2356        }
2357    
2358      if ( str.endsWith("\"") || str.endsWith("'") ) {
2359        str = str.substring(0, str.length()-1); 
2360        }
2361    
2362      return str;
2363      }
2364    
2365    //===============================================
2366    
2367    public static void main (String args[]) throws IOException
2368      {
2369      Args myargs = new Args(args);
2370      myargs.setUsage("java " + myargs.getMainClassName() 
2371        + "\n"
2372          + "Required params:\n"
2373        + "     -classname output_class_name\n" 
2374        + "     -in        input_page_file\n"
2375        + "\nOptional params:\n" 
2376        + "     -encoding    <page_encoding>\n"
2377        + "     -contextRoot <webapp root-directory or any other directory>\n"
2378        + "        this directory is used as the starting directory for absolute (starting\n"
2379        + "        with a \"/\") include/forward directives in a page>. If not specified\n"
2380        + "        defaults to the same directory as the page file\n"
2381        + "     -out <output_file_name>\n"
2382        + "        the output file is optional and defaults to the standard out if not specified."
2383        );
2384      //String encoding = myargs.get("encoding", Page.DEFAULT_ENCODING);
2385    
2386      File input     = new File(myargs.getRequired("in"));
2387      File contextRoot = null;
2388      
2389      if (myargs.flagExists("contextRoot"))
2390        contextRoot = new File(myargs.get("contextRoot"));
2391      else
2392        contextRoot = input;
2393    
2394      PrintWriter output;
2395      
2396      if (myargs.get("out") != null)
2397        output = new PrintWriter(new FileWriter(myargs.get("out")));
2398      else
2399        output = new PrintWriter(new OutputStreamWriter(System.out));
2400        
2401      PageParser parser = new PageParser(contextRoot, input, output, myargs.getRequired("classname"), Log.getDefault());
2402      parser.parse();
2403      }
2404    
2405    }