001// Copyright (c) 2001 Hursh Jain (http://www.mollypages.org) 
002// The Molly framework is freely distributable under the terms of an
003// MIT-style license. For details, see the molly pages web site at:
004// http://www.mollypages.org/. Use, modify, have fun !
005
006package fc.web.page;
007
008import java.io.*;
009import java.util.*;
010import java.util.regex.*;
011import fc.io.*;
012import fc.util.*;
013
014//Code blocks of the form 
015//  [...] 
016//cause problems with java arrays
017//String[] or foo[4] etc.., craps out. So we need to use 
018//  [[...]] 
019//for the molly code blocks
020
021// 1. If you are hacking this file, start with parseText()
022//
023// 2. Turn the dbg flag to true to see how the parser works
024//
025// 3. Keep in mind that the order of switch'es in a case statement in various
026// methods is not always arbitrary (the order matters in this sort
027// of recursive descent parsing)
028//
029// 4. Read www.mollypages.org/page/grammar/index.mp for a intro
030// to parsing
031//
032// 5. This parser as shipped has a set of regression tests in the
033// fc/web/page/test directory. These consist of a bunch of *.mp
034// files and corresponding *.java files, each of which is known 
035// to be generated properly. If you change stuff around, run these
036// regression tests again by invoking "java fc.web.page.PageParserTest"
037// Note, if you change things such that the .java output of the parser
038// is different, then the tests will fail (since the new .java files
039// of your parser will be different to the test ones shipped in
040// fc/web/page/test. In this case, once you know that your parser works
041// as you like it, then you should create a new baseline for your parser
042// by invoking "java fc.web.page.PageParserTest -generateExpected" and
043// then you can use *that* as the new baseline for further changes in
044// your parser (you may have to modify the *.mp files in /fc/web/page/test
045// to use your new page syntax).
046
047/**
048Parses a page and writes out the corresponding java file to the specified
049output. The parser and scanner is combined into one class here for
050simplicity (a seperate scanner is overkill for a simple LL(1) grammar
051such as molly pages).
052
053@author hursh jain
054*/
055public final class PageParser
056{
057private static final boolean dbg    = false;
058private static final int     EOF    = -1;
059private              int     dbgtab = 0;
060
061String          classname;
062String          packagename = Page.PACKAGE_NAME;
063PageReader      in;
064PrintWriter     out;
065Log             log;
066File            inputFile;
067File            outputFile;
068File            contextRoot;
069boolean         includeMode = false;
070String          src_encoding;
071
072//Read data
073//we use these since stringbuffer/builders do not have a clear/reset function
074CharArrayWriter buf          = new CharArrayWriter(4096);
075CharArrayWriter wsbuf        = new CharArrayWriter(32);  // ^(whitespace)* 
076int             c            = EOF;
077//PageData
078List            decl         = new ArrayList();     //declarations
079List            inc_decl     = new ArrayList();     //external included declarations
080List            imps         = new ArrayList();     //imports
081List            tree         = new ArrayList();     //code, exp, text etc.
082Map             directives   = new HashMap();       //page options
083
084/** 
085The  name ("mimetype") of the[@ mimetype=....] directive. The value of
086<tt>none</tt> or an empty string will turn off writing any mimetype
087entirely (the user can then write a mimetype via the {@link
088javax.servlet.ServletResponse.setContentType} method manually).
089<p>
090Note, from {@link
091javax.servlet.ServletResponse.setContentType ServletResponse}
092<pre>
093Note that the character encoding cannot be communicated via HTTP headers
094if the servlet does not specify a content type; however, it is still used
095to encode text written via the servlet response's writer.
096</pre>
097*/
098public static String d_mimetype = "mimetype";
099
100/*
101this value (or an empty string) for mimetype means no mimetype
102will be specified (not even the default mimetype)
103*/
104public static String mimetype_none = "none";
105
106/** 
107The name ("encoding") of the [page encoding=....] directive. 
108*/
109public static String d_encoding = "encoding";
110
111/** 
112The name ("src-encoding") of the [page src-encoding=....] directive. 
113*/
114public static String d_src_encoding = "src-encoding";
115
116/** The name ("buffersize") of the [page buffersize=....] directive */
117public static String d_buffersize = "buffersize";
118
119/** The name ("out") of the [page out=....] directive */
120public static String d_out = "out";
121/** A value ("outputstream") of the [page out=outputstream] directive */
122public static String d_out_stream1 = "outputstream";
123/** A value ("outputstream") of the [page out=stream] directive */
124public static String d_out_stream2 = "stream";
125/** A value ("writer") of the [page out=writer] directive */
126public static String d_out_writer = "writer";
127
128/* 
129This constructor for internal use.
130
131The parser can be invoked recursively to parse included files as
132well..that's what the includeMode() does (and this construtor is invoked
133when including). When including, we already have a output writer
134created, we use that writer (instead of creating a new one based on
135src_encoding as we do for in normal page parsing mode).
136*/
137private PageParser(
138 File contextRoot, File input, PrintWriter outputWriter, String classname, Log log) 
139throws IOException
140  {
141  this.contextRoot = contextRoot;
142  this.inputFile = input; 
143  this.in  = new PageReader(input);
144  this.out = outputWriter;
145  this.classname = classname;
146  this.log = log;
147  }
148
149/**
150Creates a new page parser that will use the default log obtained by
151{@link Log#getDefault}
152
153@param  contextRoot absolute path to the webapp context root directory
154@param  input   absolute path to the input page file
155@param  input   absolute path to the output file (to be written to).
156@param  classname classname to give to the generated java class.
157*/
158public PageParser(File contextRoot, File input, File output, String classname) 
159throws IOException
160  {
161  this(contextRoot, input, output, classname, Log.getDefault());
162  }
163
164/**
165Creates a new page parser.
166
167@param  contextRoot absolute path to the webapp context root directory
168@param  input   absolute path to the input page file
169@param  output    absolute path to the output file (to be written to).
170@param  classname classname to give to the generated java class.
171@log  log     destination for internal logging output.
172*/
173public PageParser(
174  File contextRoot, File input, File output, String classname, Log log) 
175throws IOException
176  {
177  this.contextRoot = contextRoot;
178  this.inputFile = input; 
179  this.in  = new PageReader(input);
180  this.outputFile = output;
181  this.classname = classname;
182  this.log = log;
183  }
184
185void append(final int c)
186  {
187  Argcheck.istrue(c >= 0, "Internal error: recieved c=" + c);
188  buf.append((char)c);
189  }
190
191void append(final char c)
192  {
193  buf.append(c);
194  }
195
196void append(final String str)
197  {
198  buf.append(str);
199  }
200
201PageParser includeMode()
202  {
203  includeMode = true;
204  return this;
205  }
206
207/**
208Parses the page. If the parse is successful, the java source will be
209generated.
210
211@throws IOException   a parse failure occurred. The java source file
212            may or may not be properly generated or written
213            in this case.
214*/
215public void parse() throws IOException
216  {
217  parseText();  
218  writePage();
219  if (! includeMode)  {
220    out.close();
221    }
222  out.flush();
223  }
224
225//util method for use in the case '[' branch of parseText below.
226private Text newTextNode()
227  {
228  Text text = new Text(buf);
229  tree.add(text);
230  buf.reset();
231  return text;
232  }
233  
234void parseText() throws IOException
235  {
236  if (dbg) dbgenter(); 
237      
238  while (true)
239    { 
240    c = in.read();
241    
242    if (c == EOF) {
243      tree.add(new Text(buf));
244      buf.reset();
245      break;
246      }
247      
248    switch (c)
249      { 
250      //Escape start tags
251      case '\\':
252        /*  we don't need to do this: previously, expressions
253        were [...] but now they are [=...], previously we needed
254        to escape \[[ entirely (since if we escaped \[ the second
255        [ would start an expression
256        */
257        /*        
258        if (in.match("[["))  
259          append("[[");
260        */
261        //escape only \[... otherwise leave \ alone
262        if (in.match("["))
263          append("[");
264        else
265          append(c);
266        break;
267
268      case '[':
269        /* suppose we have
270        \[[
271        escape handling above will capture \[
272        then the second '[' drops down here. Good so far.
273        But we must not create a new text object here by
274        default...only if we see another [[ or [= or [include or
275        whatever. 
276        */
277        /*
278        But creating a text object at the top is easier
279        then repeating this code at every if..else branch below
280        but this creates superfluous line breaks.
281        
282        hello[haha]world
283        -->prints as-->
284        hello  (text node 1)
285        [haha] (text node 2)
286        world  (text node 3)
287        --> we want
288        hello[haha]world (text node 1)
289        */
290          
291        if (in.match('[')) { 
292          newTextNode();
293          parseCode(); 
294          }
295        else if (in.match('=')) {
296          Text text = newTextNode();
297          parseExpression(text);
298          }
299        else if (in.match('!')) {
300          newTextNode();
301          parseDeclaration();
302          }
303        else if (in.match("/*")) {
304          newTextNode();
305          parseComment(); 
306          }
307        else if (in.matchIgnoreCase("page")) {
308          newTextNode();
309          parseDirective();
310          }
311        //longest match: "include-file" etc., last: "include"
312        else if (in.matchIgnoreCase("include-file")) {
313          newTextNode();
314          parseIncludeFile();
315          }
316        else if (in.matchIgnoreCase("include-decl")) {
317          newTextNode();
318          parseIncludeDecl();
319          }
320        else if (in.matchIgnoreCase("include")) {
321          newTextNode();
322          parseInclude();
323          }
324        else if (in.matchIgnoreCase("forward")) {
325          newTextNode();
326          parseForward();
327          }
328        else if (in.matchIgnoreCase("import")) {
329          newTextNode();
330          parseImport();
331          }
332        else  {
333          //System.out.println("c1=" + (char)c);
334          append(c);
335          }
336        break;  
337  
338      default:
339        //System.out.println("c2=" + (char)c);
340        append(c);
341        
342      } //switch    
343    } //while
344    
345  if (dbg) dbgexit(); 
346  }
347  
348void parseCode() throws IOException
349  {
350  if (dbg) dbgenter(); 
351
352  int startline = in.getLine();
353  int startcol = in.getCol();
354  
355  while (true)
356    {
357    c = in.read();  
358  
359    switch (c) /* the order of case tags is important. */
360      {
361      case EOF:
362        unclosed("code", startline, startcol);
363        if (dbg) dbgexit(); 
364        return;
365
366      case '/':   //Top level:  // and /* comments
367        append(c);
368        c = in.read();
369        append(c);
370        if (c == '/') 
371          appendCodeSlashComment();
372        else if (c == '*') 
373          appendCodeStarComment();
374          break;        
375    
376      case '"':     //strings outside of any comment
377        append(c);
378        appendCodeString();  
379        break;
380        
381      case '\'':
382        append(c);
383        appendCodeCharLiteral();
384        break;
385        
386      case ']':
387        if (in.match(']')) {
388          tree.add(new Code(buf));
389          buf.reset();
390          if (dbg) dbgexit(); 
391          return;
392          }
393        else {
394          append(c);
395          }
396        break;
397      
398      /* 
399      a hash by itself on a line starts a hash section.
400      whitespace before the # on that line is used as an
401      printing 'out' statements for that hash.
402      
403      for (int n = 0; n < ; n++) {
404      ....# foo #
405      | }
406      |=> 4 spaces 
407      so nice if generated code looked like:
408      
409      for (int n = 0; n < ; n++) {
410          out.print(" foo ");
411          }
412      */
413      case '\n':
414      case '\r':
415        append(c);       //the \n or \r just read
416        readToFirstNonWS();  //won't read past more newlines 
417        //is '#' is first non-ws on this line ?
418        c = in.read();
419        if (c == '#') {           
420          tree.add(new Code(buf));
421          buf.reset();
422          //whitespace provides indentation offset
423          parseHash(wsbuf.toString()); 
424          }
425        else{
426          append(wsbuf.toString());  //wsbuf contains codetext
427          //let other cases also handle first non-ws or EOF
428          in.unread();    
429          }
430        break;
431      
432      /* in this case, hash does not start on a new line, like:
433         for (...) { #
434      */
435      case '#':
436        tree.add(new Code(buf));
437        buf.reset();
438        parseHash(null);
439        break;  
440      
441      default:
442        append(c);
443      } //switch    
444    } //while
445  }
446
447void parseHash(String offset) throws IOException
448  {
449  if (dbg) dbgenter(); 
450
451  int startline = in.getLine();
452  int startcol = in.getCol();
453
454  while (true)
455    {
456    c = in.read();  
457  
458    switch (c)
459      {
460      case EOF: 
461        unclosed("hash", startline, startcol);
462        if (dbg) dbgexit(); 
463        return;
464
465      //special case: very common and would be a drag to escape
466      //this every time:
467      //  # <table bgcolor="#ffffff">....   #
468      //Now, all of:
469      //  bgcolor="#xxx"  
470      //  bgcolor='#xxx'
471      //  bgcolor="\#xxx" 
472      //will work the same and give: bgcolor="#xxx"
473      //1)
474      //However to get a:
475      //  bgcolor=#xxx    (no quoted around #xxx)
476      //we still have to say:
477      //  bgcolor=\#xxx   
478      //2)
479      //Of course, since we special case this, then:
480      //  #"bar"#
481      // that ending # is lost and we end up with
482      //  #"bar"  with no closing hash
483      //So we need to make sure that we write:
484      //  #"bar" #
485      // instead
486
487      case '\'':
488      case '"':
489        append(c);
490        if (in.match('#')) 
491          append('#');
492        break;
493        
494      case '\\':
495        if (in.match('[')) 
496          append('[');      
497        else if (in.match('#'))
498          append('#');
499        else
500          append(c);
501        break;
502        
503      case '[':
504        if (in.match('=')) {
505          Hash hash = new Hash(offset, buf);
506          tree.add(hash);
507          buf.reset();
508          parseExpression(hash);
509          }
510        else{
511          append(c);
512          }
513        break;
514
515      /*
516      this case is not needed but is a bit of a optimization
517      for (int n = 0; n < 1; n++) {
518        #
519        foo
520      ....#...NL
521        }
522      avoids printing the dots (spaces) and NL in this case
523      (the newline after foo is still printed)
524      */
525      case '\n':
526      case '\r':
527        append(c);
528        readToFirstNonWS(); 
529        c = in.read();
530        //'#' is first non-ws on the line
531        if (c == '#') {
532          tree.add(new Hash(offset, buf));
533          buf.reset();
534          //skipIfWhitespaceToEnd();
535          if (dbg) dbgexit(); 
536          return;
537          }
538        else {
539          append(wsbuf.toString());
540          in.unread(); //let other cases also handle first non-ws   
541          }
542        break;
543
544      case '#':
545        tree.add(new Hash(offset, buf));  
546        //skipIfWhitespaceToEnd();
547        buf.reset();
548        if (dbg) dbgexit(); 
549        return;
550        
551      default:
552        append(c);
553      }  //switch 
554    } //while
555  }
556
557/**
558[page <<<FOO]
559...as-is..no parse, no interpolation..
560FOO
561*/
562void parseHeredoc(StringBuilder directives_buf) throws IOException
563  {
564  if (dbg) dbgenter(); 
565
566  int startline = in.getLine();
567  int startcol = in.getCol();
568      
569  int i = directives_buf.indexOf("<<<"); /* "<<<".length = 3 */
570  CharSequence subseq = directives_buf.substring(
571            i+3, 
572            /*directives_buf does not have a ending ']' */
573            directives_buf.length() 
574            );
575    
576  final String      heredoc     = subseq.toString().trim();
577  final int         heredoc_len = heredoc.length();
578  final CharArrayWriter heredoc_buf = new CharArrayWriter(2048);
579
580  /* 
581  the ending heredoc after newline speeds things up a bit
582  which is why is traditionally used i guess, otherwise
583  we have to try a full match every first match. this 
584  implementation doesn't care where the ending heredoc
585  appears (can be anywhere)...simplifies the implementation.
586  */
587  
588  while (true)
589    { 
590    c = in.read();
591    
592    if (c == EOF) {
593      unclosed("heredoc: <<<"+heredoc, startline, startcol);
594      break;
595      }
596      
597    if (c == heredoc.charAt(0))
598      {
599      boolean matched = true;
600      if (heredoc_len > 1) {
601        matched = in.match(heredoc.substring(1));
602        }
603      if (matched) {  
604        tree.add(new Heredoc(heredoc_buf));
605        break;
606        }
607      }
608    
609    //default action
610    heredoc_buf.append((char)c);  
611    } //while
612    
613  if (dbg) dbgexit(); 
614  }
615
616/*
617Text is the parent node for the expression. A new expression is parsed,
618created and added to the text object by this method
619*/
620void parseExpression(Element parent) throws IOException
621  {
622  if (dbg) dbgenter(); 
623
624  int startline = in.getLine();
625  int startcol = in.getCol();
626
627  while (true)
628    {
629    c = in.read();      
630  
631    switch (c)
632      {
633      case EOF:
634        unclosed("expression", startline, startcol);
635        if (dbg) dbgexit(); 
636        return;
637
638      case '\\':
639        if (in.match(']')) 
640          append(']');    
641        else
642          append(c);
643        break;
644
645      case ']':
646        if (buf.toString().trim().length() == 0)
647          error("Empty expression not allowed", startline, startcol);
648        parent.addExp(new Exp(buf));
649        buf.reset();  
650        if (dbg) dbgexit(); 
651        return;
652        
653      default:
654        append(c);
655      }
656    }
657  }
658
659void parseComment() throws IOException
660  {
661  if (dbg) dbgenter(); 
662
663  int startline = in.getLine();
664  int startcol = in.getCol();
665
666  while (true)
667    {
668    c = in.read();      
669  
670    switch (c)
671      {
672      case EOF:
673        unclosed("comment", startline, startcol);
674        if (dbg) dbgexit(); 
675        return;
676        
677      case '*':
678        if (in.match("/]"))
679          {
680          tree.add(new Comment(buf));
681          buf.reset();  
682          if (dbg) dbgexit(); 
683          return;
684          }
685        else
686          append(c);  
687        break;
688      
689      default:
690        append(c);
691      }
692    }
693  }
694
695void parseDeclaration() throws IOException
696  {
697  if (dbg) dbgenter(); 
698  int startline = in.getLine();
699  int startcol = in.getCol();
700
701  while (true)
702    {
703    c = in.read();      
704  
705    switch (c)
706      {
707      case EOF:
708        unclosed("declaration", startline, startcol);
709        if (dbg) dbgexit(); 
710        return;
711      
712      case '!':
713        if (in.match(']')) {
714          decl.add(new Decl(buf));
715          buf.reset();  
716          if (dbg) dbgexit(); 
717          return;
718          }
719        else{
720          append(c);
721          }
722        break;
723
724      //top level // and /* comments, ']' (close decl tag)
725      //is ignored within them
726      case '/':   
727        append(c);
728        c = in.read();
729        append(c);
730        if (c == '/') 
731          appendCodeSlashComment();
732        else if (c == '*') 
733          appendCodeStarComment();
734          break;        
735    
736      //close tags are ignored within them
737      case '"':     //strings outside of any comment
738        append(c);
739        appendCodeString();  
740        break;
741        
742      case '\'':
743        append(c);
744        appendCodeCharLiteral();
745        break;
746            
747      default:
748        append(c);
749      }
750    }
751
752  }
753
754void parseDirective() throws IOException
755  {
756  if (dbg) dbgenter(); 
757
758  int startline = in.getLine();
759  int startcol = in.getCol();
760
761  StringBuilder directives_buf = new StringBuilder(1024);
762
763  while (true)
764    {
765    c = in.read();      
766  
767    switch (c)
768      {
769      case EOF:
770        unclosed("directive", startline, startcol);
771        if (dbg) dbgexit(); 
772        return;
773        
774      case ']':
775        if (directives_buf.indexOf("<<<") >= 0)  {
776          parseHeredoc(directives_buf); 
777          }
778        else{/* other directives used at page-generation time */
779          addDirectives(directives_buf);
780          }
781          
782        if (dbg) dbgexit(); 
783        return;
784      
785      default:
786        directives_buf.append((char)c);
787      }
788    }
789
790  }
791
792//[a-zA-Z_\-0-9] == ( \w | - )
793static final Pattern directive_pat = Pattern.compile(
794  //foo = "bar baz" (embd. spaces)
795  "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*\"((?:.|\r|\n)+?)\""  
796  + "|"
797  //foo = "bar$@#$" (no spaces) OR foo = bar (quotes optional)
798  + "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*(\\S+)" 
799  );
800  
801    
802void addDirectives(StringBuilder directives_buf) throws ParseException
803  {
804  if (dbg) {
805    dbgenter(); 
806    System.out.println("-------directives section--------");
807    System.out.println(directives_buf.toString());
808    System.out.println("-------end directives-------");
809    }
810  
811  String name, value;
812  try {
813    Matcher m = directive_pat.matcher(directives_buf);
814    while (m.find()) 
815      {
816      if (dbg) System.out.println(">>>>[0]->" + m.group() 
817        + "; [1]->" + m.group(1)  
818        + " [2]->" + m.group(2)  
819        + " [3]->" + m.group(3)  
820        + " [4]->" + m.group(4));
821        
822      name = m.group(1) != null ? m.group(1).toLowerCase() :
823                    m.group(3).toLowerCase();
824      value = m.group(2) != null ? m.group(2).toLowerCase() :
825                     m.group(4).toLowerCase();
826
827      if (name.equals(d_buffersize)) 
828        {
829        //can throw parse exception
830        directives.put(name, 
831          IOUtil.stringToFileSize(value.replace("\"|'",""))); 
832        }
833      else if (name.equals(d_encoding)) {
834        directives.put(name, value.replace("\"|'",""));       
835        }
836      else if (name.equals(d_src_encoding)) {
837        directives.put(name, value.replace("\"|'",""));       
838        } 
839      else if (name.equals(d_mimetype)) {
840        directives.put(name, value.replace("\"|'",""));       
841        }
842      else if (name.equals(d_out)) {
843        directives.put(name, value.replace("\"|'",""));       
844        } 
845      //else if .... other directives here as needed....
846      else 
847        throw new Exception("Do not understand directive: " + m.group());
848      }
849    if (dbg) System.out.println("Added directives: " + directives);
850    }
851  catch (Exception e) {
852    throw new ParseException("File: " + inputFile.getAbsolutePath() 
853                  + ";\n" + e.toString());
854    }
855
856  if (dbg) dbgexit(); 
857  }
858
859void parseIncludeFile() throws IOException
860  {
861  if (dbg) dbgenter(); 
862
863  int startline = in.getLine();
864  int startcol = in.getCol();
865  String option = null;
866  
867  while (true)
868    {
869    c = in.read();      
870  
871    switch (c)
872      {
873      case EOF:
874        unclosed("include-file", startline, startcol);
875        if (dbg) dbgexit(); 
876        return;
877        
878      case '[':
879        if (in.match('=')) {
880  //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\"
881  //in [include-file... section starting at:", startline, startcol);
882  //instead of warn, we will error out. failing early is better.
883  //this does preclude having '[=' in the file name, but it's a good
884  //tradeoff
885          error("Expressions cannot exist in file includes. The offending static-include section starts at:", startline, startcol);
886          }
887        append(c);
888        break;
889      
890      case ']':
891        IncludeFile i = new IncludeFile(buf);
892        if (option != null)
893          i.setOption(option);
894        tree.add(i);
895        buf.reset();  
896        if (dbg) dbgexit(); 
897        return;
898      
899      case 'o':
900        if (! in.match("ption"))
901          append(c);
902        else{
903          skipWS();
904          if (! in.match("=")) {
905            error("bad option parameter in file include: ", startline, startcol);
906            }
907          skipWS();
908          
909          int c2;
910          StringBuilder optionbuf = new StringBuilder();
911          while (true) {
912            c2 = in.read();
913            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
914              in.unread();
915              break;
916              }
917            optionbuf.append((char)c2);
918            }
919          
920          option = optionbuf.toString();
921          //System.out.println(option);
922          } //else
923        break;
924  
925      default:
926        append(c);
927      }
928    }
929  }
930
931void parseIncludeDecl() throws IOException
932  {
933  if (dbg) dbgenter(); 
934
935  int startline = in.getLine();
936  int startcol = in.getCol();
937  String option = null;
938  
939  while (true)
940    {
941    c = in.read();      
942  
943    switch (c)
944      {
945      case EOF:
946        unclosed("include-decl", startline, startcol);
947        if (dbg) dbgexit(); 
948        return;
949        
950      case '[':
951        if (in.match('=')) {
952    //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\" in [include-static... section starting at:", startline, startcol);
953    //we will throw an exception. failing early is better. this
954    //does preclude having '[=' in the file name, but it's a good tradeoff
955          error("Expressions cannot exist in include-decl. The offending static-include section starts at:", startline, startcol);
956          }
957        append(c);
958        break;
959      
960      case ']':
961        IncludeDecl i = new IncludeDecl(buf);
962        if (option != null)
963          i.setOption(option);
964        inc_decl.add(i);
965        buf.reset();  
966        if (dbg) dbgexit(); 
967        return;
968      
969      case 'o':
970        if (! in.match("ption"))
971          append(c);
972        else{
973          skipWS();
974          if (! in.match("=")) {
975            error("bad option parameter in include-code: ", startline, startcol);
976            }
977          skipWS();
978          
979          int c2;
980          StringBuilder optionbuf = new StringBuilder();
981          while (true) {
982            c2 = in.read();
983            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
984              in.unread();
985              break;
986              }
987            optionbuf.append((char)c2);
988            }
989          
990          option = optionbuf.toString();
991          //System.out.println(option);
992          } //else
993        break;
994  
995      default:
996        append(c);
997      }
998    }
999  }
1000
1001//the filename/url can be optionally double quoted. leading/trailing
1002//double quotes (if any) are ignored when an include is rendered...
1003//this way there isn't any additional parsing needed here...I could
1004//ignore the optional quote here (and that's the formal proper way) 
1005//and then not move the ignore quote logic into the render() method but
1006//this way is good too...and simpler..
1007//same goes for the other parseIncludeXX/ForwardXX functions.
1008void parseInclude() throws IOException
1009  {
1010  if (dbg) dbgenter(); 
1011
1012  int startline = in.getLine();
1013  int startcol = in.getCol();
1014  Include include = new Include();
1015  while (true)
1016    {
1017    c = in.read();      
1018  
1019    switch (c)
1020      {
1021      case EOF:
1022        unclosed("include", startline, startcol);
1023        if (dbg) dbgexit(); 
1024        return;
1025        
1026      case '[':
1027        if (in.match('=')) {
1028          include.add(buf);
1029          buf.reset();
1030          parseExpression(include);
1031          }
1032        else{
1033          append(c);
1034          }
1035        break;
1036      
1037      case ']':
1038        include.add(buf);
1039        tree.add(include);
1040        buf.reset();  
1041        if (dbg) dbgexit(); 
1042        return;
1043      
1044      default:
1045        append(c);
1046      }
1047    }
1048  }
1049
1050void parseForward() throws IOException
1051  {
1052  if (dbg) dbgenter(); 
1053
1054  int startline = in.getLine();
1055  int startcol = in.getCol();
1056
1057  Forward forward = new Forward();
1058  while (true)
1059    {
1060    c = in.read();      
1061  
1062    switch (c)
1063      {
1064      case EOF:
1065        unclosed("forward", startline, startcol);
1066        if (dbg) dbgexit(); 
1067        return;
1068        
1069      case '[':
1070        if (in.match('=')) {
1071          forward.add(buf);
1072          buf.reset();
1073          parseExpression(forward);
1074          }
1075        else{
1076          append(c);
1077          }
1078        break;
1079      
1080      case ']':
1081        forward.add(buf);
1082        tree.add(forward);
1083        buf.reset();  
1084        if (dbg) dbgexit(); 
1085        return;
1086      
1087      default:
1088        append(c);
1089      }
1090    }
1091  }
1092
1093//we need to parse imports seperately because they go outside
1094//a class declaration (and [!...!] goes inside a class)
1095//import XXX.*;
1096//class YYY {
1097//[!....stuff from here ....!]
1098//...
1099void parseImport() throws IOException
1100  {
1101  if (dbg) dbgenter(); 
1102
1103  int startline = in.getLine();
1104  int startcol = in.getCol();
1105
1106  while (true)
1107    {
1108    c = in.read();      
1109  
1110    switch (c)
1111      {
1112      case EOF:
1113        unclosed("import", startline, startcol);
1114        if (dbg) dbgexit(); 
1115        return;
1116      
1117      case '\n':
1118        imps.add(new Import(buf));
1119        buf.reset();
1120        break;
1121        
1122      case ']':
1123        imps.add(new Import(buf));
1124        buf.reset();  
1125        if (dbg) dbgexit(); 
1126        return;
1127      
1128      default:
1129        append(c);
1130      }
1131    }
1132  }
1133
1134/*
1135Called when // was read at the top level inside a code block. Appends
1136the contents of a // comment to the buffer (not including the trailing
1137newline)
1138*/
1139void appendCodeSlashComment() throws IOException
1140  {
1141  if (dbg) dbgenter();
1142  
1143  while (true) 
1144    {
1145    c = in.read();
1146    
1147    if (c == EOF)
1148      break;
1149  
1150    //do not append \r, \r\n, or \n, that finishes the // comment
1151    //we need that newline to figure out if the next line is a hash
1152    //line
1153    if (c == '\r') {
1154      in.unread();
1155      break;
1156      }
1157    
1158    if (c == '\n') {
1159      in.unread();
1160      break;  
1161      }
1162
1163    append(c);
1164    }
1165  
1166  if (dbg) dbgread("CodeSLASHComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1167  if (dbg) dbgexit();
1168  }
1169
1170/*
1171Called when /* was read at the top level inside a code block. Appends
1172the contents of a /*comment to the buffer. (not including any trailing
1173newline or spaces)
1174*/
1175void appendCodeStarComment() throws IOException
1176  {
1177  if (dbg) dbgenter(); 
1178  
1179  while (true) 
1180    {
1181    c = in.read();  
1182
1183    if (c == EOF)
1184      break;
1185  
1186    append(c);
1187    
1188    if (c == '*') 
1189      {
1190      if (in.match('/')) {
1191        append('/');
1192        break;
1193        }
1194      }
1195    }
1196
1197  if (dbg) dbgread("CodeSTARComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1198  if (dbg) dbgexit(); 
1199  }
1200
1201/*
1202Called (outside of any comments in the code block) when: 
1203--> parseCode()
1204     ... "
1205         ^ (we are here)
1206*/
1207void appendCodeString() throws IOException
1208  {
1209  if (dbg) dbgenter(); 
1210
1211  int startline = in.getLine();
1212  int startcol = in.getCol();
1213
1214  while (true) 
1215    {
1216    c = in.read();
1217  
1218    if (c == EOF || c == '\r' || c == '\n')
1219      unclosed("string literal", startline, startcol);
1220  
1221    append(c);
1222  
1223    if (c == '\\') {
1224      c = in.read();
1225      if (c == EOF)
1226        unclosed("string literal", startline, startcol);
1227      else {
1228        append(c);
1229        continue;   //so \" does not hit the if below and break
1230        }
1231      }
1232    
1233    if (c == '"')
1234      break;
1235    }
1236
1237  if (dbg) dbgread("appendCodeString Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1238  if (dbg) dbgexit(); 
1239  }
1240
1241
1242/*
1243Called (outside of any comments in the code block) when: 
1244--> parseCode()
1245     ... '
1246         ^ (we are here)
1247*/
1248void appendCodeCharLiteral() throws IOException
1249  {
1250  if (dbg) dbgenter(); 
1251
1252  int startline = in.getLine();
1253  int startcol = in.getCol();
1254
1255  while (true) 
1256    {
1257    c = in.read();
1258  
1259    if (c == EOF || c == '\r' || c == '\n')
1260      unclosed("char literal", startline, startcol);
1261  
1262    append(c);
1263  
1264    if (c == '\\') {
1265      c = in.read();
1266      if (c == EOF)
1267        unclosed("char literal", startline, startcol);
1268      else {
1269        append(c);
1270        continue;   //so \' does not hit the if below and break
1271        }
1272      }
1273    
1274    if (c == '\'')
1275      break;
1276    }
1277
1278  if (dbg) dbgread("appendCodeCharLiteral Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1279  if (dbg) dbgexit(); 
1280  }
1281
1282
1283/*
1284Reads from the current position till the first nonwhitespace char, EOF or
1285newline is encountered. Reads are into the whitespace buffer. does not
1286consume the character past the non-whitespace character and does
1287NOT read multiple lines of whitespace.
1288*/
1289void readToFirstNonWS() throws IOException 
1290  {
1291  wsbuf.reset();
1292
1293  while (true)
1294    {
1295    c = in.read();
1296  
1297    if (c == '\r' || c == '\n')
1298      break;
1299      
1300    if (c == EOF || ! Character.isWhitespace(c))
1301      break;
1302  
1303    wsbuf.append((char)c);
1304    }
1305    
1306  in.unread();
1307  }
1308
1309//skip till end of whitespace or EOF. does not consume any chars past 
1310//the whitespace.
1311void skipWS() throws IOException
1312  {
1313  int c2 = EOF;
1314  while (true) {
1315    c2 = in.read();
1316    if (c2 == EOF || ! Character.isWhitespace(c2)) {
1317      in.unread();
1318      break;
1319      }
1320    } 
1321  }
1322  
1323//skips to the end of line if the rest of the line is (from the current
1324//position), all whitespace till the end. otherwise, does not change 
1325//current position. consumes trailing newlines (if present) when reading 
1326//whitespace.
1327void skipIfWhitespaceToEnd() throws IOException
1328  {
1329  int count = 0;
1330  
1331  while (true) 
1332    {
1333    c = in.read();
1334      count++;
1335
1336    if (c == '\r') {
1337      in.match('\n');
1338      return;
1339      }
1340      
1341    if (c == '\n' || c == EOF)
1342      return;
1343      
1344    if (! Character.isWhitespace(c))
1345      break;
1346      }
1347
1348  in.unread(count);
1349  }
1350
1351//not used anymore but left here for potential future use. does not
1352//consume the newline (if present)
1353void skipToLineEnd() throws IOException 
1354  {
1355    while (true) 
1356      {
1357      int c = in.read();
1358      if (c == EOF) {
1359        in.unread();
1360      break;
1361        }
1362      if (c == '\n' || c == '\r') { 
1363        in.unread();
1364        break;
1365        }
1366      }
1367    }
1368
1369String quote(final char c) 
1370  {
1371    switch (c)
1372      {
1373      case '\r':
1374            return "\\r";
1375            
1376      case '\n':
1377            return "\\n";
1378 
1379    case '\"':
1380      //can also say: new String(new char[] {'\', '"'})
1381            return "\\\"";    //--> \"
1382 
1383    case '\\':
1384            return "\\\\";
1385    
1386      default:
1387        return String.valueOf(c);
1388      }
1389    }
1390
1391//======= util and debug methods ==========================
1392String methodName(int framenum)
1393  {
1394  StackTraceElement ste[] = new Exception().getStackTrace();
1395  //get method that called us, we are ste[0]
1396  StackTraceElement st = ste[framenum];
1397  String file = st.getFileName();
1398  int line = st.getLineNumber();
1399  String method = st.getMethodName();
1400  String threadname = Thread.currentThread().getName();
1401  return method + "()";   
1402  }
1403
1404void dbgenter() {
1405  System.out.format("%s-->%s\n", StringUtil.repeat('\t', dbgtab++), methodName(2));
1406  }
1407  
1408void dbgexit() {
1409  System.out.format("%s<--%s\n", StringUtil.repeat('\t', --dbgtab), methodName(2));
1410  }
1411
1412void dbgread(String str) {
1413  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1414  }
1415
1416void dbgread(String str, List list) {
1417  System.out.format("%s %s: ", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1418  for (int n = 0; n < list.size(); n++) {
1419    System.out.print( StringUtil.viewableAscii( (String)list.get(n) ) );
1420    }
1421  System.out.println("");
1422  }
1423
1424void dbgread(char c) {
1425  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(c));
1426  }
1427
1428void dbgread(CharArrayWriter buf) {
1429  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(buf.toString()));
1430  }
1431
1432void unclosed(String blockname, int startline, int startcol) throws IOException
1433  {
1434  throw new IOException(blockname + " tag not closed.\nThis tag was possibly opened in: \nFile:"
1435    + inputFile + ", line:" 
1436    + startline + " column:" + startcol +
1437    ".\nCurrent line:" + in.getLine() + " column:" + in.getCol());  
1438  }
1439
1440void error(String msg, int line, int col) throws IOException
1441  {
1442  throw new IOException("Error in File:" + inputFile + " Line:" + line + " Col:" + col + " " + msg);  
1443  }
1444
1445void error(String msg) throws IOException
1446  {
1447  throw new IOException("Error in File:" + inputFile + " " + msg);  
1448  }
1449
1450//============== Non Parsing methods ================================
1451void o(Object str) {
1452  out.print(str);
1453  }
1454
1455void ol(Object str) {
1456  out.println(str); 
1457  }
1458
1459void ol() {
1460  out.println();
1461  }
1462  
1463/**
1464Returns the src_encoding directive (if any) defined in this page.
1465*/
1466String getSourceEncoding() {
1467  return src_encoding;
1468  }
1469  
1470void writePage() throws IOException
1471  { 
1472  if (! includeMode)
1473    {
1474    if (directives.containsKey(d_src_encoding)) {
1475      this.src_encoding = (String) directives.get(d_src_encoding);
1476      this.src_encoding = removeLeadingTrailingQuote(this.src_encoding);
1477      }
1478  
1479    //create a appropriate PrintWriter based on either the default
1480    //jave encoding or the page specified java encoding
1481    //the java source file will be written out in this encoding
1482  
1483    FileOutputStream  fout = new FileOutputStream(outputFile);
1484    OutputStreamWriter  fw   = (src_encoding != null) ?
1485        new OutputStreamWriter(fout, src_encoding) :
1486        new OutputStreamWriter(fout);
1487        
1488    out = new PrintWriter(new BufferedWriter(fw));
1489    }
1490    
1491  if (! includeMode) 
1492    {
1493    writePackage();
1494    writeImports();
1495    
1496    o ("public class ");
1497    o (classname);
1498    ol(" extends fc.web.page.PageImpl");
1499    ol("{");
1500    }
1501
1502  writeFields();
1503
1504  if (! includeMode) {
1505    writeConstructor();
1506    }
1507    
1508  writeMethods();
1509  
1510  if (! includeMode) {
1511    ol("}");
1512    }
1513  }
1514
1515void writePackage()
1516  {
1517  o ("package ");
1518  o (packagename);
1519  ol(";");
1520  ol();
1521  }
1522  
1523void writeImports() throws IOException
1524  {
1525  ol("import javax.servlet.*;");
1526  ol("import javax.servlet.http.*;");
1527  ol("import java.io.*;");
1528  ol("import java.util.*;");
1529  //write this in case (very rare) that a page overrides the 
1530  //Page.init()/destory methods [we need pageservlet for init(..)]
1531  ol("import fc.web.page.PageServlet;");
1532  for (int n = 0; n < imps.size(); n++) {
1533    ((Element)imps.get(n)).render();
1534    ol();
1535    }
1536  ol();
1537  }
1538
1539void writeFields()
1540  {
1541  }
1542
1543void writeConstructor()
1544  {
1545  }
1546
1547void writeMethods() throws IOException
1548  {
1549  writeDeclaredMethods();
1550  writeIncludedMethods();
1551  writeRenderMethod();
1552  }
1553  
1554void writeDeclaredMethods() throws IOException
1555  {
1556  for (int n = 0; n < decl.size(); n++) {
1557    ((Element)decl.get(n)).render();
1558    }
1559  
1560  if (decl.size() > 0)
1561    ol();
1562  }
1563
1564void writeIncludedMethods() throws IOException
1565  {
1566  for (int n = 0; n < inc_decl.size(); n++) {
1567    ((Element)inc_decl.get(n)).render();
1568    }
1569    
1570  if (inc_decl.size() > 0)
1571    ol();
1572  }
1573
1574void writeRenderMethod() throws IOException
1575  {
1576  if  (! includeMode) 
1577    writeRenderTop();
1578    
1579  for (int n = 0; n < tree.size(); n++) {
1580    ((Element)tree.get(n)).render();
1581    }
1582    
1583  if (! includeMode) 
1584    writeRenderBottom();
1585      
1586  }
1587  
1588void writeRenderTop() throws IOException
1589  {
1590  ol("public void render(HttpServletRequest req, HttpServletResponse res) throws Exception");
1591  ol("\t{");
1592    ol("  /* for people used to typing 'request/response' */");
1593  ol("  final HttpServletRequest  request = req;");
1594  ol("  final HttpServletResponse response = res;");
1595  ol();
1596  //mime+charset
1597  String content_type = "";
1598  if (directives.containsKey(d_mimetype)) 
1599    {
1600    String mtype = (String) directives.get(d_mimetype);
1601    if (!  (mtype.equals("") || mtype.equals(mimetype_none)) ) 
1602      {
1603      mtype = removeLeadingTrailingQuote(mtype);
1604      content_type += mtype;
1605      }
1606    } 
1607  else{
1608    content_type += Page.DEFAULT_MIME_TYPE;
1609    }
1610
1611    
1612  if (directives.containsKey(d_encoding)) {
1613    String encoding = (String) directives.get(d_encoding);
1614    encoding = removeLeadingTrailingQuote(encoding);
1615    /*an empty encoding means that the encoding is specified in the
1616    html header*/
1617    if (! encoding.trim().equals("")) { 
1618      content_type += "; charset=";
1619      content_type += encoding; 
1620      }
1621    }
1622  else{
1623    content_type += "; charset=";
1624    content_type += Page.DEFAULT_ENCODING;
1625    }
1626
1627  o ("  res.setContentType(\""); o (content_type); ol("\");");
1628
1629  //buffer
1630  if (directives.containsKey(d_buffersize)) {
1631    o ("  res.setBufferSize(");
1632    o (directives.get(d_buffersize));
1633    ol(");");
1634    }
1635    
1636  //stream or writer
1637  boolean stream = false;
1638  if (directives.containsKey(d_out)) 
1639    {
1640    String stream_type = ((String) directives.get(d_out))
1641                    .toLowerCase().intern();
1642
1643    if (stream_type == d_out_stream1 || stream_type == d_out_stream2)
1644      stream = true;
1645    else if (stream_type == d_out_writer)
1646      stream = false;
1647    else
1648      error("Did not understand directive [directive name=out, value=" + stream_type + "]. Choose between (" +  d_out_stream1 + ") and (" + d_out_writer + ")");      
1649    }
1650    
1651  if (stream)
1652    ol("  ServletOutputStream out = res.getOutputStream();");
1653  else
1654    ol("  PrintWriter out = res.getWriter();");
1655
1656  }
1657
1658void writeRenderBottom() throws IOException
1659  {
1660  ol();
1661  ol("\t} //~render end");
1662  }
1663
1664
1665/*
1666int tabcount = 1;
1667String tab = "\t";
1668void tabInc() {
1669  tab = StringUtil.repeat('\t', ++tabcount);
1670  }
1671void tabDec() {
1672  tab = StringUtil.repeat('\t', --tabcount);
1673  }
1674*/
1675
1676abstract class Element {
1677  abstract void render() throws IOException;
1678  //text, include etc., implement this as needed. 
1679  void addExp(Exp e) {  
1680    throw new RuntimeException("Internal error: not implemented by this object"); 
1681    }
1682  }
1683    
1684//this should NOT be added to the tree directly but added to Text or Hash
1685//via the addExp() method. This is because exps must be printed inline
1686class Exp extends Element
1687  {
1688  String str;
1689  
1690  Exp(CharArrayWriter buf) {
1691    this.str = buf.toString();
1692    if (dbg) dbgread("<new EXP> "+ str); 
1693    }
1694
1695  void render() {
1696    o("out.print  (");
1697    o(str);
1698    ol(");");
1699    }
1700  }
1701  
1702class Text extends Element
1703  {
1704  String  offset_space;
1705  final   List list = new ArrayList();
1706
1707  //each text section is parsed by a text node. Within EACH text
1708  //node, we split it's contained text into separate lines and
1709  //generate code to print each line with a "out.println(...)"
1710  //statement. This maintains the same source order as the molly
1711  //page. If we munge together everything and print all of it's
1712  //contents with just one out.println(...)" statement, we would
1713  //get one large line with embedded \n and that would make
1714  //things more difficult to co-relate with the source file.
1715
1716  Text(final String offset, final CharArrayWriter b) 
1717    {
1718    if (offset == null)
1719      offset_space = "\t";
1720    else
1721      offset_space = "\t" + offset;
1722  
1723    final char[] buf = b.toCharArray();
1724
1725    boolean prevWasCR = false;
1726    //jdk default is 32. we say 256. not too large, maybe
1727    //less cache pressure. not too important, gets resized
1728    //as needed anyway.
1729    final CharArrayWriter tmp = new CharArrayWriter(256);
1730    
1731    for (int i=0, j=1; i < buf.length; i++, j++) 
1732      {
1733      char c = buf[i];   
1734      if (j == buf.length) {
1735        tmp.append(quote(c));
1736        list.add(tmp.toString());
1737        tmp.reset();
1738        }
1739      else if (c == '\n') {
1740        tmp.append(quote(c));
1741        if (! prevWasCR) {
1742          list.add(tmp.toString());
1743          tmp.reset();
1744          }
1745        }
1746      else if (c == '\r') {
1747        tmp.append(quote(c));
1748        list.add(tmp.toString());
1749        tmp.reset();
1750        prevWasCR = true;
1751        }
1752      else{
1753        tmp.append(quote(c));
1754        prevWasCR = false;
1755        }
1756      }
1757
1758    if (dbg) {
1759      String classname = getClass().getName();
1760      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1761      }
1762    }
1763
1764  Text(CharArrayWriter b) 
1765    {
1766    this(null, b);
1767    }
1768    
1769  void addExp(Exp e)
1770    {
1771    list.add(e);
1772    }
1773
1774  void render() 
1775    {
1776    for (int i=0, j=1; i<list.size(); i++, j++) 
1777      {
1778      Object obj = list.get(i); //can be String or Exp
1779      if (obj instanceof Exp) {
1780        o(offset_space);
1781        ((Exp)obj).render();
1782        }
1783      else{
1784        o(offset_space);
1785        o("out.print  (\"");
1786        o(obj);
1787        ol("\");"); 
1788        }
1789      }
1790    } //render
1791  }
1792
1793class Hash extends Text
1794  {
1795  Hash(final String offset, final CharArrayWriter b) 
1796    {
1797    super(offset, b);
1798    }
1799
1800  //same as super.render() except for j == list.size() o/ol() below
1801  void render() 
1802    {
1803    for (int i=0, j=1; i<list.size(); i++, j++) 
1804      {
1805      Object obj = list.get(i); //can be String or Exp
1806      if (obj instanceof Exp) {
1807        o(offset_space);
1808        ((Exp)obj).render();
1809        }
1810      else{
1811        o(offset_space);
1812        o("out.print  (\"");
1813        o(obj);
1814        
1815        if (j == list.size()) 
1816          o ("\");");
1817        else
1818          ol("\");"); 
1819        }
1820      }
1821    } //render
1822  }
1823
1824class Heredoc extends Text
1825  {
1826  Heredoc(final CharArrayWriter buf) 
1827    {
1828    super(null, buf);
1829    }
1830
1831  //override, exp cannot be added to heredoc sections
1832  void addExp(Exp e)
1833    {
1834    throw new IllegalStateException("Internal implementation error: this method should not be called for a Heredoc object");
1835    }
1836    
1837  void render() 
1838    {
1839    for (int i=0, j=1; i<list.size(); i++, j++) 
1840      {
1841      Object obj = list.get(i); 
1842      o(offset_space);
1843      o("out.print  (\"");
1844      o(obj);
1845      ol("\");"); 
1846      }
1847    } //render
1848  }
1849
1850
1851class Code extends Element
1852  {
1853  List list = new ArrayList();
1854  
1855  Code(CharArrayWriter b) 
1856    {
1857    //we split the code section into separate lines and 
1858    //print each line with a out.print(...). This maintains
1859    //the same source order as the molly page. If we munge together
1860    //everything, we would get one large line with embedded \n
1861    //and that would make things more difficult to co-relate.
1862    final char[] buf = b.toCharArray();
1863    CharArrayWriter tmp = new CharArrayWriter();
1864    for (int i=0, j=1; i < buf.length; i++, j++) {
1865      char c = buf[i];   
1866      if (j == buf.length) { //end of buffer
1867        tmp.append(c);
1868        list.add(tmp.toString());
1869        tmp.reset();
1870        }
1871      else if (c == '\n') {
1872        tmp.append(c);
1873        list.add(tmp.toString());
1874        tmp.reset();
1875        }
1876      else
1877        tmp.append(c);
1878      }
1879    if (dbg) {
1880      String classname = getClass().getName();
1881      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1882      }
1883    }
1884
1885  void render() {
1886    for (int i = 0; i < list.size(); i++) {
1887      o('\t');
1888      o(list.get(i));
1889      }
1890    }
1891  }
1892
1893class Comment extends Element
1894  {
1895  String str;
1896  
1897  Comment(CharArrayWriter buf) {
1898    this.str = buf.toString();
1899    if (dbg) dbgread("<new COMMENT> "+ str); 
1900    }
1901
1902  void render() {
1903    //we don't print commented sections
1904    }
1905  }
1906
1907class Decl extends Code
1908  {
1909  Decl(CharArrayWriter buf) {
1910    super(buf);
1911    }
1912
1913  void render() {
1914    for (int i = 0; i < list.size(); i++) {
1915      o (list.get(i));
1916      }
1917    }
1918  }
1919
1920/* base class for Forward and Include */
1921class ForwardIncludeElement extends Element
1922  {
1923  List    parts = new ArrayList();
1924  boolean useBuf = false;
1925  
1926  // the following is for includes with expressions 
1927  // [include foo[=i].html]  
1928  // i could be 1,2,3.. the parser adds the xpression [=i] to this
1929  // object if it's present via the addExp method below
1930  void add(CharArrayWriter buf)
1931    {
1932    parts.add(buf.toString().trim());
1933    if (parts.size() > 1) {
1934      useBuf = true;
1935      }
1936    }
1937
1938  void addExp(Exp e)
1939    {
1940    parts.add(e);
1941    useBuf = true;
1942    }
1943
1944  void render() throws IOException
1945    {
1946    if (parts.size() == 0) {
1947      //log.warn("possible internal error, parts.size()==0 in Forward");
1948      return;
1949      }
1950
1951    ol("\t{ //this code block gives 'rd' its own namespace");
1952  
1953    if (! useBuf) {
1954      o ("\tfinal RequestDispatcher rd = req.getRequestDispatcher(\"");
1955      //only 1 string
1956      o (removeLeadingTrailingQuote(parts.get(0).toString())); 
1957      ol("\");");
1958      }
1959    else{
1960      ol("\tfinal StringBuilder buf = new StringBuilder();");
1961      for (int n = 0; n < parts.size(); n++) {
1962        Object obj = parts.get(n);
1963        if ( n == 0 || (n + 1) == parts.size() ) {
1964          obj = removeLeadingTrailingQuote(obj.toString());
1965          }
1966        if (obj instanceof String) {
1967          o ("\tbuf.append(\"");
1968          o (obj);
1969          ol("\");");
1970          }
1971        else{
1972          o ("\tbuf.append(");
1973          o ( ((Exp)obj).str );
1974          ol(");");
1975          }
1976        } //for
1977      ol("\tfinal RequestDispatcher rd = req.getRequestDispatcher(buf.toString());");
1978      } //else
1979    }
1980  }
1981
1982/* a request dispatcher based include. */
1983class Include extends ForwardIncludeElement
1984  {
1985  Include() {
1986    if (dbg) dbgread("<new INCLUDE> "); 
1987    }
1988    
1989  void render() throws IOException
1990    {
1991    super.render();
1992    ol("\trd.include(req, res);");
1993    ol("\t}   //end rd block");
1994    }
1995  }
1996
1997/* a request dispatcher based forward */
1998class Forward extends ForwardIncludeElement
1999  {
2000  Forward() {
2001    if (dbg) dbgread("<new FORWARD>"); 
2002    }
2003
2004  void render() throws IOException
2005    {
2006    super.render();
2007    ol("\t//WARNING: any uncommitted page content before this forward will be discarded.");
2008    ol("\t//If the response has already been committed an exception will be thrown. ");
2009
2010    ol("\trd.forward(req, res);");
2011
2012    ol("\t//NOTE: You should 'return' right after this line. There should be no content in your ");
2013    ol("\t//page after the forward statement");
2014    ol("\t}   //end rd block");
2015    }
2016  }
2017
2018/* 
2019A molly mechanism to include an external file whose contents will
2020be rendered as part of the page.
2021*/ 
2022class IncludeFile extends Element
2023  {
2024  String str;
2025  String opt;
2026  
2027  IncludeFile(CharArrayWriter buf) {
2028    if (dbg) dbgread("<new INCLUDE-FILE> "); 
2029    str = removeLeadingTrailingQuote(buf.toString().trim());
2030    }
2031  
2032  void setOption(String opt) {
2033    this.opt = opt;
2034    }
2035  
2036  void render() throws IOException
2037    {
2038    File includeFile = null;
2039    File parentDir = inputFile.getParentFile();
2040    if (parentDir == null) {
2041      parentDir = new File(".");
2042      }
2043
2044    if (str.startsWith("/"))
2045      includeFile = new File(contextRoot, str);
2046    else
2047      includeFile = new File(parentDir, str);
2048        
2049    //System.out.println(">>>>>>>>>> f="+f +";root="+contextRoot);
2050      
2051    if (! includeFile.exists()) {
2052      throw new IOException("Include file does not exist: " + includeFile.getCanonicalPath());
2053      }
2054
2055    o("//>>>START INCLUDE from: ");
2056    o(includeFile.getAbsolutePath());
2057    ol();
2058        
2059    if (opt != null && opt.toLowerCase().equalsIgnoreCase("noparse")) {
2060      o(IOUtil.inputStreamToString(new FileInputStream(includeFile)));
2061      }
2062    else{     
2063      PageParser pp = new PageParser(contextRoot, includeFile, out, classname, log);
2064      pp.includeMode().parse();  /*writes to out*/
2065      }
2066
2067    o("//>>>END INCLUDE from: ");
2068    o(includeFile.getAbsolutePath());
2069    ol();
2070    
2071    //circularities are tricky, later
2072    //includeMap.put(pageloc, includeFile.getCanonicalPath());
2073    }
2074  }
2075
2076/* a molly mechanism to include an external file containing code and method
2077   declarations. These are typically commom utility methods and global
2078   vars. The included file is not parsed by the molly parser... the contents
2079   are treated as if they were written directly inside a [!....!] block.
2080*/ 
2081class IncludeDecl extends Element
2082  {
2083  String str;
2084  String opt;
2085  
2086  IncludeDecl(CharArrayWriter buf) {
2087    if (dbg) dbgread("<new INCLUDE-DECL> "); 
2088    str = removeLeadingTrailingQuote(buf.toString().trim());
2089    }
2090  
2091  void setOption(String opt) {
2092    this.opt = opt;
2093    }
2094  
2095  void render() throws IOException
2096    {
2097    File f = null;
2098    File parentDir = inputFile.getParentFile();
2099    if (parentDir == null) {
2100      parentDir = new File(".");
2101      }
2102
2103    final int strlen = str.length();
2104    
2105    if (str.startsWith("\"") || str.startsWith("'")) 
2106      {
2107      if (strlen == 1) //just " or ' 
2108        throw new IOException("Bad include file name: " + str);
2109        
2110      str = str.substring(1, strlen);
2111      }
2112
2113    if (str.endsWith("\"") || str.endsWith("'")) 
2114      {
2115      if (strlen == 1) //just " or ' 
2116        throw new IOException("Bad include file name: " + str);
2117        
2118      str = str.substring(0, strlen-1);
2119      }
2120
2121    if (str.startsWith("/"))
2122      f = new File(contextRoot, str);
2123    else
2124      f = new File(parentDir, str);
2125    
2126    /* f = new File(parentDir, str); */
2127    
2128    if (! f.exists()) {
2129      throw new IOException("Include file does not exist: " + f.getCanonicalPath());
2130      }
2131
2132    o("//>>>START INCLUDE DECLARTIONS from: ");
2133    o(f.getAbsolutePath());
2134    ol();
2135        
2136    o(IOUtil.inputStreamToString(new FileInputStream(f)));
2137  
2138    o("//>>>END INCLUDE DECLARATIONS from: ");
2139    o(f.getAbsolutePath());
2140    ol();
2141    
2142    //circularities are tricky, later
2143    //includeMap.put(pageloc, f.getCanonicalPath());
2144    }
2145  }
2146
2147class Import extends Code
2148  {
2149  Import(CharArrayWriter buf) {
2150    super(buf);
2151    }
2152
2153  void render() {
2154    for (int i = 0; i < list.size(); i++) {
2155      o (list.get(i));
2156      }
2157    }
2158  }
2159
2160/**
2161removes starting and trailing single/double quotes. used by the
2162include/forward render methods only, NOT used while parsing.
2163*/
2164private static String removeLeadingTrailingQuote(String str)
2165  {
2166  if (str == null)
2167    return str;
2168
2169  if ( str.startsWith("\"") || str.startsWith("'") )  {
2170    str = str.substring(1, str.length());
2171    }
2172
2173  if ( str.endsWith("\"") || str.endsWith("'") ) {
2174    str = str.substring(0, str.length()-1); 
2175    }
2176
2177  return str;
2178  }
2179
2180//===============================================
2181
2182public static void main (String args[]) throws IOException
2183  {
2184  Args myargs = new Args(args);
2185  myargs.setUsage("java " + myargs.getMainClassName() 
2186    + "\n"
2187      + "Required params:\n"
2188    + "     -classname output_class_name\n" 
2189    + "     -in        input_page_file\n"
2190    + "\nOptional params:\n" 
2191    + "     -encoding    <page_encoding>\n"
2192    + "     -contextRoot <webapp root-directory or any other directory>\n"
2193    + "        this directory is used as the starting directory for absolute (starting\n"
2194    + "        with a \"/\") include/forward directives in a page>. If not specified\n"
2195    + "        defaults to the same directory as the page file\n"
2196    + "     -out <output_file_name>\n"
2197    + "        the output file is optional and defaults to the standard out if not specified."
2198    );
2199  //String encoding = myargs.get("encoding", Page.DEFAULT_ENCODING);
2200
2201  File input     = new File(myargs.getRequired("in"));
2202  File contextRoot = null;
2203  
2204  if (myargs.flagExists("contextRoot"))
2205    contextRoot = new File(myargs.get("contextRoot"));
2206  else
2207    contextRoot = input;
2208
2209  PrintWriter output;
2210  
2211  if (myargs.get("out") != null)
2212    output = new PrintWriter(new FileWriter(myargs.get("out")));
2213  else
2214    output = new PrintWriter(new OutputStreamWriter(System.out));
2215    
2216  PageParser parser = new PageParser(contextRoot, input, output, myargs.getRequired("classname"), Log.getDefault());
2217  parser.parse();
2218  }
2219
2220}