001    // Copyright (c) 2001 Hursh Jain (http://www.mollypages.org) 
002    // The Molly framework is freely distributable under the terms of an
003    // MIT-style license. For details, see the molly pages web site at:
004    // http://www.mollypages.org/. Use, modify, have fun !
005    
006    package fc.web.page;
007    
008    import java.io.*;
009    import java.util.*;
010    import java.util.regex.*;
011    import fc.io.*;
012    import fc.util.*;
013    
014    //Code blocks of the form 
015    //  [...] 
016    //cause problems with java arrays
017    //String[] or foo[4] etc.., craps out. So we need to use 
018    //  [[...]] 
019    //for the molly code blocks
020    
021    // 1. If you are hacking this file, start with parseText()
022    //
023    // 2. Turn the dbg flag to true to see how the parser works
024    //
025    // 3. Keep in mind that the order of switch'es in a case statement in various
026    // methods is not always arbitrary (the order matters in this sort
027    // of recursive descent parsing)
028    //
029    // 4. Read www.mollypages.org/page/grammar/index.mp for a intro
030    // to parsing
031    //
032    // 5. This parser as shipped has a set of regression tests in the
033    // fc/web/page/test directory. These consist of a bunch of *.mp
034    // files and corresponding *.java files, each of which is known 
035    // to be generated properly. If you change stuff around, run these
036    // regression tests again by invoking "java fc.web.page.PageParserTest"
037    // Note, if you change things such that the .java output of the parser
038    // is different, then the tests will fail (since the new .java files
039    // of your parser will be different to the test ones shipped in
040    // fc/web/page/test. In this case, once you know that your parser works
041    // as you like it, then you should create a new baseline for your parser
042    // by invoking "java fc.web.page.PageParserTest -generateExpected" and
043    // then you can use *that* as the new baseline for further changes in
044    // your parser (you may have to modify the *.mp files in /fc/web/page/test
045    // to use your new page syntax).
046    
047    /**
048    Parses a page and writes out the corresponding java file to the specified
049    output. The parser and scanner is combined into one class here for
050    simplicity (a seperate scanner is overkill for a simple LL(1) grammar
051    such as molly pages).
052    
053    @author hursh jain
054    */
055    public final class PageParser
056    {
057    private static final boolean dbg    = false;
058    private static final int     EOF    = -1;
059    private              int     dbgtab = 0;
060    
061    String          classname;
062    String          packagename = Page.PACKAGE_NAME;
063    PageReader      in;
064    PrintWriter     out;
065    Log             log;
066    File            inputFile;
067    File            outputFile;
068    File            contextRoot;
069    boolean         includeMode = false;
070    String          src_encoding;
071    
072    //Read data
073    //we use these since stringbuffer/builders do not have a clear/reset function
074    CharArrayWriter buf          = new CharArrayWriter(4096);
075    CharArrayWriter wsbuf        = new CharArrayWriter(32);  // ^(whitespace)* 
076    int             c            = EOF;
077    //PageData
078    List            decl         = new ArrayList();     //declarations
079    List            inc_decl     = new ArrayList();     //external included declarations
080    List            imps         = new ArrayList();     //imports
081    List            tree         = new ArrayList();     //code, exp, text etc.
082    Map             directives   = new HashMap();       //page options
083    
084    /** 
085    The  name ("mimetype") of the[@ mimetype=....] directive. The value of
086    <tt>none</tt> or an empty string will turn off writing any mimetype
087    entirely (the user can then write a mimetype via the {@link
088    javax.servlet.ServletResponse.setContentType} method manually).
089    <p>
090    Note, from {@link
091    javax.servlet.ServletResponse.setContentType ServletResponse}
092    <pre>
093    Note that the character encoding cannot be communicated via HTTP headers
094    if the servlet does not specify a content type; however, it is still used
095    to encode text written via the servlet response's writer.
096    </pre>
097    */
098    public static String d_mimetype = "mimetype";
099    
100    /*
101    this value (or an empty string) for mimetype means no mimetype
102    will be specified (not even the default mimetype)
103    */
104    public static String mimetype_none = "none";
105    
106    /** 
107    The name ("encoding") of the [page encoding=....] directive. 
108    */
109    public static String d_encoding = "encoding";
110    
111    /** 
112    The name ("src-encoding") of the [page src-encoding=....] directive. 
113    */
114    public static String d_src_encoding = "src-encoding";
115    
116    /** The name ("buffersize") of the [page buffersize=....] directive */
117    public static String d_buffersize = "buffersize";
118    
119    /** The name ("out") of the [page out=....] directive */
120    public static String d_out = "out";
121    /** A value ("outputstream") of the [page out=outputstream] directive */
122    public static String d_out_stream1 = "outputstream";
123    /** A value ("outputstream") of the [page out=stream] directive */
124    public static String d_out_stream2 = "stream";
125    /** A value ("writer") of the [page out=writer] directive */
126    public static String d_out_writer = "writer";
127    
128    /* 
129    This constructor for internal use.
130    
131    The parser can be invoked recursively to parse included files as
132    well..that's what the includeMode() does (and this construtor is invoked
133    when including). When including, we already have a output writer
134    created, we use that writer (instead of creating a new one based on
135    src_encoding as we do for in normal page parsing mode).
136    */
137    private PageParser(
138     File contextRoot, File input, PrintWriter outputWriter, String classname, Log log) 
139    throws IOException
140      {
141      this.contextRoot = contextRoot;
142      this.inputFile = input; 
143      this.in  = new PageReader(input);
144      this.out = outputWriter;
145      this.classname = classname;
146      this.log = log;
147      }
148    
149    /**
150    Creates a new page parser that will use the default log obtained by
151    {@link Log#getDefault}
152    
153    @param  contextRoot absolute path to the webapp context root directory
154    @param  input   absolute path to the input page file
155    @param  input   absolute path to the output file (to be written to).
156    @param  classname classname to give to the generated java class.
157    */
158    public PageParser(File contextRoot, File input, File output, String classname) 
159    throws IOException
160      {
161      this(contextRoot, input, output, classname, Log.getDefault());
162      }
163    
164    /**
165    Creates a new page parser.
166    
167    @param  contextRoot absolute path to the webapp context root directory
168    @param  input   absolute path to the input page file
169    @param  output    absolute path to the output file (to be written to).
170    @param  classname classname to give to the generated java class.
171    @log  log     destination for internal logging output.
172    */
173    public PageParser(
174      File contextRoot, File input, File output, String classname, Log log) 
175    throws IOException
176      {
177      this.contextRoot = contextRoot;
178      this.inputFile = input; 
179      this.in  = new PageReader(input);
180      this.outputFile = output;
181      this.classname = classname;
182      this.log = log;
183      }
184    
185    void append(final int c)
186      {
187      Argcheck.istrue(c >= 0, "Internal error: recieved c=" + c);
188      buf.append((char)c);
189      }
190    
191    void append(final char c)
192      {
193      buf.append(c);
194      }
195    
196    void append(final String str)
197      {
198      buf.append(str);
199      }
200    
201    PageParser includeMode()
202      {
203      includeMode = true;
204      return this;
205      }
206    
207    /**
208    Parses the page. If the parse is successful, the java source will be
209    generated.
210    
211    @throws IOException   a parse failure occurred. The java source file
212                may or may not be properly generated or written
213                in this case.
214    */
215    public void parse() throws IOException
216      {
217      parseText();  
218      writePage();
219      if (! includeMode)  {
220        out.close();
221        }
222      out.flush();
223      }
224    
225    //util method for use in the case '[' branch of parseText below.
226    private Text newTextNode()
227      {
228      Text text = new Text(buf);
229      tree.add(text);
230      buf.reset();
231      return text;
232      }
233      
234    void parseText() throws IOException
235      {
236      if (dbg) dbgenter(); 
237          
238      while (true)
239        { 
240        c = in.read();
241        
242        if (c == EOF) {
243          tree.add(new Text(buf));
244          buf.reset();
245          break;
246          }
247          
248        switch (c)
249          { 
250          //Escape start tags
251          case '\\':
252            /*  we don't need to do this: previously, expressions
253            were [...] but now they are [=...], previously we needed
254            to escape \[[ entirely (since if we escaped \[ the second
255            [ would start an expression
256            */
257            /*        
258            if (in.match("[["))  
259              append("[[");
260            */
261            //escape only \[... otherwise leave \ alone
262            if (in.match("["))
263              append("[");
264            else
265              append(c);
266            break;
267    
268          case '[':
269            /* suppose we have
270            \[[
271            escape handling above will capture \[
272            then the second '[' drops down here. Good so far.
273            But we must not create a new text object here by
274            default...only if we see another [[ or [= or [include or
275            whatever. 
276            */
277            /*
278            But creating a text object at the top is easier
279            then repeating this code at every if..else branch below
280            but this creates superfluous line breaks.
281            
282            hello[haha]world
283            -->prints as-->
284            hello  (text node 1)
285            [haha] (text node 2)
286            world  (text node 3)
287            --> we want
288            hello[haha]world (text node 1)
289            */
290              
291            if (in.match('[')) { 
292              newTextNode();
293              parseCode(); 
294              }
295            else if (in.match('=')) {
296              Text text = newTextNode();
297              parseExpression(text);
298              }
299            else if (in.match('!')) {
300              newTextNode();
301              parseDeclaration();
302              }
303            else if (in.match("/*")) {
304              newTextNode();
305              parseComment(); 
306              }
307            else if (in.matchIgnoreCase("page")) {
308              newTextNode();
309              parseDirective();
310              }
311            //longest match: "include-file" etc., last: "include"
312            else if (in.matchIgnoreCase("include-file")) {
313              newTextNode();
314              parseIncludeFile();
315              }
316            else if (in.matchIgnoreCase("include-decl")) {
317              newTextNode();
318              parseIncludeDecl();
319              }
320            else if (in.matchIgnoreCase("include")) {
321              newTextNode();
322              parseInclude();
323              }
324            else if (in.matchIgnoreCase("forward")) {
325              newTextNode();
326              parseForward();
327              }
328            else if (in.matchIgnoreCase("import")) {
329              newTextNode();
330              parseImport();
331              }
332            else  {
333              //System.out.println("c1=" + (char)c);
334              append(c);
335              }
336            break;  
337      
338          default:
339            //System.out.println("c2=" + (char)c);
340            append(c);
341            
342          } //switch    
343        } //while
344        
345      if (dbg) dbgexit(); 
346      }
347      
348    void parseCode() throws IOException
349      {
350      if (dbg) dbgenter(); 
351    
352      int startline = in.getLine();
353      int startcol = in.getCol();
354      
355      while (true)
356        {
357        c = in.read();  
358      
359        switch (c) /* the order of case tags is important. */
360          {
361          case EOF:
362            unclosed("code", startline, startcol);
363            if (dbg) dbgexit(); 
364            return;
365    
366          case '/':   //Top level:  // and /* comments
367            append(c);
368            c = in.read();
369            append(c);
370            if (c == '/') 
371              appendCodeSlashComment();
372            else if (c == '*') 
373              appendCodeStarComment();
374              break;        
375        
376          case '"':     //strings outside of any comment
377            append(c);
378            appendCodeString();  
379            break;
380            
381          case '\'':
382            append(c);
383            appendCodeCharLiteral();
384            break;
385            
386          case ']':
387            if (in.match(']')) {
388              tree.add(new Code(buf));
389              buf.reset();
390              if (dbg) dbgexit(); 
391              return;
392              }
393            else {
394              append(c);
395              }
396            break;
397          
398          /* 
399          a hash by itself on a line starts a hash section.
400          whitespace before the # on that line is used as an
401          printing 'out' statements for that hash.
402          
403          for (int n = 0; n < ; n++) {
404          ....# foo #
405          | }
406          |=> 4 spaces 
407          so nice if generated code looked like:
408          
409          for (int n = 0; n < ; n++) {
410              out.print(" foo ");
411              }
412          */
413          case '\n':
414          case '\r':
415            append(c);       //the \n or \r just read
416            readToFirstNonWS();  //won't read past more newlines 
417            //is '#' is first non-ws on this line ?
418            c = in.read();
419            if (c == '#') {           
420              tree.add(new Code(buf));
421              buf.reset();
422              //whitespace provides indentation offset
423              parseHash(wsbuf.toString()); 
424              }
425            else{
426              append(wsbuf.toString());  //wsbuf contains codetext
427              //let other cases also handle first non-ws or EOF
428              in.unread();    
429              }
430            break;
431          
432          /* in this case, hash does not start on a new line, like:
433             for (...) { #
434          */
435          case '#':
436            tree.add(new Code(buf));
437            buf.reset();
438            parseHash(null);
439            break;  
440          
441          default:
442            append(c);
443          } //switch    
444        } //while
445      }
446    
447    void parseHash(String offset) throws IOException
448      {
449      if (dbg) dbgenter(); 
450    
451      int startline = in.getLine();
452      int startcol = in.getCol();
453    
454      while (true)
455        {
456        c = in.read();  
457      
458        switch (c)
459          {
460          case EOF: 
461            unclosed("hash", startline, startcol);
462            if (dbg) dbgexit(); 
463            return;
464    
465          //special case: very common and would be a drag to escape
466          //this every time:
467          //  # <table bgcolor="#ffffff">....   #
468          //Now, all of:
469          //  bgcolor="#xxx"  
470          //  bgcolor='#xxx'
471          //  bgcolor="\#xxx" 
472          //will work the same and give: bgcolor="#xxx"
473          //1)
474          //However to get a:
475          //  bgcolor=#xxx    (no quoted around #xxx)
476          //we still have to say:
477          //  bgcolor=\#xxx   
478          //2)
479          //Of course, since we special case this, then:
480          //  #"bar"#
481          // that ending # is lost and we end up with
482          //  #"bar"  with no closing hash
483          //So we need to make sure that we write:
484          //  #"bar" #
485          // instead
486    
487          case '\'':
488          case '"':
489            append(c);
490            if (in.match('#')) 
491              append('#');
492            break;
493            
494          case '\\':
495            if (in.match('[')) 
496              append('[');      
497            else if (in.match('#'))
498              append('#');
499            else
500              append(c);
501            break;
502            
503          case '[':
504            if (in.match('=')) {
505              Hash hash = new Hash(offset, buf);
506              tree.add(hash);
507              buf.reset();
508              parseExpression(hash);
509              }
510            else{
511              append(c);
512              }
513            break;
514    
515          /*
516          this case is not needed but is a bit of a optimization
517          for (int n = 0; n < 1; n++) {
518            #
519            foo
520          ....#...NL
521            }
522          avoids printing the dots (spaces) and NL in this case
523          (the newline after foo is still printed)
524          */
525          case '\n':
526          case '\r':
527            append(c);
528            readToFirstNonWS(); 
529            c = in.read();
530            //'#' is first non-ws on the line
531            if (c == '#') {
532              tree.add(new Hash(offset, buf));
533              buf.reset();
534              //skipIfWhitespaceToEnd();
535              if (dbg) dbgexit(); 
536              return;
537              }
538            else {
539              append(wsbuf.toString());
540              in.unread(); //let other cases also handle first non-ws   
541              }
542            break;
543    
544          case '#':
545            tree.add(new Hash(offset, buf));  
546            //skipIfWhitespaceToEnd();
547            buf.reset();
548            if (dbg) dbgexit(); 
549            return;
550            
551          default:
552            append(c);
553          }  //switch 
554        } //while
555      }
556    
557    /**
558    [page <<<FOO]
559    ...as-is..no parse, no interpolation..
560    FOO
561    */
562    void parseHeredoc(StringBuilder directives_buf) throws IOException
563      {
564      if (dbg) dbgenter(); 
565    
566      int startline = in.getLine();
567      int startcol = in.getCol();
568          
569      int i = directives_buf.indexOf("<<<"); /* "<<<".length = 3 */
570      CharSequence subseq = directives_buf.substring(
571                i+3, 
572                /*directives_buf does not have a ending ']' */
573                directives_buf.length() 
574                );
575        
576      final String      heredoc     = subseq.toString().trim();
577      final int         heredoc_len = heredoc.length();
578      final CharArrayWriter heredoc_buf = new CharArrayWriter(2048);
579    
580      /* 
581      the ending heredoc after newline speeds things up a bit
582      which is why is traditionally used i guess, otherwise
583      we have to try a full match every first match. this 
584      implementation doesn't care where the ending heredoc
585      appears (can be anywhere)...simplifies the implementation.
586      */
587      
588      while (true)
589        { 
590        c = in.read();
591        
592        if (c == EOF) {
593          unclosed("heredoc: <<<"+heredoc, startline, startcol);
594          break;
595          }
596          
597        if (c == heredoc.charAt(0))
598          {
599          boolean matched = true;
600          if (heredoc_len > 1) {
601            matched = in.match(heredoc.substring(1));
602            }
603          if (matched) {  
604            tree.add(new Heredoc(heredoc_buf));
605            break;
606            }
607          }
608        
609        //default action
610        heredoc_buf.append((char)c);  
611        } //while
612        
613      if (dbg) dbgexit(); 
614      }
615    
616    /*
617    Text is the parent node for the expression. A new expression is parsed,
618    created and added to the text object by this method
619    */
620    void parseExpression(Element parent) throws IOException
621      {
622      if (dbg) dbgenter(); 
623    
624      int startline = in.getLine();
625      int startcol = in.getCol();
626    
627      while (true)
628        {
629        c = in.read();      
630      
631        switch (c)
632          {
633          case EOF:
634            unclosed("expression", startline, startcol);
635            if (dbg) dbgexit(); 
636            return;
637    
638          case '\\':
639            if (in.match(']')) 
640              append(']');    
641            else
642              append(c);
643            break;
644    
645          case ']':
646            if (buf.toString().trim().length() == 0)
647              error("Empty expression not allowed", startline, startcol);
648            parent.addExp(new Exp(buf));
649            buf.reset();  
650            if (dbg) dbgexit(); 
651            return;
652            
653          default:
654            append(c);
655          }
656        }
657      }
658    
659    void parseComment() throws IOException
660      {
661      if (dbg) dbgenter(); 
662    
663      int startline = in.getLine();
664      int startcol = in.getCol();
665    
666      while (true)
667        {
668        c = in.read();      
669      
670        switch (c)
671          {
672          case EOF:
673            unclosed("comment", startline, startcol);
674            if (dbg) dbgexit(); 
675            return;
676            
677          case '*':
678            if (in.match("/]"))
679              {
680              tree.add(new Comment(buf));
681              buf.reset();  
682              if (dbg) dbgexit(); 
683              return;
684              }
685            else
686              append(c);  
687            break;
688          
689          default:
690            append(c);
691          }
692        }
693      }
694    
695    void parseDeclaration() throws IOException
696      {
697      if (dbg) dbgenter(); 
698      int startline = in.getLine();
699      int startcol = in.getCol();
700    
701      while (true)
702        {
703        c = in.read();      
704      
705        switch (c)
706          {
707          case EOF:
708            unclosed("declaration", startline, startcol);
709            if (dbg) dbgexit(); 
710            return;
711          
712          case '!':
713            if (in.match(']')) {
714              decl.add(new Decl(buf));
715              buf.reset();  
716              if (dbg) dbgexit(); 
717              return;
718              }
719            else{
720              append(c);
721              }
722            break;
723    
724          //top level // and /* comments, ']' (close decl tag)
725          //is ignored within them
726          case '/':   
727            append(c);
728            c = in.read();
729            append(c);
730            if (c == '/') 
731              appendCodeSlashComment();
732            else if (c == '*') 
733              appendCodeStarComment();
734              break;        
735        
736          //close tags are ignored within them
737          case '"':     //strings outside of any comment
738            append(c);
739            appendCodeString();  
740            break;
741            
742          case '\'':
743            append(c);
744            appendCodeCharLiteral();
745            break;
746                
747          default:
748            append(c);
749          }
750        }
751    
752      }
753    
754    void parseDirective() throws IOException
755      {
756      if (dbg) dbgenter(); 
757    
758      int startline = in.getLine();
759      int startcol = in.getCol();
760    
761      StringBuilder directives_buf = new StringBuilder(1024);
762    
763      while (true)
764        {
765        c = in.read();      
766      
767        switch (c)
768          {
769          case EOF:
770            unclosed("directive", startline, startcol);
771            if (dbg) dbgexit(); 
772            return;
773            
774          case ']':
775            if (directives_buf.indexOf("<<<") >= 0)  {
776              parseHeredoc(directives_buf); 
777              }
778            else{/* other directives used at page-generation time */
779              addDirectives(directives_buf);
780              }
781              
782            if (dbg) dbgexit(); 
783            return;
784          
785          default:
786            directives_buf.append((char)c);
787          }
788        }
789    
790      }
791    
792    //[a-zA-Z_\-0-9] == ( \w | - )
793    static final Pattern directive_pat = Pattern.compile(
794      //foo = "bar baz" (embd. spaces)
795      "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*\"((?:.|\r|\n)+?)\""  
796      + "|"
797      //foo = "bar$@#$" (no spaces) OR foo = bar (quotes optional)
798      + "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*(\\S+)" 
799      );
800      
801        
802    void addDirectives(StringBuilder directives_buf) throws ParseException
803      {
804      if (dbg) {
805        dbgenter(); 
806        System.out.println("-------directives section--------");
807        System.out.println(directives_buf.toString());
808        System.out.println("-------end directives-------");
809        }
810      
811      String name, value;
812      try {
813        Matcher m = directive_pat.matcher(directives_buf);
814        while (m.find()) 
815          {
816          if (dbg) System.out.println(">>>>[0]->" + m.group() 
817            + "; [1]->" + m.group(1)  
818            + " [2]->" + m.group(2)  
819            + " [3]->" + m.group(3)  
820            + " [4]->" + m.group(4));
821            
822          name = m.group(1) != null ? m.group(1).toLowerCase() :
823                        m.group(3).toLowerCase();
824          value = m.group(2) != null ? m.group(2).toLowerCase() :
825                         m.group(4).toLowerCase();
826    
827          if (name.equals(d_buffersize)) 
828            {
829            //can throw parse exception
830            directives.put(name, 
831              IOUtil.stringToFileSize(value.replace("\"|'",""))); 
832            }
833          else if (name.equals(d_encoding)) {
834            directives.put(name, value.replace("\"|'",""));       
835            }
836          else if (name.equals(d_src_encoding)) {
837            directives.put(name, value.replace("\"|'",""));       
838            } 
839          else if (name.equals(d_mimetype)) {
840            directives.put(name, value.replace("\"|'",""));       
841            }
842          else if (name.equals(d_out)) {
843            directives.put(name, value.replace("\"|'",""));       
844            } 
845          //else if .... other directives here as needed....
846          else 
847            throw new Exception("Do not understand directive: " + m.group());
848          }
849        if (dbg) System.out.println("Added directives: " + directives);
850        }
851      catch (Exception e) {
852        throw new ParseException("File: " + inputFile.getAbsolutePath() 
853                      + ";\n" + e.toString());
854        }
855    
856      if (dbg) dbgexit(); 
857      }
858    
859    void parseIncludeFile() throws IOException
860      {
861      if (dbg) dbgenter(); 
862    
863      int startline = in.getLine();
864      int startcol = in.getCol();
865      String option = null;
866      
867      while (true)
868        {
869        c = in.read();      
870      
871        switch (c)
872          {
873          case EOF:
874            unclosed("include-file", startline, startcol);
875            if (dbg) dbgexit(); 
876            return;
877            
878          case '[':
879            if (in.match('=')) {
880      //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\"
881      //in [include-file... section starting at:", startline, startcol);
882      //instead of warn, we will error out. failing early is better.
883      //this does preclude having '[=' in the file name, but it's a good
884      //tradeoff
885              error("Expressions cannot exist in file includes. The offending static-include section starts at:", startline, startcol);
886              }
887            append(c);
888            break;
889          
890          case ']':
891            IncludeFile i = new IncludeFile(buf);
892            if (option != null)
893              i.setOption(option);
894            tree.add(i);
895            buf.reset();  
896            if (dbg) dbgexit(); 
897            return;
898          
899          case 'o':
900            if (! in.match("ption"))
901              append(c);
902            else{
903              skipWS();
904              if (! in.match("=")) {
905                error("bad option parameter in file include: ", startline, startcol);
906                }
907              skipWS();
908              
909              int c2;
910              StringBuilder optionbuf = new StringBuilder();
911              while (true) {
912                c2 = in.read();
913                if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
914                  in.unread();
915                  break;
916                  }
917                optionbuf.append((char)c2);
918                }
919              
920              option = optionbuf.toString();
921              //System.out.println(option);
922              } //else
923            break;
924      
925          default:
926            append(c);
927          }
928        }
929      }
930    
931    void parseIncludeDecl() throws IOException
932      {
933      if (dbg) dbgenter(); 
934    
935      int startline = in.getLine();
936      int startcol = in.getCol();
937      String option = null;
938      
939      while (true)
940        {
941        c = in.read();      
942      
943        switch (c)
944          {
945          case EOF:
946            unclosed("include-decl", startline, startcol);
947            if (dbg) dbgexit(); 
948            return;
949            
950          case '[':
951            if (in.match('=')) {
952        //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\" in [include-static... section starting at:", startline, startcol);
953        //we will throw an exception. failing early is better. this
954        //does preclude having '[=' in the file name, but it's a good tradeoff
955              error("Expressions cannot exist in include-decl. The offending static-include section starts at:", startline, startcol);
956              }
957            append(c);
958            break;
959          
960          case ']':
961            IncludeDecl i = new IncludeDecl(buf);
962            if (option != null)
963              i.setOption(option);
964            inc_decl.add(i);
965            buf.reset();  
966            if (dbg) dbgexit(); 
967            return;
968          
969          case 'o':
970            if (! in.match("ption"))
971              append(c);
972            else{
973              skipWS();
974              if (! in.match("=")) {
975                error("bad option parameter in include-code: ", startline, startcol);
976                }
977              skipWS();
978              
979              int c2;
980              StringBuilder optionbuf = new StringBuilder();
981              while (true) {
982                c2 = in.read();
983                if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
984                  in.unread();
985                  break;
986                  }
987                optionbuf.append((char)c2);
988                }
989              
990              option = optionbuf.toString();
991              //System.out.println(option);
992              } //else
993            break;
994      
995          default:
996            append(c);
997          }
998        }
999      }
1000    
1001    //the filename/url can be optionally double quoted. leading/trailing
1002    //double quotes (if any) are ignored when an include is rendered...
1003    //this way there isn't any additional parsing needed here...I could
1004    //ignore the optional quote here (and that's the formal proper way) 
1005    //and then not move the ignore quote logic into the render() method but
1006    //this way is good too...and simpler..
1007    //same goes for the other parseIncludeXX/ForwardXX functions.
1008    void parseInclude() throws IOException
1009      {
1010      if (dbg) dbgenter(); 
1011    
1012      int startline = in.getLine();
1013      int startcol = in.getCol();
1014      Include include = new Include();
1015      while (true)
1016        {
1017        c = in.read();      
1018      
1019        switch (c)
1020          {
1021          case EOF:
1022            unclosed("include", startline, startcol);
1023            if (dbg) dbgexit(); 
1024            return;
1025            
1026          case '[':
1027            if (in.match('=')) {
1028              include.add(buf);
1029              buf.reset();
1030              parseExpression(include);
1031              }
1032            else{
1033              append(c);
1034              }
1035            break;
1036          
1037          case ']':
1038            include.add(buf);
1039            tree.add(include);
1040            buf.reset();  
1041            if (dbg) dbgexit(); 
1042            return;
1043          
1044          default:
1045            append(c);
1046          }
1047        }
1048      }
1049    
1050    void parseForward() throws IOException
1051      {
1052      if (dbg) dbgenter(); 
1053    
1054      int startline = in.getLine();
1055      int startcol = in.getCol();
1056    
1057      Forward forward = new Forward();
1058      while (true)
1059        {
1060        c = in.read();      
1061      
1062        switch (c)
1063          {
1064          case EOF:
1065            unclosed("forward", startline, startcol);
1066            if (dbg) dbgexit(); 
1067            return;
1068            
1069          case '[':
1070            if (in.match('=')) {
1071              forward.add(buf);
1072              buf.reset();
1073              parseExpression(forward);
1074              }
1075            else{
1076              append(c);
1077              }
1078            break;
1079          
1080          case ']':
1081            forward.add(buf);
1082            tree.add(forward);
1083            buf.reset();  
1084            if (dbg) dbgexit(); 
1085            return;
1086          
1087          default:
1088            append(c);
1089          }
1090        }
1091      }
1092    
1093    //we need to parse imports seperately because they go outside
1094    //a class declaration (and [!...!] goes inside a class)
1095    //import XXX.*;
1096    //class YYY {
1097    //[!....stuff from here ....!]
1098    //...
1099    void parseImport() throws IOException
1100      {
1101      if (dbg) dbgenter(); 
1102    
1103      int startline = in.getLine();
1104      int startcol = in.getCol();
1105    
1106      while (true)
1107        {
1108        c = in.read();      
1109      
1110        switch (c)
1111          {
1112          case EOF:
1113            unclosed("import", startline, startcol);
1114            if (dbg) dbgexit(); 
1115            return;
1116          
1117          case '\n':
1118            imps.add(new Import(buf));
1119            buf.reset();
1120            break;
1121            
1122          case ']':
1123            imps.add(new Import(buf));
1124            buf.reset();  
1125            if (dbg) dbgexit(); 
1126            return;
1127          
1128          default:
1129            append(c);
1130          }
1131        }
1132      }
1133    
1134    /*
1135    Called when // was read at the top level inside a code block. Appends
1136    the contents of a // comment to the buffer (not including the trailing
1137    newline)
1138    */
1139    void appendCodeSlashComment() throws IOException
1140      {
1141      if (dbg) dbgenter();
1142      
1143      while (true) 
1144        {
1145        c = in.read();
1146        
1147        if (c == EOF)
1148          break;
1149      
1150        //do not append \r, \r\n, or \n, that finishes the // comment
1151        //we need that newline to figure out if the next line is a hash
1152        //line
1153        if (c == '\r') {
1154          in.unread();
1155          break;
1156          }
1157        
1158        if (c == '\n') {
1159          in.unread();
1160          break;  
1161          }
1162    
1163        append(c);
1164        }
1165      
1166      if (dbg) dbgread("CodeSLASHComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1167      if (dbg) dbgexit();
1168      }
1169    
1170    /*
1171    Called when /* was read at the top level inside a code block. Appends
1172    the contents of a /*comment to the buffer. (not including any trailing
1173    newline or spaces)
1174    */
1175    void appendCodeStarComment() throws IOException
1176      {
1177      if (dbg) dbgenter(); 
1178      
1179      while (true) 
1180        {
1181        c = in.read();  
1182    
1183        if (c == EOF)
1184          break;
1185      
1186        append(c);
1187        
1188        if (c == '*') 
1189          {
1190          if (in.match('/')) {
1191            append('/');
1192            break;
1193            }
1194          }
1195        }
1196    
1197      if (dbg) dbgread("CodeSTARComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1198      if (dbg) dbgexit(); 
1199      }
1200    
1201    /*
1202    Called (outside of any comments in the code block) when: 
1203    --> parseCode()
1204         ... "
1205             ^ (we are here)
1206    */
1207    void appendCodeString() throws IOException
1208      {
1209      if (dbg) dbgenter(); 
1210    
1211      int startline = in.getLine();
1212      int startcol = in.getCol();
1213    
1214      while (true) 
1215        {
1216        c = in.read();
1217      
1218        if (c == EOF || c == '\r' || c == '\n')
1219          unclosed("string literal", startline, startcol);
1220      
1221        append(c);
1222      
1223        if (c == '\\') {
1224          c = in.read();
1225          if (c == EOF)
1226            unclosed("string literal", startline, startcol);
1227          else {
1228            append(c);
1229            continue;   //so \" does not hit the if below and break
1230            }
1231          }
1232        
1233        if (c == '"')
1234          break;
1235        }
1236    
1237      if (dbg) dbgread("appendCodeString Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1238      if (dbg) dbgexit(); 
1239      }
1240    
1241    
1242    /*
1243    Called (outside of any comments in the code block) when: 
1244    --> parseCode()
1245         ... '
1246             ^ (we are here)
1247    */
1248    void appendCodeCharLiteral() throws IOException
1249      {
1250      if (dbg) dbgenter(); 
1251    
1252      int startline = in.getLine();
1253      int startcol = in.getCol();
1254    
1255      while (true) 
1256        {
1257        c = in.read();
1258      
1259        if (c == EOF || c == '\r' || c == '\n')
1260          unclosed("char literal", startline, startcol);
1261      
1262        append(c);
1263      
1264        if (c == '\\') {
1265          c = in.read();
1266          if (c == EOF)
1267            unclosed("char literal", startline, startcol);
1268          else {
1269            append(c);
1270            continue;   //so \' does not hit the if below and break
1271            }
1272          }
1273        
1274        if (c == '\'')
1275          break;
1276        }
1277    
1278      if (dbg) dbgread("appendCodeCharLiteral Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1279      if (dbg) dbgexit(); 
1280      }
1281    
1282    
1283    /*
1284    Reads from the current position till the first nonwhitespace char, EOF or
1285    newline is encountered. Reads are into the whitespace buffer. does not
1286    consume the character past the non-whitespace character and does
1287    NOT read multiple lines of whitespace.
1288    */
1289    void readToFirstNonWS() throws IOException 
1290      {
1291      wsbuf.reset();
1292    
1293      while (true)
1294        {
1295        c = in.read();
1296      
1297        if (c == '\r' || c == '\n')
1298          break;
1299          
1300        if (c == EOF || ! Character.isWhitespace(c))
1301          break;
1302      
1303        wsbuf.append((char)c);
1304        }
1305        
1306      in.unread();
1307      }
1308    
1309    //skip till end of whitespace or EOF. does not consume any chars past 
1310    //the whitespace.
1311    void skipWS() throws IOException
1312      {
1313      int c2 = EOF;
1314      while (true) {
1315        c2 = in.read();
1316        if (c2 == EOF || ! Character.isWhitespace(c2)) {
1317          in.unread();
1318          break;
1319          }
1320        } 
1321      }
1322      
1323    //skips to the end of line if the rest of the line is (from the current
1324    //position), all whitespace till the end. otherwise, does not change 
1325    //current position. consumes trailing newlines (if present) when reading 
1326    //whitespace.
1327    void skipIfWhitespaceToEnd() throws IOException
1328      {
1329      int count = 0;
1330      
1331      while (true) 
1332        {
1333        c = in.read();
1334          count++;
1335    
1336        if (c == '\r') {
1337          in.match('\n');
1338          return;
1339          }
1340          
1341        if (c == '\n' || c == EOF)
1342          return;
1343          
1344        if (! Character.isWhitespace(c))
1345          break;
1346          }
1347    
1348      in.unread(count);
1349      }
1350    
1351    //not used anymore but left here for potential future use. does not
1352    //consume the newline (if present)
1353    void skipToLineEnd() throws IOException 
1354      {
1355        while (true) 
1356          {
1357          int c = in.read();
1358          if (c == EOF) {
1359            in.unread();
1360          break;
1361            }
1362          if (c == '\n' || c == '\r') { 
1363            in.unread();
1364            break;
1365            }
1366          }
1367        }
1368    
1369    String quote(final char c) 
1370      {
1371        switch (c)
1372          {
1373          case '\r':
1374                return "\\r";
1375                
1376          case '\n':
1377                return "\\n";
1378     
1379        case '\"':
1380          //can also say: new String(new char[] {'\', '"'})
1381                return "\\\"";    //--> \"
1382     
1383        case '\\':
1384                return "\\\\";
1385        
1386          default:
1387            return String.valueOf(c);
1388          }
1389        }
1390    
1391    //======= util and debug methods ==========================
1392    String methodName(int framenum)
1393      {
1394      StackTraceElement ste[] = new Exception().getStackTrace();
1395      //get method that called us, we are ste[0]
1396      StackTraceElement st = ste[framenum];
1397      String file = st.getFileName();
1398      int line = st.getLineNumber();
1399      String method = st.getMethodName();
1400      String threadname = Thread.currentThread().getName();
1401      return method + "()";   
1402      }
1403    
1404    void dbgenter() {
1405      System.out.format("%s-->%s\n", StringUtil.repeat('\t', dbgtab++), methodName(2));
1406      }
1407      
1408    void dbgexit() {
1409      System.out.format("%s<--%s\n", StringUtil.repeat('\t', --dbgtab), methodName(2));
1410      }
1411    
1412    void dbgread(String str) {
1413      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1414      }
1415    
1416    void dbgread(String str, List list) {
1417      System.out.format("%s %s: ", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1418      for (int n = 0; n < list.size(); n++) {
1419        System.out.print( StringUtil.viewableAscii( (String)list.get(n) ) );
1420        }
1421      System.out.println("");
1422      }
1423    
1424    void dbgread(char c) {
1425      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(c));
1426      }
1427    
1428    void dbgread(CharArrayWriter buf) {
1429      System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(buf.toString()));
1430      }
1431    
1432    void unclosed(String blockname, int startline, int startcol) throws IOException
1433      {
1434      throw new IOException(blockname + " tag not closed.\nThis tag was possibly opened in: \nFile:"
1435        + inputFile + ", line:" 
1436        + startline + " column:" + startcol +
1437        ".\nCurrent line:" + in.getLine() + " column:" + in.getCol());  
1438      }
1439    
1440    void error(String msg, int line, int col) throws IOException
1441      {
1442      throw new IOException("Error in File:" + inputFile + " Line:" + line + " Col:" + col + " " + msg);  
1443      }
1444    
1445    void error(String msg) throws IOException
1446      {
1447      throw new IOException("Error in File:" + inputFile + " " + msg);  
1448      }
1449    
1450    //============== Non Parsing methods ================================
1451    void o(Object str) {
1452      out.print(str);
1453      }
1454    
1455    void ol(Object str) {
1456      out.println(str); 
1457      }
1458    
1459    void ol() {
1460      out.println();
1461      }
1462      
1463    /**
1464    Returns the src_encoding directive (if any) defined in this page.
1465    */
1466    String getSourceEncoding() {
1467      return src_encoding;
1468      }
1469      
1470    void writePage() throws IOException
1471      { 
1472      if (! includeMode)
1473        {
1474        if (directives.containsKey(d_src_encoding)) {
1475          this.src_encoding = (String) directives.get(d_src_encoding);
1476          this.src_encoding = removeLeadingTrailingQuote(this.src_encoding);
1477          }
1478      
1479        //create a appropriate PrintWriter based on either the default
1480        //jave encoding or the page specified java encoding
1481        //the java source file will be written out in this encoding
1482      
1483        FileOutputStream  fout = new FileOutputStream(outputFile);
1484        OutputStreamWriter  fw   = (src_encoding != null) ?
1485            new OutputStreamWriter(fout, src_encoding) :
1486            new OutputStreamWriter(fout);
1487            
1488        out = new PrintWriter(new BufferedWriter(fw));
1489        }
1490        
1491      if (! includeMode) 
1492        {
1493        writePackage();
1494        writeImports();
1495        
1496        o ("public class ");
1497        o (classname);
1498        ol(" extends fc.web.page.PageImpl");
1499        ol("{");
1500        }
1501    
1502      writeFields();
1503    
1504      if (! includeMode) {
1505        writeConstructor();
1506        }
1507        
1508      writeMethods();
1509      
1510      if (! includeMode) {
1511        ol("}");
1512        }
1513      }
1514    
1515    void writePackage()
1516      {
1517      o ("package ");
1518      o (packagename);
1519      ol(";");
1520      ol();
1521      }
1522      
1523    void writeImports() throws IOException
1524      {
1525      ol("import javax.servlet.*;");
1526      ol("import javax.servlet.http.*;");
1527      ol("import java.io.*;");
1528      ol("import java.util.*;");
1529      //write this in case (very rare) that a page overrides the 
1530      //Page.init()/destory methods [we need pageservlet for init(..)]
1531      ol("import fc.web.page.PageServlet;");
1532      for (int n = 0; n < imps.size(); n++) {
1533        ((Element)imps.get(n)).render();
1534        ol();
1535        }
1536      ol();
1537      }
1538    
1539    void writeFields()
1540      {
1541      }
1542    
1543    void writeConstructor()
1544      {
1545      }
1546    
1547    void writeMethods() throws IOException
1548      {
1549      writeDeclaredMethods();
1550      writeIncludedMethods();
1551      writeRenderMethod();
1552      }
1553      
1554    void writeDeclaredMethods() throws IOException
1555      {
1556      for (int n = 0; n < decl.size(); n++) {
1557        ((Element)decl.get(n)).render();
1558        }
1559      
1560      if (decl.size() > 0)
1561        ol();
1562      }
1563    
1564    void writeIncludedMethods() throws IOException
1565      {
1566      for (int n = 0; n < inc_decl.size(); n++) {
1567        ((Element)inc_decl.get(n)).render();
1568        }
1569        
1570      if (inc_decl.size() > 0)
1571        ol();
1572      }
1573    
1574    void writeRenderMethod() throws IOException
1575      {
1576      if  (! includeMode) 
1577        writeRenderTop();
1578        
1579      for (int n = 0; n < tree.size(); n++) {
1580        ((Element)tree.get(n)).render();
1581        }
1582        
1583      if (! includeMode) 
1584        writeRenderBottom();
1585          
1586      }
1587      
1588    void writeRenderTop() throws IOException
1589      {
1590      ol("public void render(HttpServletRequest req, HttpServletResponse res) throws Exception");
1591      ol("\t{");
1592        ol("  /* for people used to typing 'request/response' */");
1593      ol("  final HttpServletRequest  request = req;");
1594      ol("  final HttpServletResponse response = res;");
1595      ol();
1596      //mime+charset
1597      String content_type = "";
1598      if (directives.containsKey(d_mimetype)) 
1599        {
1600        String mtype = (String) directives.get(d_mimetype);
1601        if (!  (mtype.equals("") || mtype.equals(mimetype_none)) ) 
1602          {
1603          mtype = removeLeadingTrailingQuote(mtype);
1604          content_type += mtype;
1605          }
1606        } 
1607      else{
1608        content_type += Page.DEFAULT_MIME_TYPE;
1609        }
1610    
1611        
1612      if (directives.containsKey(d_encoding)) {
1613        String encoding = (String) directives.get(d_encoding);
1614        encoding = removeLeadingTrailingQuote(encoding);
1615        /*an empty encoding means that the encoding is specified in the
1616        html header*/
1617        if (! encoding.trim().equals("")) { 
1618          content_type += "; charset=";
1619          content_type += encoding; 
1620          }
1621        }
1622      else{
1623        content_type += "; charset=";
1624        content_type += Page.DEFAULT_ENCODING;
1625        }
1626    
1627      o ("  res.setContentType(\""); o (content_type); ol("\");");
1628    
1629      //buffer
1630      if (directives.containsKey(d_buffersize)) {
1631        o ("  res.setBufferSize(");
1632        o (directives.get(d_buffersize));
1633        ol(");");
1634        }
1635        
1636      //stream or writer
1637      boolean stream = false;
1638      if (directives.containsKey(d_out)) 
1639        {
1640        String stream_type = ((String) directives.get(d_out))
1641                        .toLowerCase().intern();
1642    
1643        if (stream_type == d_out_stream1 || stream_type == d_out_stream2)
1644          stream = true;
1645        else if (stream_type == d_out_writer)
1646          stream = false;
1647        else
1648          error("Did not understand directive [directive name=out, value=" + stream_type + "]. Choose between (" +  d_out_stream1 + ") and (" + d_out_writer + ")");      
1649        }
1650        
1651      if (stream)
1652        ol("  ServletOutputStream out = res.getOutputStream();");
1653      else
1654        ol("  PrintWriter out = res.getWriter();");
1655    
1656      }
1657    
1658    void writeRenderBottom() throws IOException
1659      {
1660      ol();
1661      ol("\t} //~render end");
1662      }
1663    
1664    
1665    /*
1666    int tabcount = 1;
1667    String tab = "\t";
1668    void tabInc() {
1669      tab = StringUtil.repeat('\t', ++tabcount);
1670      }
1671    void tabDec() {
1672      tab = StringUtil.repeat('\t', --tabcount);
1673      }
1674    */
1675    
1676    abstract class Element {
1677      abstract void render() throws IOException;
1678      //text, include etc., implement this as needed. 
1679      void addExp(Exp e) {  
1680        throw new RuntimeException("Internal error: not implemented by this object"); 
1681        }
1682      }
1683        
1684    //this should NOT be added to the tree directly but added to Text or Hash
1685    //via the addExp() method. This is because exps must be printed inline
1686    class Exp extends Element
1687      {
1688      String str;
1689      
1690      Exp(CharArrayWriter buf) {
1691        this.str = buf.toString();
1692        if (dbg) dbgread("<new EXP> "+ str); 
1693        }
1694    
1695      void render() {
1696        o("out.print  (");
1697        o(str);
1698        ol(");");
1699        }
1700      }
1701      
1702    class Text extends Element
1703      {
1704      String  offset_space;
1705      final   List list = new ArrayList();
1706    
1707      //each text section is parsed by a text node. Within EACH text
1708      //node, we split it's contained text into separate lines and
1709      //generate code to print each line with a "out.println(...)"
1710      //statement. This maintains the same source order as the molly
1711      //page. If we munge together everything and print all of it's
1712      //contents with just one out.println(...)" statement, we would
1713      //get one large line with embedded \n and that would make
1714      //things more difficult to co-relate with the source file.
1715    
1716      Text(final String offset, final CharArrayWriter b) 
1717        {
1718        if (offset == null)
1719          offset_space = "\t";
1720        else
1721          offset_space = "\t" + offset;
1722      
1723        final char[] buf = b.toCharArray();
1724    
1725        boolean prevWasCR = false;
1726        //jdk default is 32. we say 256. not too large, maybe
1727        //less cache pressure. not too important, gets resized
1728        //as needed anyway.
1729        final CharArrayWriter tmp = new CharArrayWriter(256);
1730        
1731        for (int i=0, j=1; i < buf.length; i++, j++) 
1732          {
1733          char c = buf[i];   
1734          if (j == buf.length) {
1735            tmp.append(quote(c));
1736            list.add(tmp.toString());
1737            tmp.reset();
1738            }
1739          else if (c == '\n') {
1740            tmp.append(quote(c));
1741            if (! prevWasCR) {
1742              list.add(tmp.toString());
1743              tmp.reset();
1744              }
1745            }
1746          else if (c == '\r') {
1747            tmp.append(quote(c));
1748            list.add(tmp.toString());
1749            tmp.reset();
1750            prevWasCR = true;
1751            }
1752          else{
1753            tmp.append(quote(c));
1754            prevWasCR = false;
1755            }
1756          }
1757    
1758        if (dbg) {
1759          String classname = getClass().getName();
1760          dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1761          }
1762        }
1763    
1764      Text(CharArrayWriter b) 
1765        {
1766        this(null, b);
1767        }
1768        
1769      void addExp(Exp e)
1770        {
1771        list.add(e);
1772        }
1773    
1774      void render() 
1775        {
1776        for (int i=0, j=1; i<list.size(); i++, j++) 
1777          {
1778          Object obj = list.get(i); //can be String or Exp
1779          if (obj instanceof Exp) {
1780            o(offset_space);
1781            ((Exp)obj).render();
1782            }
1783          else{
1784            o(offset_space);
1785            o("out.print  (\"");
1786            o(obj);
1787            ol("\");"); 
1788            }
1789          }
1790        } //render
1791      }
1792    
1793    class Hash extends Text
1794      {
1795      Hash(final String offset, final CharArrayWriter b) 
1796        {
1797        super(offset, b);
1798        }
1799    
1800      //same as super.render() except for j == list.size() o/ol() below
1801      void render() 
1802        {
1803        for (int i=0, j=1; i<list.size(); i++, j++) 
1804          {
1805          Object obj = list.get(i); //can be String or Exp
1806          if (obj instanceof Exp) {
1807            o(offset_space);
1808            ((Exp)obj).render();
1809            }
1810          else{
1811            o(offset_space);
1812            o("out.print  (\"");
1813            o(obj);
1814            
1815            if (j == list.size()) 
1816              o ("\");");
1817            else
1818              ol("\");"); 
1819            }
1820          }
1821        } //render
1822      }
1823    
1824    class Heredoc extends Text
1825      {
1826      Heredoc(final CharArrayWriter buf) 
1827        {
1828        super(null, buf);
1829        }
1830    
1831      //override, exp cannot be added to heredoc sections
1832      void addExp(Exp e)
1833        {
1834        throw new IllegalStateException("Internal implementation error: this method should not be called for a Heredoc object");
1835        }
1836        
1837      void render() 
1838        {
1839        for (int i=0, j=1; i<list.size(); i++, j++) 
1840          {
1841          Object obj = list.get(i); 
1842          o(offset_space);
1843          o("out.print  (\"");
1844          o(obj);
1845          ol("\");"); 
1846          }
1847        } //render
1848      }
1849    
1850    
1851    class Code extends Element
1852      {
1853      List list = new ArrayList();
1854      
1855      Code(CharArrayWriter b) 
1856        {
1857        //we split the code section into separate lines and 
1858        //print each line with a out.print(...). This maintains
1859        //the same source order as the molly page. If we munge together
1860        //everything, we would get one large line with embedded \n
1861        //and that would make things more difficult to co-relate.
1862        final char[] buf = b.toCharArray();
1863        CharArrayWriter tmp = new CharArrayWriter();
1864        for (int i=0, j=1; i < buf.length; i++, j++) {
1865          char c = buf[i];   
1866          if (j == buf.length) { //end of buffer
1867            tmp.append(c);
1868            list.add(tmp.toString());
1869            tmp.reset();
1870            }
1871          else if (c == '\n') {
1872            tmp.append(c);
1873            list.add(tmp.toString());
1874            tmp.reset();
1875            }
1876          else
1877            tmp.append(c);
1878          }
1879        if (dbg) {
1880          String classname = getClass().getName();
1881          dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1882          }
1883        }
1884    
1885      void render() {
1886        for (int i = 0; i < list.size(); i++) {
1887          o('\t');
1888          o(list.get(i));
1889          }
1890        }
1891      }
1892    
1893    class Comment extends Element
1894      {
1895      String str;
1896      
1897      Comment(CharArrayWriter buf) {
1898        this.str = buf.toString();
1899        if (dbg) dbgread("<new COMMENT> "+ str); 
1900        }
1901    
1902      void render() {
1903        //we don't print commented sections
1904        }
1905      }
1906    
1907    class Decl extends Code
1908      {
1909      Decl(CharArrayWriter buf) {
1910        super(buf);
1911        }
1912    
1913      void render() {
1914        for (int i = 0; i < list.size(); i++) {
1915          o (list.get(i));
1916          }
1917        }
1918      }
1919    
1920    /* base class for Forward and Include */
1921    class ForwardIncludeElement extends Element
1922      {
1923      List    parts = new ArrayList();
1924      boolean useBuf = false;
1925      
1926      // the following is for includes with expressions 
1927      // [include foo[=i].html]  
1928      // i could be 1,2,3.. the parser adds the xpression [=i] to this
1929      // object if it's present via the addExp method below
1930      void add(CharArrayWriter buf)
1931        {
1932        parts.add(buf.toString().trim());
1933        if (parts.size() > 1) {
1934          useBuf = true;
1935          }
1936        }
1937    
1938      void addExp(Exp e)
1939        {
1940        parts.add(e);
1941        useBuf = true;
1942        }
1943    
1944      void render() throws IOException
1945        {
1946        if (parts.size() == 0) {
1947          //log.warn("possible internal error, parts.size()==0 in Forward");
1948          return;
1949          }
1950    
1951        ol("\t{ //this code block gives 'rd' its own namespace");
1952      
1953        if (! useBuf) {
1954          o ("\tfinal RequestDispatcher rd = req.getRequestDispatcher(\"");
1955          //only 1 string
1956          o (removeLeadingTrailingQuote(parts.get(0).toString())); 
1957          ol("\");");
1958          }
1959        else{
1960          ol("\tfinal StringBuilder buf = new StringBuilder();");
1961          for (int n = 0; n < parts.size(); n++) {
1962            Object obj = parts.get(n);
1963            if ( n == 0 || (n + 1) == parts.size() ) {
1964              obj = removeLeadingTrailingQuote(obj.toString());
1965              }
1966            if (obj instanceof String) {
1967              o ("\tbuf.append(\"");
1968              o (obj);
1969              ol("\");");
1970              }
1971            else{
1972              o ("\tbuf.append(");
1973              o ( ((Exp)obj).str );
1974              ol(");");
1975              }
1976            } //for
1977          ol("\tfinal RequestDispatcher rd = req.getRequestDispatcher(buf.toString());");
1978          } //else
1979        }
1980      }
1981    
1982    /* a request dispatcher based include. */
1983    class Include extends ForwardIncludeElement
1984      {
1985      Include() {
1986        if (dbg) dbgread("<new INCLUDE> "); 
1987        }
1988        
1989      void render() throws IOException
1990        {
1991        super.render();
1992        ol("\trd.include(req, res);");
1993        ol("\t}   //end rd block");
1994        }
1995      }
1996    
1997    /* a request dispatcher based forward */
1998    class Forward extends ForwardIncludeElement
1999      {
2000      Forward() {
2001        if (dbg) dbgread("<new FORWARD>"); 
2002        }
2003    
2004      void render() throws IOException
2005        {
2006        super.render();
2007        ol("\t//WARNING: any uncommitted page content before this forward will be discarded.");
2008        ol("\t//If the response has already been committed an exception will be thrown. ");
2009    
2010        ol("\trd.forward(req, res);");
2011    
2012        ol("\t//NOTE: You should 'return' right after this line. There should be no content in your ");
2013        ol("\t//page after the forward statement");
2014        ol("\t}   //end rd block");
2015        }
2016      }
2017    
2018    /* 
2019    A molly mechanism to include an external file whose contents will
2020    be rendered as part of the page.
2021    */ 
2022    class IncludeFile extends Element
2023      {
2024      String str;
2025      String opt;
2026      
2027      IncludeFile(CharArrayWriter buf) {
2028        if (dbg) dbgread("<new INCLUDE-FILE> "); 
2029        str = removeLeadingTrailingQuote(buf.toString().trim());
2030        }
2031      
2032      void setOption(String opt) {
2033        this.opt = opt;
2034        }
2035      
2036      void render() throws IOException
2037        {
2038        File includeFile = null;
2039        File parentDir = inputFile.getParentFile();
2040        if (parentDir == null) {
2041          parentDir = new File(".");
2042          }
2043    
2044        if (str.startsWith("/"))
2045          includeFile = new File(contextRoot, str);
2046        else
2047          includeFile = new File(parentDir, str);
2048            
2049        //System.out.println(">>>>>>>>>> f="+f +";root="+contextRoot);
2050          
2051        if (! includeFile.exists()) {
2052          throw new IOException("Include file does not exist: " + includeFile.getCanonicalPath());
2053          }
2054    
2055        o("//>>>START INCLUDE from: ");
2056        o(includeFile.getAbsolutePath());
2057        ol();
2058            
2059        if (opt != null && opt.toLowerCase().equalsIgnoreCase("noparse")) {
2060          o(IOUtil.inputStreamToString(new FileInputStream(includeFile)));
2061          }
2062        else{     
2063          PageParser pp = new PageParser(contextRoot, includeFile, out, classname, log);
2064          pp.includeMode().parse();  /*writes to out*/
2065          }
2066    
2067        o("//>>>END INCLUDE from: ");
2068        o(includeFile.getAbsolutePath());
2069        ol();
2070        
2071        //circularities are tricky, later
2072        //includeMap.put(pageloc, includeFile.getCanonicalPath());
2073        }
2074      }
2075    
2076    /* a molly mechanism to include an external file containing code and method
2077       declarations. These are typically commom utility methods and global
2078       vars. The included file is not parsed by the molly parser... the contents
2079       are treated as if they were written directly inside a [!....!] block.
2080    */ 
2081    class IncludeDecl extends Element
2082      {
2083      String str;
2084      String opt;
2085      
2086      IncludeDecl(CharArrayWriter buf) {
2087        if (dbg) dbgread("<new INCLUDE-DECL> "); 
2088        str = removeLeadingTrailingQuote(buf.toString().trim());
2089        }
2090      
2091      void setOption(String opt) {
2092        this.opt = opt;
2093        }
2094      
2095      void render() throws IOException
2096        {
2097        File f = null;
2098        File parentDir = inputFile.getParentFile();
2099        if (parentDir == null) {
2100          parentDir = new File(".");
2101          }
2102    
2103        final int strlen = str.length();
2104        
2105        if (str.startsWith("\"") || str.startsWith("'")) 
2106          {
2107          if (strlen == 1) //just " or ' 
2108            throw new IOException("Bad include file name: " + str);
2109            
2110          str = str.substring(1, strlen);
2111          }
2112    
2113        if (str.endsWith("\"") || str.endsWith("'")) 
2114          {
2115          if (strlen == 1) //just " or ' 
2116            throw new IOException("Bad include file name: " + str);
2117            
2118          str = str.substring(0, strlen-1);
2119          }
2120    
2121        if (str.startsWith("/"))
2122          f = new File(contextRoot, str);
2123        else
2124          f = new File(parentDir, str);
2125        
2126        /* f = new File(parentDir, str); */
2127        
2128        if (! f.exists()) {
2129          throw new IOException("Include file does not exist: " + f.getCanonicalPath());
2130          }
2131    
2132        o("//>>>START INCLUDE DECLARTIONS from: ");
2133        o(f.getAbsolutePath());
2134        ol();
2135            
2136        o(IOUtil.inputStreamToString(new FileInputStream(f)));
2137      
2138        o("//>>>END INCLUDE DECLARATIONS from: ");
2139        o(f.getAbsolutePath());
2140        ol();
2141        
2142        //circularities are tricky, later
2143        //includeMap.put(pageloc, f.getCanonicalPath());
2144        }
2145      }
2146    
2147    class Import extends Code
2148      {
2149      Import(CharArrayWriter buf) {
2150        super(buf);
2151        }
2152    
2153      void render() {
2154        for (int i = 0; i < list.size(); i++) {
2155          o (list.get(i));
2156          }
2157        }
2158      }
2159    
2160    /**
2161    removes starting and trailing single/double quotes. used by the
2162    include/forward render methods only, NOT used while parsing.
2163    */
2164    private static String removeLeadingTrailingQuote(String str)
2165      {
2166      if (str == null)
2167        return str;
2168    
2169      if ( str.startsWith("\"") || str.startsWith("'") )  {
2170        str = str.substring(1, str.length());
2171        }
2172    
2173      if ( str.endsWith("\"") || str.endsWith("'") ) {
2174        str = str.substring(0, str.length()-1); 
2175        }
2176    
2177      return str;
2178      }
2179    
2180    //===============================================
2181    
2182    public static void main (String args[]) throws IOException
2183      {
2184      Args myargs = new Args(args);
2185      myargs.setUsage("java " + myargs.getMainClassName() 
2186        + "\n"
2187          + "Required params:\n"
2188        + "     -classname output_class_name\n" 
2189        + "     -in        input_page_file\n"
2190        + "\nOptional params:\n" 
2191        + "     -encoding    <page_encoding>\n"
2192        + "     -contextRoot <webapp root-directory or any other directory>\n"
2193        + "        this directory is used as the starting directory for absolute (starting\n"
2194        + "        with a \"/\") include/forward directives in a page>. If not specified\n"
2195        + "        defaults to the same directory as the page file\n"
2196        + "     -out <output_file_name>\n"
2197        + "        the output file is optional and defaults to the standard out if not specified."
2198        );
2199      //String encoding = myargs.get("encoding", Page.DEFAULT_ENCODING);
2200    
2201      File input     = new File(myargs.getRequired("in"));
2202      File contextRoot = null;
2203      
2204      if (myargs.flagExists("contextRoot"))
2205        contextRoot = new File(myargs.get("contextRoot"));
2206      else
2207        contextRoot = input;
2208    
2209      PrintWriter output;
2210      
2211      if (myargs.get("out") != null)
2212        output = new PrintWriter(new FileWriter(myargs.get("out")));
2213      else
2214        output = new PrintWriter(new OutputStreamWriter(System.out));
2215        
2216      PageParser parser = new PageParser(contextRoot, input, output, myargs.getRequired("classname"), Log.getDefault());
2217      parser.parse();
2218      }
2219    
2220    }