package org.paneris.bibliomania.fti;
   
   import java.io.BufferedInputStream;
   import java.io.EOFException;
   import java.io.FileInputStream;
   import java.io.IOException;
   import java.io.InputStream;
   import java.util.Enumeration;
   import java.util.NoSuchElementException;
  
  import org.melati.util.UnexpectedExceptionException;
  
  public class IndexTokenizer implements Enumeration {
  
    private InputStream reader;
    private int pushedBack = -1;
    private int offset = -1, wordIndex = -1, wordOffset;
    private StringBuffer buffer = new StringBuffer();
    private String nextWord = null;
    private boolean hadBreak = false;
    private boolean lastWasAnchor = false;
    private boolean inStrong = false;
    private boolean inScript = false;
    private boolean wantEverything;
  
    public IndexTokenizer(InputStream reader, boolean wantEverything) {
      this.reader = reader;
      this.wantEverything = wantEverything;
    }
  
    public IndexTokenizer(InputStream reader) {
      this(reader, false);
    }
  
    private int nextChar() throws IOException {
      int c;
      if (pushedBack == -1) {
        c = reader.read();
        if (c == -1)
          throw new EOFException();
      } else {
        c = pushedBack;
        pushedBack = -1;
      }
  
      ++offset;
      return c;
    }
  
    private void pushback(int c) {
      if (pushedBack != -1)
        throw new IllegalArgumentException("pushed back already");
      pushedBack = c;
      --offset;
    }
  
    private void string(int term) throws IOException {
      int c;
      do {
        if ((c = nextChar()) == '\\') {
          nextChar();
          c = nextChar();
        }
      } while (c != term);
    }
  
    private void comment() throws IOException {
      for (;;) {
        int c;
        if ((c = nextChar()) == '-')
          if ((c = nextChar()) == '-' && (c = nextChar()) == '>')
            break;
          else
            pushback(c);
      }
    }
  
    private String tag() throws IOException {
      int tagOffset = offset;
      String anchorName = null;
      int c;
      if ((c = nextChar()) == '!'
        && (c = nextChar()) == '-'
        && (c = nextChar()) == '-')
        comment();
      else {
        boolean sense;
        if (c == '/') {
          sense = false;
          c = nextChar();
        } else
          sense = true;
  
        boolean isA = false;
  
        while (Character.isWhitespace((char)c))
          c = nextChar();
  
        boolean lastWhite = false;
 
       inStrong = false;
 
       if (c == 'a' || c == 'A') {
         if (Character.isWhitespace((char) (c = nextChar()))) {
           isA = sense;
           lastWhite = true;
         }
       } else if (c == 's' || c == 'S') {
         c = nextChar();
         if ((c == 't' || c == 'T')
           && ((c = nextChar()) == 'r' || c == 'R')
           && ((c = nextChar()) == 'o' || c == 'O')
           && ((c = nextChar()) == 'n' || c == 'N')
           && ((c = nextChar()) == 'g' || c == 'G'))
           inStrong = sense;
         else if (
           (c == 'c' || c == 'C')
             && ((c = nextChar()) == 'r' || c == 'R')
             && ((c = nextChar()) == 'i' || c == 'I')
             && ((c = nextChar()) == 'p' || c == 'P')
             && ((c = nextChar()) == 't' || c == 'T'))
           inScript = sense;
       }
 
       for (; c != '>'; c = nextChar())
         if (c == '"' || c == '\'')
           string(c);
         else if (isA) {
           if (lastWhite
             && (c == 'n' || c == 'N')
             && ((c = nextChar()) == 'a' || c == 'A')
             && ((c = nextChar()) == 'm' || c == 'M')
             && ((c = nextChar()) == 'e' || c == 'E')
             && ((c = nextChar()) == '=')) {
             StringBuffer anchorNameBuf = new StringBuffer();
             if ((c = nextChar()) == '"' || c == '\'') {
               int term = c;
               while ((c = nextChar()) != term)
                 anchorNameBuf.append((char)c);
             } else {
               for (;
                 Character.isLetterOrDigit((char)c) || c == '_';
                 c = nextChar())
                 anchorNameBuf.append((char)c);
               pushback(c);
             }
 
             anchorName = anchorNameBuf.toString();
           } else
             lastWhite = Character.isWhitespace((char)c);
         }
     }
 
     if (anchorName != null)
       wordOffset = tagOffset;
     return anchorName;
   }
 
   private void element() throws IOException {
     int c;
     while (Character.isLetterOrDigit((char) (c = nextChar())));
     if (c != ';')
       pushback(c);
   }
 
   private static boolean isPrint(char c) {
     return !Character.isWhitespace(c)
       && !Character.isISOControl(c)
       && c != '<'
       && c != '&';
   }
 
   private boolean isInteresting(char c) {
     return wantEverything ? isPrint(c) : Character.isLetter(c);
   }
 
   private boolean isInterestingNonLetter(char c) {
     return isPrint(c) && !Character.isLetter(c);
   }
 
   private String _nextWord() throws IOException {
     try {
       int c;
 
       hadBreak = lastWasAnchor;
       while (!isInteresting((char) (c = nextChar())) || inScript) {
         hadBreak = true;
         if (c == '<') {
           String anchorName = tag();
           if (anchorName != null) {
             // wordOffset has been set above
             lastWasAnchor = true;
             return "#" + anchorName;
           }
         } else if (c == '&')
           element();
       }
 
       lastWasAnchor = false;
       wordOffset = offset;
 
       buffer.setLength(1);
       buffer.setCharAt(0, (char)c);
 
       if (Character.isLetter((char)c)) {
         while (Character.isLetter((char) (c = nextChar())))
           buffer.append((char)c);
         ++wordIndex;
       } else
         while (isInterestingNonLetter((char) (c = nextChar())))
           buffer.append((char)c);
 
       pushback(c);
 
       String word = buffer.toString();
 
       if (buffer.capacity() > 1000)
         buffer = new StringBuffer();
 
       return inStrong && !wantEverything ? "$" + word : word;
     } catch (EOFException e) {
       return null;
     }
   }
 
   public boolean hadBreak() {
     return hadBreak;
   }
 
   public int wordOffset() {
     return wordOffset;
   }
 
   public int bytesReadFromUnderlyingStream() {
     return offset + (pushedBack == -1 ? 1 : 2);
   }
 
   public int wordIndex() {
     return wordIndex;
   }
 
   public synchronized String nextWord() {
     if (!hasMoreElements())
       throw new NoSuchElementException();
 
     try {
       return nextWord;
     } finally {
       nextWord = null;
     }
   }
 
   public final Object nextElement() {
     return nextWord();
   }
 
   public synchronized boolean hasMoreWords() throws IOException {
     return nextWord != null || (nextWord = _nextWord()) != null;
   }
 
   public final boolean hasMoreElements() {
     try {
       return hasMoreWords();
     } catch (IOException e) {
       throw new UnexpectedExceptionException(e);
     }
   }
 
   public static void main(String[] args) throws Exception {
     if (args[0].equals("-context")) {
       for (IndexTokenizer words =
         new IndexTokenizer(
           new BufferedInputStream(new FileInputStream(args[1])),
           true);
         words.hasMoreWords();
         ) {
         String word = words.nextWord();
         if (!word.startsWith("#")) {
           if (words.hadBreak())
             System.out.print(" ");
           System.out.print(word);
         }
       }
       System.out.println();
     } else {
       for (IndexTokenizer words =
         new IndexTokenizer(
           new BufferedInputStream(new FileInputStream(args[0])),
           args[1].equals("all"));
         words.hasMoreWords();
         )
         System.out.print(
           words.nextWord()
             + "("
             + words.wordOffset()
             + ":"
             + words.wordIndex()
             + ") ");
       System.out.println();
     }
   }
 }