package org.paneris.bibliomania.logs;
   
   import java.io.BufferedReader;
   import java.io.File;
   import java.io.FileReader;
   import java.io.FileWriter;
   import java.io.FilenameFilter;
   import java.io.InputStreamReader;
   import java.io.PrintWriter;
  import java.util.Calendar;
  import java.util.Date;
  import java.util.GregorianCalendar;
  
  import org.melati.poem.AccessToken;
  import org.melati.poem.NoSuchRowPoemException;
  import org.melati.poem.PoemTask;
  import org.melati.poem.UnexpectedExceptionPoemException;
  import org.melati.poem.util.StringUtils;
  import org.paneris.bibliomania.BibliomaniaDatabase;
  
  /*
   * This program will analyse the stats generated by the bibliomania.com site.
   * 
   * It is assumed to be run from cron at 2am EDT ie a crontab entry of
   * 0   2 * * *  /usr/local/logs/stats.sh 
   * where stats.sh runs this and outputs error and reports files.
   * 
   * It uses 2 analysis programs - weblaizer and analog.
   * It expects log files in a directory tree:
   * 
   * year/month/
   *
   * with each directory containing 1 log file per day.  
   * It will not do anything with today's file (as it will probably not be complete)
   * FIXME this means that the last day of every month is missed.
   *
   * Output for webalizer is according to the config file bibliomania.conf
   *
   * Output for analog is in the file analog.html in the directory for each month,
   * and the directory for the year.  The annual stats are recomputed every day.
   *
   * The stats are translated before being analysed stats to be trnaslated need
   * to be unzipped first.
   *
   * You can optionally supply a single parameter of the form yyyy/mm in order to 
   * process the stats for the specific month.  
   * Annual stats are still calculated.
   */  
  
  public class LogAnalysis {
    
    public static void main(final String[] args) throws Exception {
      
      // get today's date
      Calendar today = new GregorianCalendar();
      int year = today.get(Calendar.YEAR);
      int month = today.get(Calendar.MONTH) + 1;
      String monthString = month + "";
      if (month < 10) monthString = "0" + month;
      int day = today.get(Calendar.DAY_OF_MONTH);
      String dayString = day + "";
      if (day < 10) dayString = "0" + day;
      String todayYearmonth = year + File.separator + monthString;
      
      // default 
      String yearmonth = todayYearmonth;
      
      // allow us to run month by month
      if (args.length > 0) {
        yearmonth = args[0];
      }
      
      System.out.println("Running Stats for " + yearmonth);
      System.out.println("Started at: " + new Date());
      File dir = new File(yearmonth);
  
      Translator trans = new Translator();
  
      // find files that need translating, translate them,
      // analize them, and then zip up the output
      FilenameFilter webalizerFilter = new WebalizerFilter(todayYearmonth, dayString);
      File[] files = dir.listFiles(webalizerFilter);
      java.util.Arrays.sort(files);
      if (files != null) {
        for (int i=0; i<files.length; i++) {    
          System.out.println("Translating " + files[i]);
          File out = trans.doIt(files[i]);
          String outName = out.toString();
          out = null;
          System.out.println("Webalizering " + outName);
          run("webalizer -c bibliomania.conf " + outName);
          System.out.println("Gzipping " + files[i]);
          run("gzip " + files[i]);
          System.out.println("Gzipping " + outName);
          run("gzip " + outName);
        }
      }    
      
      // run the files for this month through analog
     File[] dirs = new File[1];
     dirs[0] = dir;
     analog(dirs, todayYearmonth, dayString, yearmonth);
     
     // do the annual stuff.
     int i = yearmonth.indexOf("/");
     if (i > 0) {
       String yeardir = yearmonth.substring(0,i);
       File updir =  new File(yeardir);
       files = updir.listFiles();
       analog(files,todayYearmonth,dayString,yeardir);
     }
     
     System.out.println("Completed at: " + new Date());
   }
   
   
   /**
    * Run analog on all translated files in the supplied list of directories.
    */
   static void analog(File[] dirs, String yearmonth, String dayString, String destination) throws Exception {
     FilenameFilter translatedFilter = new TranslatedFilter(yearmonth, dayString);
     String filenames = "";
     for (int i=0; i<dirs.length; i++) {    
       File[] files = dirs[i].listFiles(translatedFilter);
       if (files != null) {
         for (int j=0; j<files.length; j++) {    
           filenames += " " + files[j];
         }
       }
     }
     System.out.println("Analogging " + filenames);
     run("analog -G +ganalog.cfg -O" + destination + File.separator + "index.html" + filenames);
   }   
   
   static void run(String command) throws Exception {
     Process p = Runtime.getRuntime().exec(command);
     InputStreamReader error = new InputStreamReader(p.getErrorStream());
     InputStreamReader input = new InputStreamReader(p.getInputStream());
     boolean ok = true;
     while (ok) {
       ok = ((error.read() != -1) || (input.read() != -1));
     }
     p.waitFor();
   }
 
 }
 
 /**
  * Select all files that are not for today.
  */
 class NotTodayFilter implements FilenameFilter {
 
   String yearmonth, day;
   
   public NotTodayFilter(String yearmonth, String day) {
     this.yearmonth = yearmonth;
     this.day = day;
   }
   
   public boolean accept(File dir, String name) {
     if (!dir.toString().equals(yearmonth)) return true;
     if (name.length() > 1) {
       if (day.equals(name.substring(0,2))) return false;
     }
     return true;
   }
 }  
 
 
  
 /**
  * Select files that have been translated.
  */
 class TranslatedFilter extends NotTodayFilter {
   
   public TranslatedFilter(String yearmonth, String day) {
     super(yearmonth, day);
   }
 
   public boolean accept(File dir, String name) {
     if (name.endsWith("access.log.translated.gz") && super.accept(dir, name)) return true;
     return false;
   }
 }
 
 /**
  *  Select files that need translating and webalizing.
  */
 class WebalizerFilter extends NotTodayFilter {
   
   public WebalizerFilter(String yearmonth, String day) {
     super(yearmonth, day);
   }
 
   public boolean accept(File dir, String name) {
     File translated1 = new File(dir, name + ".translated.gz");
     File translated2 = new File(dir, name + ".translated");
     if (!translated1.exists() && !translated2.exists() && name.endsWith("access.log") && super.accept(dir, name)) return true;
     return false;
   }
 }  
 
 /**
  * Translate a log file.
  */
 class Translator {
 
   File fileoutput;
   final BibliomaniaDatabase db;
     
   public Translator() {
     db = new BibliomaniaDatabase(false);
     db.connect("bibliomania", "org.melati.poem.dbms.Postgresql",
     "jdbc:postgresql:bibliomania", "postgres", "*",4);
   }
   
   public File doIt(final File file) throws Exception {
     
     db.inSession(
     AccessToken.root,       // FIXME
     new PoemTask() {
       public void run() {
         try {
           doTranslate(db, file);
         }
         catch (Exception e) {
           throw new UnexpectedExceptionPoemException(e);
         }
       }
     }
     );
     return fileoutput;
   }
 
   private void doTranslate(BibliomaniaDatabase dbP, File fileinput) throws Exception {
     System.out.println("Started translation at: " + new Date());
     fileoutput = new File(fileinput.toString() + ".translated");
     FileReader filein = new FileReader(fileinput);
     PrintWriter fileout = new PrintWriter(new FileWriter(fileoutput));
     BufferedReader in = new BufferedReader(filein);
     int totalLines = 0;
     int translatedLines = 0;
     int failedLines = 0;
     int failedSectionGroup = 0;
     int failedSection = 0;
     int failedAuthor = 0;
     int failedBook = 0;
     String line = "";
     while ((line = in.readLine()) != null) {
       totalLines++;
       LogLine ln = new LogLine(dbP,line, totalLines);
       fileout.println(ln.translate());
       if (ln.translatedAll == LogLine.TRANSLATED) translatedLines++;
       if (ln.translatedAll == LogLine.FAILED_TRANSLATED) failedLines++;
       if (ln.translatedSectionGroup == LogLine.FAILED_TRANSLATED) failedSectionGroup++;
       if (ln.translatedSection == LogLine.FAILED_TRANSLATED) failedSection++;
       if (ln.translatedAuthor == LogLine.FAILED_TRANSLATED) failedAuthor++;
       if (ln.translatedBook == LogLine.FAILED_TRANSLATED) failedBook++;
     }
     in.close();
     fileout.close();
     System.out.println("Found " + totalLines + " lines, translated " + translatedLines + " lines, failed on "+ failedLines + " lines.");
     System.out.println("Failed:  SectionGroup " + failedSectionGroup + ", Section " + failedSection + ", Author " + failedAuthor + ", Book " + failedBook);
   }
 
 }
 
   class LogLine {
     
     static String lookForStart = "GET ";
     static String lookForEnd = " HTTP"; 
     static int lookForStartLength = lookForStart.length();
     
     static int NOT_TRANSLATED = 0;
     static int TRANSLATED = 1;
     static int FAILED_TRANSLATED = 2;
     
     int translatedAll = NOT_TRANSLATED;
     int translatedSectionGroup = NOT_TRANSLATED;
     int translatedSection = NOT_TRANSLATED;
     int translatedAuthor = NOT_TRANSLATED;
     int translatedBook = NOT_TRANSLATED;
     
     int lineno;
     
     String in;
     String middle = "";
     String end = "";
     BibliomaniaDatabase db;
     
     public LogLine(BibliomaniaDatabase db, String in, int lineno) {
       this.in = in;
       this.db = db;
       this.lineno = lineno;
     }
     
     public String translate() {
       String start = in;
       String endBit = "";
       String middleBit = "";
       int length = in.length();
       int startPoint = in.indexOf(lookForStart);
       if (startPoint > -1) {
         startPoint += lookForStartLength;
         start = in.substring(0,startPoint);
         endBit = in.substring(startPoint, length);
         int middlePoint = in.indexOf(lookForEnd, startPoint);
         if (middlePoint > -1) {
           middleBit = in.substring(startPoint,middlePoint);
           middleBit = lookup(middleBit);
           endBit = in.substring(middlePoint, length);
         }
       }
       if (translatedSectionGroup == TRANSLATED || 
           translatedSection == TRANSLATED ||
           translatedAuthor == TRANSLATED ||
           translatedBook == TRANSLATED) {
         translatedAll = TRANSLATED;
       }
       if (translatedSectionGroup == FAILED_TRANSLATED || 
           translatedSection == FAILED_TRANSLATED ||
           translatedAuthor == FAILED_TRANSLATED ||
           translatedBook == FAILED_TRANSLATED) {
         translatedAll = FAILED_TRANSLATED;
       }
       return start + middleBit + endBit;
     }
     
     public String lookup(String inP) {
       if (inP.startsWith("/")) inP = inP.substring(1,inP.length());
       String[] split = StringUtils.split(inP, '/');
       int len = split.length;
       if (len > 4) len = 4;
       try {
         for (int i=0; i<len; i++) {
           split[i] = look(i,split[i]);
         }
       } catch (NumberFormatException e) {}
       return join(split);
     }
     
     public String look(int place,String inP) throws NumberFormatException {
       Integer i = new Integer(inP);
       switch (place) {
         case 0: 
           return lookupSectionGroup(i);
         case 1: 
           return lookupSection(i);
         case 2: 
           return lookupAuthor(i);
         case 3: 
           return lookupBook(i);
         default:
           return inP;
       }
     }
 
     public String lookupSectionGroup(Integer i) {
       try {
         translatedSectionGroup = TRANSLATED;
         return nospaces(db.getSectionGroupTable().getSectionGroupObject(i).getDisplayname());
       } catch (NoSuchRowPoemException e) {
         translatedSectionGroup = FAILED_TRANSLATED;
         System.err.println("Line no " + lineno + "  Not Found: Section Group " + i);
         System.err.println("        " + in);
         return i + "";
       }
     }
 
     public String lookupSection(Integer i) throws NoSuchRowPoemException {
       try {
         translatedSection = TRANSLATED;
         return nospaces(db.getSectionTable().getSectionObject(i).getDisplayname());
       } catch (NoSuchRowPoemException e) {
         translatedSection = FAILED_TRANSLATED;
         System.err.println("Line no " + lineno + "  Not Found:       Section " + i);
         System.err.println("        " + in);
         return i + "";
       }
     }
 
     public String lookupAuthor(Integer i) throws NoSuchRowPoemException {
       try {
         translatedAuthor = TRANSLATED;
         return nospaces(db.getAuthorTable().getAuthorObject(i).getSortname());
       } catch (NoSuchRowPoemException e) {
         translatedAuthor = FAILED_TRANSLATED;
         System.err.println("Line no " + lineno + "  Not Found:        Author " + i);
         System.err.println("        " + in);
         return i + "";
       }
     }
 
     public String lookupBook(Integer i) throws NoSuchRowPoemException {
       try {
         translatedBook = TRANSLATED;
         return nospaces(db.getBookTable().getBookObject(i).getTitle());
       } catch (NoSuchRowPoemException e) {
         translatedBook = FAILED_TRANSLATED;
         System.err.println("Line no " + lineno + "  Not Found:          Book " + i);
         System.err.println("        " + in);
         return i + "";
       }
     }
 
     public String join(String[] split) {
       String out = "";
       for (int i=0; i<split.length; i++) {
         out += "/" + split[i];
       }
       return out;
     }
     
     public static String nospaces(String s) {
       if (s == null) return null;
       String out = s.replace(' ','_');
       return out.replace('&','n');
     }
   }