Java Tutorial/File/StreamTokenizer

Tokenizing Java Source Code

import java.io.FileReader;
import java.io.StreamTokenizer;
public class Main {
  public static void main(String[] argv) throws Exception {
    FileReader rd = new FileReader("filename.java");
    StreamTokenizer st = new StreamTokenizer(rd);
    st.parseNumbers();
    st.wordChars("_", "_");
    st.eolIsSignificant(true);
    st.ordinaryChars(0, " ");
    st.slashSlashComments(true);
    st.slashStarComments(true);
    int token = st.nextToken();
    while (token != StreamTokenizer.TT_EOF) {
      token = st.nextToken();
      switch (token) {
      case StreamTokenizer.TT_NUMBER:
        double num = st.nval;
        System.out.println(num);
        break;
      case StreamTokenizer.TT_WORD:
        String word = st.sval;
        System.out.println(word);
        break;
      case """:
        String dquoteVal = st.sval;
        System.out.println(dquoteVal);
        break;
      case "\"":
        String squoteVal = st.sval;
        System.out.println(squoteVal);
        break;
      case StreamTokenizer.TT_EOL:
        break;
      case StreamTokenizer.TT_EOF:
        break;
      default:
        char ch = (char) st.ttype;
        System.out.println(ch);
        break;
      }
    }
    rd.close();
  }
}

Using StreamTokenizer to count words and numbers in a file

import java.io.FileReader;
import java.io.StreamTokenizer;
public class Main {
  public static void main(String[] args) throws Exception {
    int wordCount = 0, numberCount = 0;
    StreamTokenizer sTokenizer = new StreamTokenizer(new FileReader("yourFile.txt"));
    while (sTokenizer.nextToken() != StreamTokenizer.TT_EOF) {
      if (sTokenizer.ttype == StreamTokenizer.TT_WORD)
        wordCount++;
      else if (sTokenizer.ttype == StreamTokenizer.TT_NUMBER)
        numberCount++;
    }
    System.out.println("Number of words in file: " + wordCount);
    System.out.println("Number of numbers in file: " + numberCount);
  }
}