Java/File Input Output/StreamTokenizer

Create StreamTokenizer from BufferedReader

 
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
class StreamTokenizerDemo {
  public static void main(String args[]) throws Exception {
    FileReader fr = new FileReader(args[0]);
    BufferedReader br = new BufferedReader(fr);
    StreamTokenizer st = new StreamTokenizer(br);
    st.ordinaryChar(".");
    st.wordChars("\"", "\"");
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case StreamTokenizer.TT_WORD:
        System.out.println(st.lineno() + ") " + st.sval);
        break;
      case StreamTokenizer.TT_NUMBER:
        System.out.println(st.lineno() + ") " + st.nval);
        break;
      default:
        System.out.println(st.lineno() + ") " + (char) st.ttype);
      }
    }
    fr.close();
  }
}

Create StreamTokenizer from FileReader

 
import java.io.FileReader;
import java.io.StreamTokenizer;
public class ExtractTokens {
  public static void main(String[] args) throws Exception {
    FileReader fr = null;
    fr = new FileReader(args[0]);
    StreamTokenizer st = new StreamTokenizer(fr);
    st.lowerCaseMode(true);
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case "\"":
      case """:
        System.out.println("String = " + st.sval);
        break;
      case StreamTokenizer.TT_EOL:
        System.out.println("End-of-line");
        break;
      case StreamTokenizer.TT_NUMBER:
        System.out.println("Number = " + st.nval);
        break;
      case StreamTokenizer.TT_WORD:
        System.out.println("Word = " + st.sval);
        break;
      default:
        System.out.println("Other = " + (char) st.ttype);
      }
    }
  }
}

Deal with StreamTokenizer.TT_WORD, StreamTokenizer.TT_NUMBER

 

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
class MainClass {
  public static void main(String args[]) throws Exception {
    FileReader fr = new FileReader(args[0]);
    BufferedReader br = new BufferedReader(fr);
    StreamTokenizer st = new StreamTokenizer(br);
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case StreamTokenizer.TT_WORD:
        System.out.println(st.lineno() + ") " + st.sval);
        break;
      case StreamTokenizer.TT_NUMBER:
        System.out.println(st.lineno() + ") " + st.nval);
        break;
      default:
        System.out.println(st.lineno() + ") " + (char) st.ttype);
      }
    }
    fr.close();
  }
}

Extract Tokens with StreamTokenizer

 
import java.io.FileReader;
import java.io.StreamTokenizer;
class ExtractTokens {
  public static void main(String[] args) throws Exception {
    FileReader fr = null;
    fr = new FileReader(args[0]);
    StreamTokenizer st = new StreamTokenizer(fr);
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case """:
        System.out.println("String = " + st.sval);
        break;
      case StreamTokenizer.TT_EOL:
        System.out.println("End-of-line");
        break;
      case StreamTokenizer.TT_NUMBER:
        System.out.println("Number = " + st.nval);
        break;
      case StreamTokenizer.TT_WORD:
        System.out.println("Word = " + st.sval);
        break;
      default:
        System.out.println("Other = " + (char) st.ttype);
      }
    }
  }
}

Read file with StreamTokenizer

 
import java.io.FileReader;
import java.io.Reader;
import java.io.StreamTokenizer;
public class StreamTokenizerExample {
  public static void main(String[] args) throws Exception {
    Reader r = new FileReader("in.txt");
    StreamTokenizer st = new StreamTokenizer(r);
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      System.out.println(st.sval);
    }
    r.close();
  }
}

Set eolIsSignificant

 
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
class Main {
  public static void main(String args[]) throws Exception {
    FileReader fr = new FileReader(args[0]);
    BufferedReader br = new BufferedReader(fr);
    StreamTokenizer st = new StreamTokenizer(br);
    st.eolIsSignificant(true);
    int lines = 1;
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case StreamTokenizer.TT_EOL:
        ++lines;
      }
    }
    System.out.println("There are " + lines + " lines");
    fr.close();
  }
}

Set whitespaceChars

 
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
class Main {
  public static void main(String args[]) throws Exception {
    FileReader fr = new FileReader(args[0]);
    BufferedReader br = new BufferedReader(fr);
    StreamTokenizer st = new StreamTokenizer(br);
    st.whitespaceChars(",", ",");
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
      switch (st.ttype) {
      case StreamTokenizer.TT_WORD:
        System.out.println(st.lineno() + ") " + st.sval);
        break;
      case StreamTokenizer.TT_NUMBER:
        System.out.println(st.lineno() + ") " + st.nval);
        break;
      default:
        System.out.println(st.lineno() + ") " + (char) st.ttype);
      }
    }
    fr.close();
  }
}

Stream Token

  
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StreamTokenizer;
public class StreamTokenApp {
  public static void main(String args[]) throws IOException {
    BufferedReader inData = new BufferedReader(new InputStreamReader(System.in));
    StreamTokenizer inStream = new StreamTokenizer(inData);
    inStream.rumentChar("#");
    boolean eof = false;
    do {
      int token = inStream.nextToken();
      switch (token) {
      case inStream.TT_EOF:
        System.out.println("EOF encountered.");
        eof = true;
        break;
      case inStream.TT_EOL:
        System.out.println("EOL encountered.");
        break;
      case inStream.TT_WORD:
        System.out.println("Word: " + inStream.sval);
        break;
      case inStream.TT_NUMBER:
        System.out.println("Number: " + inStream.nval);
        break;
      default:
        System.out.println((char) token + " encountered.");
        if (token == "!")
          eof = true;
      }
    } while (!eof);
  }
}

Tokenizing Java Source Code

 
import java.io.FileReader;
import java.io.StreamTokenizer;
public class Main {
  public static void main(String[] argv) throws Exception {
    FileReader rd = new FileReader("filename.java");
    StreamTokenizer st = new StreamTokenizer(rd);
    st.parseNumbers();
    st.wordChars("_", "_");
    st.eolIsSignificant(true);
    st.ordinaryChars(0, " ");
    st.slashSlashComments(true);
    st.slashStarComments(true);
    int token = st.nextToken();
    while (token != StreamTokenizer.TT_EOF) {
      token = st.nextToken();
      switch (token) {
      case StreamTokenizer.TT_NUMBER:
        double num = st.nval;
        System.out.println(num);
        break;
      case StreamTokenizer.TT_WORD:
        String word = st.sval;
        System.out.println(word);
        break;
      case """:
        String dquoteVal = st.sval;
        System.out.println(dquoteVal);
        break;
      case "\"":
        String squoteVal = st.sval;
        System.out.println(squoteVal);
        break;
      case StreamTokenizer.TT_EOL:
        break;
      case StreamTokenizer.TT_EOF:
        break;
      default:
        char ch = (char) st.ttype;
        System.out.println(ch);
        break;
      }
    }
    rd.close();
  }
}

Using StreamTokenizer to count words and numbers in a file

 

import java.io.FileReader;
import java.io.StreamTokenizer;
public class Main {
  public static void main(String[] args) throws Exception {
    int wordCount = 0, numberCount = 0;
    StreamTokenizer sTokenizer = new StreamTokenizer(new FileReader("yourFile.txt"));
    while (sTokenizer.nextToken() != StreamTokenizer.TT_EOF) {
      if (sTokenizer.ttype == StreamTokenizer.TT_WORD)
        wordCount++;
      else if (sTokenizer.ttype == StreamTokenizer.TT_NUMBER)
        numberCount++;
    }
    System.out.println("Number of words in file: " + wordCount);
    System.out.println("Number of numbers in file: " + numberCount);
  }
}