Java/File Input Output/StreamTokenizer
Содержание
- 1 Create StreamTokenizer from BufferedReader
- 2 Create StreamTokenizer from FileReader
- 3 Deal with StreamTokenizer.TT_WORD, StreamTokenizer.TT_NUMBER
- 4 Extract Tokens with StreamTokenizer
- 5 Read file with StreamTokenizer
- 6 Set eolIsSignificant
- 7 Set whitespaceChars
- 8 Stream Token
- 9 Tokenizing Java Source Code
- 10 Using StreamTokenizer to count words and numbers in a file
Create StreamTokenizer from BufferedReader
<source lang="java">
import java.io.BufferedReader; import java.io.FileReader; import java.io.StreamTokenizer; class StreamTokenizerDemo {
public static void main(String args[]) throws Exception {
FileReader fr = new FileReader(args[0]);
BufferedReader br = new BufferedReader(fr);
StreamTokenizer st = new StreamTokenizer(br);
st.ordinaryChar(".");
st.wordChars("\"", "\"");
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_WORD:
System.out.println(st.lineno() + ") " + st.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println(st.lineno() + ") " + st.nval);
break;
default:
System.out.println(st.lineno() + ") " + (char) st.ttype);
}
}
fr.close();
}
}
</source>
Create StreamTokenizer from FileReader
<source lang="java">
import java.io.FileReader; import java.io.StreamTokenizer; public class ExtractTokens {
public static void main(String[] args) throws Exception {
FileReader fr = null;
fr = new FileReader(args[0]);
StreamTokenizer st = new StreamTokenizer(fr);
st.lowerCaseMode(true);
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case "\"":
case """:
System.out.println("String = " + st.sval);
break;
case StreamTokenizer.TT_EOL:
System.out.println("End-of-line");
break;
case StreamTokenizer.TT_NUMBER:
System.out.println("Number = " + st.nval);
break;
case StreamTokenizer.TT_WORD:
System.out.println("Word = " + st.sval);
break;
default:
System.out.println("Other = " + (char) st.ttype);
}
}
}
}
</source>
Deal with StreamTokenizer.TT_WORD, StreamTokenizer.TT_NUMBER
<source lang="java">
import java.io.BufferedReader; import java.io.FileReader; import java.io.StreamTokenizer; class MainClass {
public static void main(String args[]) throws Exception {
FileReader fr = new FileReader(args[0]);
BufferedReader br = new BufferedReader(fr);
StreamTokenizer st = new StreamTokenizer(br);
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_WORD:
System.out.println(st.lineno() + ") " + st.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println(st.lineno() + ") " + st.nval);
break;
default:
System.out.println(st.lineno() + ") " + (char) st.ttype);
}
}
fr.close();
}
}
</source>
Extract Tokens with StreamTokenizer
<source lang="java">
import java.io.FileReader; import java.io.StreamTokenizer; class ExtractTokens {
public static void main(String[] args) throws Exception {
FileReader fr = null;
fr = new FileReader(args[0]);
StreamTokenizer st = new StreamTokenizer(fr);
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case """:
System.out.println("String = " + st.sval);
break;
case StreamTokenizer.TT_EOL:
System.out.println("End-of-line");
break;
case StreamTokenizer.TT_NUMBER:
System.out.println("Number = " + st.nval);
break;
case StreamTokenizer.TT_WORD:
System.out.println("Word = " + st.sval);
break;
default:
System.out.println("Other = " + (char) st.ttype);
}
}
}
}
</source>
Read file with StreamTokenizer
<source lang="java">
import java.io.FileReader; import java.io.Reader; import java.io.StreamTokenizer; public class StreamTokenizerExample {
public static void main(String[] args) throws Exception {
Reader r = new FileReader("in.txt");
StreamTokenizer st = new StreamTokenizer(r);
while (st.nextToken() != StreamTokenizer.TT_EOF) {
System.out.println(st.sval);
}
r.close();
}
}
</source>
Set eolIsSignificant
<source lang="java">
import java.io.BufferedReader; import java.io.FileReader; import java.io.StreamTokenizer; class Main {
public static void main(String args[]) throws Exception {
FileReader fr = new FileReader(args[0]);
BufferedReader br = new BufferedReader(fr);
StreamTokenizer st = new StreamTokenizer(br);
st.eolIsSignificant(true);
int lines = 1;
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_EOL:
++lines;
}
}
System.out.println("There are " + lines + " lines");
fr.close();
}
}
</source>
Set whitespaceChars
<source lang="java">
import java.io.BufferedReader; import java.io.FileReader; import java.io.StreamTokenizer; class Main {
public static void main(String args[]) throws Exception {
FileReader fr = new FileReader(args[0]);
BufferedReader br = new BufferedReader(fr);
StreamTokenizer st = new StreamTokenizer(br);
st.whitespaceChars(",", ",");
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_WORD:
System.out.println(st.lineno() + ") " + st.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println(st.lineno() + ") " + st.nval);
break;
default:
System.out.println(st.lineno() + ") " + (char) st.ttype);
}
}
fr.close();
}
}
</source>
Stream Token
<source lang="java">
import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.StreamTokenizer; public class StreamTokenApp {
public static void main(String args[]) throws IOException {
BufferedReader inData = new BufferedReader(new InputStreamReader(System.in));
StreamTokenizer inStream = new StreamTokenizer(inData);
inStream.rumentChar("#");
boolean eof = false;
do {
int token = inStream.nextToken();
switch (token) {
case inStream.TT_EOF:
System.out.println("EOF encountered.");
eof = true;
break;
case inStream.TT_EOL:
System.out.println("EOL encountered.");
break;
case inStream.TT_WORD:
System.out.println("Word: " + inStream.sval);
break;
case inStream.TT_NUMBER:
System.out.println("Number: " + inStream.nval);
break;
default:
System.out.println((char) token + " encountered.");
if (token == "!")
eof = true;
}
} while (!eof);
}
}
</source>
Tokenizing Java Source Code
<source lang="java">
import java.io.FileReader; import java.io.StreamTokenizer; public class Main {
public static void main(String[] argv) throws Exception {
FileReader rd = new FileReader("filename.java");
StreamTokenizer st = new StreamTokenizer(rd);
st.parseNumbers();
st.wordChars("_", "_");
st.eolIsSignificant(true);
st.ordinaryChars(0, " ");
st.slashSlashComments(true);
st.slashStarComments(true);
int token = st.nextToken();
while (token != StreamTokenizer.TT_EOF) {
token = st.nextToken();
switch (token) {
case StreamTokenizer.TT_NUMBER:
double num = st.nval;
System.out.println(num);
break;
case StreamTokenizer.TT_WORD:
String word = st.sval;
System.out.println(word);
break;
case """:
String dquoteVal = st.sval;
System.out.println(dquoteVal);
break;
case "\"":
String squoteVal = st.sval;
System.out.println(squoteVal);
break;
case StreamTokenizer.TT_EOL:
break;
case StreamTokenizer.TT_EOF:
break;
default:
char ch = (char) st.ttype;
System.out.println(ch);
break;
}
}
rd.close();
}
}
</source>
Using StreamTokenizer to count words and numbers in a file
<source lang="java">
import java.io.FileReader; import java.io.StreamTokenizer; public class Main {
public static void main(String[] args) throws Exception {
int wordCount = 0, numberCount = 0;
StreamTokenizer sTokenizer = new StreamTokenizer(new FileReader("yourFile.txt"));
while (sTokenizer.nextToken() != StreamTokenizer.TT_EOF) {
if (sTokenizer.ttype == StreamTokenizer.TT_WORD)
wordCount++;
else if (sTokenizer.ttype == StreamTokenizer.TT_NUMBER)
numberCount++;
}
System.out.println("Number of words in file: " + wordCount);
System.out.println("Number of numbers in file: " + numberCount);
}
}
</source>
