Java/Network Protocol/Web Server Client

A Simple Web Server

/*
Common Port Assignments and Corresponding RFC Numbers              
Port Common Name RFC#  Purpose
7     Echo        862   Echoes data back. Used mostly for testing.
9     Discard     863   Discards all data sent to it. Used mostly for testing.
13    Daytime     867   Gets the date and time.
17    Quotd       865   Gets the quote of the day.
19    Chargen     864   Generates characters. Used mostly for testing.
20    ftp-data    959   Transfers files. FTP stands for File Transfer Protocol.
21    ftp         959   Transfers files as well as commands.
23    telnet      854   Logs on to remote systems.
25    SMTP        821   Transfers Internet mail. Stands for Simple Mail Transfer Protocol.
37    Time        868   Determines the system time on computers.
43    whois       954   Determines a user"s name on a remote system.
70    gopher     1436   Looks up documents, but has been mostly replaced by HTTP.
79    finger     1288   Determines information about users on other systems.
80    http       1945   Transfer documents. Forms the foundation of the Web.
110   pop3       1939   Accesses message stored on servers. Stands for Post Office Protocol, version 3.
443   https      n/a    Allows HTTP communications to be secure. Stands for Hypertext Transfer Protocol over Secure Sockets Layer (SSL).
*/
///A Simple Web Server (WebServer.java)
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.ServerSocket;
import java.net.Socket;
/**
 * Example program from Chapter 1 Programming Spiders, Bots and Aggregators in
 * Java Copyright 2001 by Jeff Heaton
 * 
 * WebServer is a very simple web-server. Any request is responded with a very
 * simple web-page.
 * 
 * @author Jeff Heaton
 * @version 1.0
 */
/*
 * Examining the Mini Web Server
 * 
 * Server sockets use the ServerSocket object rather than the Socket object that
 * client sockets use. There are several constructors available with the
 * ServerSocket object. The simplest constructor accepts only the port number on
 * which the program should be listening. Listening refers to the mode that a
 * server is in while it waits for clients to connect. The following lines of
 * code are used in Listing 1.3 to create a new ServerSocket object and reserve
 * port 80 as the port number on which the web server should listen for
 * connections:
 * 
 * try { 
 * // create the main server 
 * socket s = new ServerSocket(80); 
 * }
 * catch(Exception e) { 
 * System.out.println("Error: " + e ); 
 * return; 
 * }
 * 
 * The try block is necessary because any number of errors could occur when the
 * program attempts to register port 80. The most common error that would result
 * is that there is already a server listening to port 80 on this machine.
 * Warning
 * 
 * This program will not work on a machine that already has a web server, or
 * some other program, listening on port 80.
 * 
 * Once the program has port 80 registered, it can begin listening for
 * connections. The following line of code is used to wait for a connection:
 * 
 * Socket remote = s.accept();
 * 
 * The Socket object that is returned by accept is exactly the same class that
 * is used for client sockets. Once the connection is established, the
 * difference between client and server sockets fade. The primary difference
 * between client and server sockets is the way in which they connect. A client
 * sever connects to something. A server socket waits for something to connect
 * to it.
 * 
 * The accept method is a blocking call, which means the current thread will
 * wait for a connection. This can present problems for your program if there
 * are other tasks it would like to accomplish while it is waiting for
 * connections. Because of this, it is very common to see the accept method call
 * placed in a worker thread. This allows the main thread to carry on other
 * tasks, while the worker thread waits for connections to arrive.
 * 
 * Once a connection is made, the accept method will return a socket object for
 * the new socket. After this point, reading and writing is the same between
 * client and server sockets. Many client server programs would create a new
 * thread to handle this new connection.
 * 
 * Now that a connection has been made, a new thread could be created to handle
 * it. This new worker thread would process all the requests from this client in
 * the background, which allows the ServerSocket object to wait for and service
 * more connections. However, the example program in Listing 1.3 does not
 * require such programming. As soon as the socket is accepted, input and output
 * objects are created; this same process was used with the SMTP client. The
 * following lines from Listing 1.3 show the process of preparing the newly
 * accepted socket for input and output:
 * 
 * //remote is now the connected socket 
 * System.out.println("Connection, sending data."); 
 * BufferedReader in 
 * = new BufferedReader( new InputStreamReader(remote.getInputStream()) ); 
 * PrintWriter out = new PrintWriter(remote.getOutputStream());
 * 
 * Now that the program has input and output objects, it can process the HTTP
 * request. It first reads the HTTP request lines. A full-featured server would
 * parse each line and determine the exact nature of this request, however, our
 * ultra-simple web server just reads in the request lines and ignores them, as
 * shown here:
 * 
 * //read the data sent. We basically ignore it, 
 * //stop reading once a blank line is hit. This 
 * //blank line signals the end of the 
 * //client HTTP headers.
 *  String str="."; 
 * while(!str.equals("")) 
 * str = in.readLine();
 * 
 * These lines cause the server to read in lines of text from the newly
 * connected socket. Once a blank line (which indicates the end of the HTTP
 * header) is reached, the loop stops, and the server stops reading. Now that
 * the HTTP header has been retrieved, the server sends an HTTP response. The
 * following lines of code accomplish this:
 * 
 * //Send the response 
 * //Send the headers out.println("HTTP/1.0 200 OK");
 * out.println("Content-Type: text/html"); 
 * out.println("Server: Bot"); 
 * //this blank line signals the end of the headers out.println("");
 * // Send the HTML page out.println( " <H1> Welcome to the Ultra Mini-WebServer </H2> ");
 * 
 * Status code 200, as shown on line 3 of the preceding code, is used to show
 * that the page was properly transferred, and that the required HTTP headers
 * were sent. (Refer to Chapter 2 for more information about HTTP headers.)
 * Following the HTTP headers, the actual HTML page is transferred. Once the
 * page is transferred, the following lines of code from Listing 1.3 are
 * executed to clean up:
 * 
 * out.flush(); remote.close();
 * 
 * The flush method is necessary to ensure that all data is transferred, and the
 * close method is necessary to close the socket. Although Java will discard the
 * Socket object, it will not generally close the socket on most platforms.
 * Because of this, you must close the socket or else you might eventually get
 * an error indicating that there are no more file handles. This becomes very
 * important for a program that opens up many connections, including one to a
 * spider.
 */
public class WebServer {
  /**
   * WebServer constructor.
   */
  protected void start() {
    ServerSocket s;
    System.out.println("Webserver starting up on port 80");
    System.out.println("(press ctrl-c to exit)");
    try {
      // create the main server socket
      s = new ServerSocket(80);
    } catch (Exception e) {
      System.out.println("Error: " + e);
      return;
    }
    System.out.println("Waiting for connection");
    for (;;) {
      try {
        // wait for a connection
        Socket remote = s.accept();
        // remote is now the connected socket
        System.out.println("Connection, sending data.");
        BufferedReader in = new BufferedReader(new InputStreamReader(
            remote.getInputStream()));
        PrintWriter out = new PrintWriter(remote.getOutputStream());
        // read the data sent. We basically ignore it,
        // stop reading once a blank line is hit. This
        // blank line signals the end of the client HTTP
        // headers.
        String str = ".";
        while (!str.equals(""))
          str = in.readLine();
        // Send the response
        // Send the headers
        out.println("HTTP/1.0 200 OK");
        out.println("Content-Type: text/html");
        out.println("Server: Bot");
        // this blank line signals the end of the headers
        out.println("");
        // Send the HTML page
        out.println("<H1>Welcome to the Ultra Mini-WebServer</H2>");
        out.flush();
        remote.close();
      } catch (Exception e) {
        System.out.println("Error: " + e);
      }
    }
  }
  /**
   * Start the application.
   * 
   * @param args
   *            Command line parameters are not used.
   */
  public static void main(String args[]) {
    WebServer ws = new WebServer();
    ws.start();
  }
}

Connect with a Web server

import java.net.InetAddress;
import java.net.Socket;
public class WebPing {
  public static void main(String[] args) {
    try {
      InetAddress addr;
      Socket sock = new Socket("www.jexp.ru", 80);
      addr = sock.getInetAddress();
      System.out.println("Connected to " + addr);
      sock.close();
    } catch (java.io.IOException e) {
      System.out.println("Can"t connect to " + args[0]);
      System.out.println(e);
    }
  }
}

Reading URLs Protected with HTTP Authentication

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.Authenticator;
import java.net.MalformedURLException;
import java.net.PasswordAuthentication;
import java.net.URL;
public class AuthDemo {
  public static void main(String args[]) throws MalformedURLException,
      IOException {
    String urlString = "";
    String username = "";
    String password = "";
    Authenticator.setDefault(new MyAuthenticator(username, password));
    URL url = new URL(urlString);
    InputStream content = (InputStream) url.getContent();
    BufferedReader in = new BufferedReader(new InputStreamReader(content));
    String line;
    while ((line = in.readLine()) != null) {
      System.out.println(line);
    }
    System.out.println("Done.");
  }
  static class MyAuthenticator extends Authenticator {
    private String username, password;
    public MyAuthenticator(String user, String pass) {
      username = user;
      password = pass;
    }
    protected PasswordAuthentication getPasswordAuthentication() {
      System.out.println("Requesting Host  : " + getRequestingHost());
      System.out.println("Requesting Port  : " + getRequestingPort());
      System.out.println("Requesting Prompt : " + getRequestingPrompt());
      System.out.println("Requesting Protocol: "
          + getRequestingProtocol());
      System.out.println("Requesting Scheme : " + getRequestingScheme());
      System.out.println("Requesting Site  : " + getRequestingSite());
      return new PasswordAuthentication(username, password.toCharArray());
    }
  }
}

Reading Web Pages with Nonblocking Channels

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.SocketChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.Set;
public class GetWebPageApp {
  static Selector selector;
  public static void main(String args[]) throws Exception {
    String resource, host, file;
    int slashPos;
    resource = "www.jexp.ru/index.htm"; // skip HTTP://
    slashPos = resource.indexOf("/"); // find host/file separator
    if (slashPos < 0) {
      resource = resource + "/";
      slashPos = resource.indexOf("/");
    }
    file = resource.substring(slashPos); // isolate host and file parts
    host = resource.substring(0, slashPos);
    System.out.println("Host to contact: "" + host + """);
    System.out.println("File to fetch : "" + file + """);
    SocketChannel channel = null;
    try {
      Charset charset = Charset.forName("ISO-8859-1");
      CharsetDecoder decoder = charset.newDecoder();
      CharsetEncoder encoder = charset.newEncoder();
      ByteBuffer buffer = ByteBuffer.allocateDirect(1024);
      CharBuffer charBuffer = CharBuffer.allocate(1024);
      InetSocketAddress socketAddress = new InetSocketAddress(host, 80);
      channel = SocketChannel.open();
      channel.configureBlocking(false);
      channel.connect(socketAddress);
      selector = Selector.open();
      channel.register(selector, SelectionKey.OP_CONNECT
          | SelectionKey.OP_READ);
      while (selector.select(500) > 0) {
        Set readyKeys = selector.selectedKeys();
        try {
          Iterator readyItor = readyKeys.iterator();
          while (readyItor.hasNext()) {
            SelectionKey key = (SelectionKey) readyItor.next();
            readyItor.remove();
            SocketChannel keyChannel = (SocketChannel) key
                .channel();
            if (key.isConnectable()) {
              if (keyChannel.isConnectionPending()) {
                keyChannel.finishConnect();
              }
              String request = "GET " + file + " \r\n\r\n";
              keyChannel.write(encoder.encode(CharBuffer
                  .wrap(request)));
            } else if (key.isReadable()) {
              keyChannel.read(buffer);
              buffer.flip();
              decoder.decode(buffer, charBuffer, false);
              charBuffer.flip();
              System.out.print(charBuffer);
              buffer.clear();
              charBuffer.clear();
            } else {
              System.err.println("Unknown key");
            }
          }
        } catch (ConcurrentModificationException e) {
        }
      }
    } catch (UnknownHostException e) {
      System.err.println(e);
    } catch (IOException e) {
      System.err.println(e);
    } finally {
      if (channel != null) {
        try {
          channel.close();
        } catch (IOException ignored) {
        }
      }
    }
    System.out.println("\nDone.");
  }
}

Reading Web Pages, with Socket Channels

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.SocketChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
public class GetWebPageDemo {
  public static void main(String args[]) throws Exception {
    String resource, host, file;
    int slashPos;
    resource = "www.jexp.ru/index.htm"; 
    slashPos = resource.indexOf("/"); // find host/file separator
    if (slashPos < 0) {
      resource = resource + "/";
      slashPos = resource.indexOf("/");
    }
    file = resource.substring(slashPos); // isolate host and file parts
    host = resource.substring(0, slashPos);
    System.out.println("Host to contact: "" + host + """);
    System.out.println("File to fetch : "" + file + """);
    SocketChannel channel = null;
    try {
      Charset charset = Charset.forName("ISO-8859-1");
      CharsetDecoder decoder = charset.newDecoder();
      CharsetEncoder encoder = charset.newEncoder();
      ByteBuffer buffer = ByteBuffer.allocateDirect(1024);
      CharBuffer charBuffer = CharBuffer.allocate(1024);
      InetSocketAddress socketAddress = new InetSocketAddress(host, 80);
      channel = SocketChannel.open();
      channel.connect(socketAddress);
      String request = "GET " + file + " \r\n\r\n";
      channel.write(encoder.encode(CharBuffer.wrap(request)));
      while ((channel.read(buffer)) != -1) {
        buffer.flip();
        decoder.decode(buffer, charBuffer, false);
        charBuffer.flip();
        System.out.println(charBuffer);
        buffer.clear();
        charBuffer.clear();
      }
    } catch (UnknownHostException e) {
      System.err.println(e);
    } catch (IOException e) {
      System.err.println(e);
    } finally {
      if (channel != null) {
        try {
          channel.close();
        } catch (IOException ignored) {
        }
      }
    }
    System.out.println("\nDone.");
  }
}

Reading Web Pages with Streams

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;
public class GetWebPage {
  public static void main(String args[]) throws IOException,
      UnknownHostException {
    String resource, host, file;
    int slashPos;
    resource = "http://www.jexp.ru/index.htm".substring(7); // skip HTTP://
    slashPos = resource.indexOf("/"); 
    if (slashPos < 0) {
      resource = resource + "/";
      slashPos = resource.indexOf("/");
    }
    file = resource.substring(slashPos); // isolate host and file parts
    host = resource.substring(0, slashPos);
    System.out.println("Host to contact: "" + host + """);
    System.out.println("File to fetch : "" + file + """);
    MyHTTPConnection webConnection = new MyHTTPConnection(host);
    if (webConnection != null) {
      BufferedReader in = webConnection.get(file);
      String line;
      while ((line = in.readLine()) != null) { // read until EOF
        System.out.println(line);
      }
    }
    System.out.println("\nDone.");
  }
  static class MyHTTPConnection {
    public final static int HTTP_PORT = 80;
    InetAddress wwwHost;
    DataInputStream dataInputStream;
    PrintStream outputStream;
    public MyHTTPConnection(String host) throws UnknownHostException {
      wwwHost = InetAddress.getByName(host);
      System.out.println("WWW host = " + wwwHost);
    }
    public BufferedReader get(String file) throws IOException {
      Socket httpPipe;
      InputStream in;
      OutputStream out;
      BufferedReader bufReader;
      PrintWriter printWinter;
      httpPipe = new Socket(wwwHost, HTTP_PORT);
      if (httpPipe == null) {
        return null;
      }
      // get raw streams
      in = httpPipe.getInputStream();
      out = httpPipe.getOutputStream();
      // turn into useful ones
      bufReader = new BufferedReader(new InputStreamReader(in));
      printWinter = new PrintWriter(new OutputStreamWriter(out), true);
      if (in == null || out == null || bufReader == null || printWinter == null) {
        System.out.println("Failed to open streams to socket.");
        return null;
      }
      // send GET request
      System.out.println("GET " + file + " HTTP/1.0\n");
      printWinter.println("GET " + file + " HTTP/1.0\n");
      // read response until blank separator line
      String response;
      while ((response = bufReader.readLine()).length() > 0) {
        System.out.println(response);
      }
      return bufReader; 
    }
  }
}

Save binary file from web

import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
public class MainClass {
  public static void main(String args[]) {
    try {
      URL root = new URL("http://");
      saveBinaryFile(root);
    } catch (MalformedURLException e) {
      System.err.println("not URL I understand.");
    }
  }
  public static void saveBinaryFile(URL u) {
    int bufferLength = 128;
    try {
      URLConnection uc = u.openConnection();
      String ct = uc.getContentType();
      int contentLength = uc.getContentLength();
      if (ct.startsWith("text/") || contentLength == -1) {
        System.err.println("This is not a binary file.");
        return;
      }
      InputStream stream = uc.getInputStream();
      byte[] buffer = new byte[contentLength];
      int bytesread = 0;
      int offset = 0;
      while (bytesread >= 0) {
        bytesread = stream.read(buffer, offset, bufferLength);
        if (bytesread == -1)
          break;
        offset += bytesread;
      }
      if (offset != contentLength) {
        System.err.println("Error: Only read " + offset + " bytes");
        System.err.println("Expected " + contentLength + " bytes");
      }
      String theFile = u.getFile();
      theFile = theFile.substring(theFile.lastIndexOf("/") + 1);
      FileOutputStream fout = new FileOutputStream(theFile);
      fout.write(buffer);
    } catch (Exception e) {
      System.err.println(e);
    }
    return;
  }
}