Java/Network Protocol/HttpURLConnection

Available Port Finder

 
/*
 *  Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 *
 */
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.ServerSocket;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeSet;
/**
 * Finds currently available server ports.
 *
 * @author The Apache MINA Project (dev@mina.apache.org)
 * @version $Rev: 576217 $
 * @see 
 */
public class AvailablePortFinder {
    /**
     * The minimum number of server port number.
     */
    public static final int MIN_PORT_NUMBER = 1;
    /**
     * The maximum number of server port number.
     */
    public static final int MAX_PORT_NUMBER = 49151;
    /**
     * Creates a new instance.
     */
    private AvailablePortFinder() {
    }
    /**
     * Returns the {@link Set} of currently available port numbers
     * ({@link Integer}).  This method is identical to
     * <code>getAvailablePorts(MIN_PORT_NUMBER, MAX_PORT_NUMBER)</code>.
     *
     * WARNING: this can take a very long time.
     */
    public static Set<Integer> getAvailablePorts() {
        return getAvailablePorts(MIN_PORT_NUMBER, MAX_PORT_NUMBER);
    }
    /**
     * Gets the next available port starting at the lowest port number.
     *
     * @throws NoSuchElementException if there are no ports available
     */
    public static int getNextAvailable() {
        return getNextAvailable(MIN_PORT_NUMBER);
    }
    /**
     * Gets the next available port starting at a port.
     *
     * @param fromPort the port to scan for availability
     * @throws NoSuchElementException if there are no ports available
     */
    public static int getNextAvailable(int fromPort) {
        if (fromPort < MIN_PORT_NUMBER || fromPort > MAX_PORT_NUMBER) {
            throw new IllegalArgumentException("Invalid start port: "
                    + fromPort);
        }
        for (int i = fromPort; i <= MAX_PORT_NUMBER; i++) {
            if (available(i)) {
                return i;
            }
        }
        throw new NoSuchElementException("Could not find an available port "
                + "above " + fromPort);
    }
    /**
     * Checks to see if a specific port is available.
     *
     * @param port the port to check for availability
     */
    public static boolean available(int port) {
        if (port < MIN_PORT_NUMBER || port > MAX_PORT_NUMBER) {
            throw new IllegalArgumentException("Invalid start port: " + port);
        }
        ServerSocket ss = null;
        DatagramSocket ds = null;
        try {
            ss = new ServerSocket(port);
            ss.setReuseAddress(true);
            ds = new DatagramSocket(port);
            ds.setReuseAddress(true);
            return true;
        } catch (IOException e) {
        } finally {
            if (ds != null) {
                ds.close();
            }
            if (ss != null) {
                try {
                    ss.close();
                } catch (IOException e) {
                    /* should not be thrown */
                }
            }
        }
        return false;
    }
    /**
     * Returns the {@link Set} of currently avaliable port numbers ({@link Integer})
     * between the specified port range.
     *
     * @throws IllegalArgumentException if port range is not between
     * {@link #MIN_PORT_NUMBER} and {@link #MAX_PORT_NUMBER} or
     * <code>fromPort</code> if greater than <code>toPort</code>.
     */
    public static Set<Integer> getAvailablePorts(int fromPort, int toPort) {
        if (fromPort < MIN_PORT_NUMBER || toPort > MAX_PORT_NUMBER
                || fromPort > toPort) {
            throw new IllegalArgumentException("Invalid port range: "
                    + fromPort + " ~ " + toPort);
        }
        Set<Integer> result = new TreeSet<Integer>();
        for (int i = fromPort; i <= toPort; i++) {
            ServerSocket s = null;
            try {
                s = new ServerSocket(i);
                result.add(new Integer(i));
            } catch (IOException e) {
            } finally {
                if (s != null) {
                    try {
                        s.close();
                    } catch (IOException e) {
                        /* should not be thrown */
                    }
                }
            }
        }
        return result;
    }
}

A Web Page Source Viewer

    
import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
public class Main extends JFrame {
  JButton go = new JButton("Get Source");
  JTextArea codeArea = new JTextArea();
  public Main() {
    JPanel inputPanel = new JPanel(new BorderLayout(3, 3));
    go.addActionListener(new ActionListener() {
      public void actionPerformed(ActionEvent e) {
        try {
          URL pageURL = new URL("http://www.google.ru");
          HttpURLConnection urlConnection = (HttpURLConnection) pageURL.openConnection();
          int respCode = urlConnection.getResponseCode();
          String response = urlConnection.getResponseMessage();
          codeArea.setText("HTTP/1.x " + respCode + " " + response + "\n");
          int count = 1;
          while (true) {
            String header = urlConnection.getHeaderField(count);
            String key = urlConnection.getHeaderFieldKey(count);
            if (header == null || key == null) {
              break;
            }
            codeArea.append(urlConnection.getHeaderFieldKey(count) + ": " + header + "\n");
            count++;
          }
          InputStream in = new BufferedInputStream(urlConnection.getInputStream());
          Reader r = new InputStreamReader(in);
          int c;
          while ((c = r.read()) != -1) {
            codeArea.append(String.valueOf((char) c));
          }
          codeArea.setCaretPosition(1);
        } catch (Exception ee) {
        }
      }
    });
    inputPanel.add(BorderLayout.EAST, go);
    JScrollPane codeScroller = new JScrollPane(codeArea);
    add(BorderLayout.NORTH, inputPanel);
    add(BorderLayout.CENTER, codeScroller);
    this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    this.setSize(700, 500);
    this.setVisible(true);
  }
  public static void main(String[] args) {
    JFrame webPageSourceViewer = new Main();
  }
}

Check if a page exists

    
import java.net.HttpURLConnection;
import java.net.URL;
public class Main {
  public static void main(String[] argv) throws Exception {
    HttpURLConnection.setFollowRedirects(false);
    HttpURLConnection con = (HttpURLConnection) new URL("http://www.google.coom").openConnection();
    con.setRequestMethod("HEAD");
    System.out.println(con.getResponseCode() == HttpURLConnection.HTTP_OK);
  }
}

Connect through a Proxy

    
import java.io.DataInputStream;
import java.io.FileOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Properties;
import sun.misc.BASE64Encoder;
public class Main {
  public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");
    URL u = new URL("http://www.google.ru");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    BASE64Encoder encoder = new BASE64Encoder();
    String encodedUserPwd = encoder.encode("mydomain\\MYUSER:MYPASSWORD".getBytes());
    con.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd);
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
      System.out.print(new String(b));
    }
  }
}

Converting x-www-form-urlencoded Data

    
import java.net.URLEncoder;
public class Main {
  public static void main(String[] argv) throws Exception {
     String line = URLEncoder.encode("name1", "UTF-8") + "=" + URLEncoder.encode("value1", "UTF-8");
  }
}

Display header information

    

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    Map<String, List<String>> hdrs = httpCon.getHeaderFields();
    Set<String> hdrKeys = hdrs.keySet();
    for (String k : hdrKeys)
      System.out.println("Key: " + k + "  Value: " + hdrs.get(k));
 }
}

Display request method

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    System.out.println("Request method is " + httpCon.getRequestMethod());
 }
}

Display response message

    

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    System.out.println("Response Message is " + httpCon.getResponseMessage());
 }
}

Download and display the content

    
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
public class Main {
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    InputStream inStrm = httpCon.getInputStream();
    System.out.println("\nContent at " + url);
    int ch;
    while (((ch = inStrm.read()) != -1))
      System.out.print((char) ch);
    inStrm.close();
  }
}

Get response code

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    System.out.println("Response code is " + httpCon.getResponseCode());
 }
}

Get the date of a url connection

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getDate();
    if (date == 0)
      System.out.println("No date information.");
    else
      System.out.println("Date: " + new Date(date));
 }
}

Get the document expiration date

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getExpiration();
    if (date == 0)
      System.out.println("No expiration information.");
    else
      System.out.println("Expires: " + new Date(date));
 }
}

Get the document Last-modified date

    

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getLastModified();
    if (date == 0)
      System.out.println("No last-modified information.");
    else
      System.out.println("Last-Modified: " + new Date(date));
 }
}

Getting the Cookies from an HTTP Connection

    
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("http://hostname:80");
    URLConnection conn = url.openConnection();
    for (int i = 0;; i++) {
      String headerName = conn.getHeaderFieldKey(i);
      String headerValue = conn.getHeaderField(i);
      if (headerName == null && headerValue == null) {
        break;
      }
      if ("Set-Cookie".equalsIgnoreCase(headerName)) {
        String[] fields = headerValue.split(";\\s*");
        for (int j = 1; j < fields.length; j++) {
          if ("secure".equalsIgnoreCase(fields[j])) {
            System.out.println("secure=true");
          } else if (fields[j].indexOf("=") > 0) {
            String[] f = fields[j].split("=");
            if ("expires".equalsIgnoreCase(f[0])) {
              System.out.println("expires"+ f[1]);
            } else if ("domain".equalsIgnoreCase(f[0])) {
              System.out.println("domain"+ f[1]);
            } else if ("path".equalsIgnoreCase(f[0])) {
              System.out.println("path"+ f[1]);
            }
          }
        }
      }
    }
  }
}

Getting the Response Headers from an HTTP Connection

    
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("http://hostname:80");
    URLConnection conn = url.openConnection();
    for (int i = 0;; i++) {
      String headerName = conn.getHeaderFieldKey(i);
      String headerValue = conn.getHeaderField(i);
      System.out.println(headerName);
      System.out.println(headerValue);
      if (headerName == null && headerValue == null) {
        System.out.println("No more headers");
        break;
      }
    }
  }
}

Grabbing a page using socket

   
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.Socket;
import java.net.URL;
public class Main {
  public static void main(String[] args) throws Exception {
    String pageAddr = "http://www.google.ru/index.htm";
    URL url = new URL(pageAddr);
    String websiteAddress = url.getHost();
    String file = url.getFile();
    Socket clientSocket = new Socket(websiteAddress, 80);
    BufferedReader inFromServer = new BufferedReader(new InputStreamReader(clientSocket
        .getInputStream()));
    OutputStreamWriter outWriter = new OutputStreamWriter(clientSocket.getOutputStream());
    outWriter.write("GET " + file + " HTTP/1.0\r\n\n");
    outWriter.flush();
    BufferedWriter out = new BufferedWriter(new FileWriter(file));
    boolean more = true;
    String input;
    while (more) {
      input = inFromServer.readLine();
      if (input == null)
        more = false;
      else {
        out.write(input);
      }
    }
    out.close();
    clientSocket.close();
  }
}

Http connection Utilities

 
/**********************************************************************************
 *
 * Copyright (c) 2003, 2004 The Regents of the University of Michigan, Trustees of Indiana University,
 *                  Board of Trustees of the Leland Stanford, Jr., University, and The MIT Corporation
 *
 * Licensed under the Educational Community License Version 1.0 (the "License");
 * By obtaining, using and/or copying this Original Work, you agree that you have read,
 * understand, and will comply with the terms and conditions of the Educational Community License.
 * You may obtain a copy of the License at:
 *
 *      http://cvs.sakaiproject.org/licenses/license_1_0.html
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE
 * AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **********************************************************************************/
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
/**
 * HTTP utilites
 */
public class HttpTransactionUtils {
  private HttpTransactionUtils() {
  }
  /**
   * Default HTTP character set
   */
  public static final String DEFAULTCS = "ISO-8859-1";
  /*
   * Parameter handling
   */
  /**
   * Format one HTTP parameter
   * 
   * @param name
   *          Parameter name
   * @param value
   *          Parameter value (URLEncoded using default chracter set)
   * @return Parameter text (ampersand+name=url-encoded-value)
   */
  public static String formatParameter(String name, String value) {
    return formatParameter(name, value, "&", DEFAULTCS);
  }
  /**
   * Format one HTTP parameter
   * 
   * @param name
   *          Parameter name
   * @param value
   *          Parameter value (will be URLEncoded)
   * @param separator
   *          Character to separate parameters
   * @param cs
   *          Character set specification (utf-8, etc)
   * @return Parameter text (separator+name=url-encoded-value)
   */
  public static String formatParameter(String name, String value, String separator, String cs) {
    StringBuilder parameter = new StringBuilder();
    parameter.append(separator);
    parameter.append(name);
    parameter.append("=");
    try {
      parameter.append(URLEncoder.encode(value, cs));
    } catch (UnsupportedEncodingException exception) {
      throw new IllegalArgumentException("Invalid character set: \"" + cs + "\"");
    }
    return parameter.toString();
  }
  /*
   * HTTP status values
   */
  /**
   * Informational status?
   * 
   * @return true if so
   */
  public static boolean isHttpInfo(int status) {
    return ((status / 100) == 1);
  }
  /**
   * HTTP redirect?
   * 
   * @return true if so
   */
  public static boolean isHttpRedirect(int status) {
    return ((status / 100) == 3);
  }
  /**
   * Success status?
   * 
   * @return true if so
   */
  public static boolean isHttpSuccess(int status) {
    return ((status / 100) == 2);
  }
  /**
   * Error in request?
   * 
   * @return true if so
   */
  public static boolean isHttpRequestError(int status) {
    return ((status / 100) == 4);
  }
  /**
   * Server error?
   * 
   * @return true if so
   */
  public static boolean isHttpServerError(int status) {
    return ((status / 100) == 5);
  }
  /**
   * General "did an error occur"?
   * 
   * @return true if so
   */
  public static boolean isHttpError(int status) {
    return isHttpRequestError(status) || isHttpServerError(status);
  }
  /**
   * Set up a simple Map of HTTP request parameters (assumes no duplicate names)
   * 
   * @param request
   *          HttpServletRequest object
   * @return Map of name=value pairs
   */
  public static Map getAttributesAsMap(HttpServletRequest request) {
    Enumeration enumeration = request.getParameterNames();
    HashMap map = new HashMap();
    while (enumeration.hasMoreElements()) {
      String name = (String) enumeration.nextElement();
      map.put(name, request.getParameter(name));
    }
    return map;
  }
  /**
   * Format a base URL string ( protocol://server[:port] )
   * 
   * @param url
   *          URL to format
   * @return URL string
   */
  public static String formatUrl(URL url) throws MalformedURLException {
    return formatUrl(url, false);
  }
  /**
   * Format a base URL string ( protocol://server[:port][/file-specification] )
   * 
   * @param url
   *          URL to format
   * @param preserveFile
   *          Keep the /directory/filename portion of the URL?
   * @return URL string
   */
  public static String formatUrl(URL url, boolean preserveFile) throws MalformedURLException {
    StringBuilder result;
    int port;
    result = new StringBuilder(url.getProtocol());
    result.append("://");
    result.append(url.getHost());
    if ((port = url.getPort()) != -1) {
      result.append(":");
      result.append(String.valueOf(port));
    }
    if (preserveFile) {
      String file = url.getFile();
      if (file != null) {
        result.append(file);
      }
    }
    return result.toString();
  }
  /**
   * Pull the server [and port] from a URL specification
   * 
   * @param url
   *          URL string
   * @return server[:port]
   */
  public static String getServer(String url) {
    String server = url;
    int protocol, slash;
    if ((protocol = server.indexOf("//")) != -1) {
      if ((slash = server.substring(protocol + 2).indexOf("/")) != -1) {
        server = server.substring(0, protocol + 2 + slash);
      }
    }
    return server;
  }
  /*
   * urlEncodeParameters(): URL component specifications
   */
  /**
   * protocol://server
   */
  public static final String SERVER = "server";
  /**
   * /file/specification
   */
  public static final String FILE = "file";
  /**
   * ?parameter1=value1&parameter2=value2
   */
  public static final String PARAMETERS = "parameters";
  /**
   * /file/specification?parameter1=value1&parameter2=value2
   */
  public static final String FILEANDPARAMS = "fileandparameters";
  /**
   * Fetch a component from a URL string
   * 
   * @param url
   *          URL String
   * @param component
   *          name (one of server, file, parameters, fileandparameters)
   * @return URL component string (null if none)
   */
  public static String getUrlComponent(String url, String component) throws MalformedURLException {
    String file;
    int index;
    if (component.equalsIgnoreCase(SERVER)) {
      return getServer(url);
    }
    if (!component.equalsIgnoreCase(FILE) && !component.equalsIgnoreCase(PARAMETERS)
        && !component.equalsIgnoreCase(FILEANDPARAMS)) {
      throw new IllegalArgumentException(component);
    }
    file = new URL(url).getFile();
    if (file == null) {
      return null;
    }
    /*
     * Fetch file and parameters?
     */
    if (component.equalsIgnoreCase(FILEANDPARAMS)) {
      return file;
    }
    /*
     * File portion only?
     */
    index = file.indexOf("?");
    if (component.equalsIgnoreCase(FILE)) {
      switch (index) {
      case -1: // No parameters
        return file;
      case 0: // Only parameters (no file)
        return null;
      default:
        return file.substring(0, index);
      }
    }
    /*
     * Isolate parameters
     */
    return (index == -1) ? null : file.substring(index);
  }
  /**
   * URLEncode parameter names and values
   * 
   * @param original
   *          Original parameter list (?a=b&c=d)
   * @return Possibly encoded parameter list
   */
  public static String urlEncodeParameters(String original) {
    StringBuilder encoded = new StringBuilder();
    for (int i = 0; i < original.length(); i++) {
      String c = original.substring(i, i + 1);
      if (!c.equals("&") && !c.equals("=") && !c.equals("?")) {
        c = URLEncoder.encode(c);
      }
      encoded.append(c);
    }
    return encoded.toString();
  }
  /*
   * Test
   */
  public static void main(String[] args) throws Exception {
    String u = "http://example.ru/dir1/dir2/file.html?parm1=1&param2=2";
    System.out.println("Server: " + getUrlComponent(u, "server"));
    System.out.println("File: " + getUrlComponent(u, "file"));
    System.out.println("Parameters: " + getUrlComponent(u, "parameters"));
    System.out.println("File & Parameters: " + getUrlComponent(u, "fileandparameters"));
    System.out.println("Bad: " + getUrlComponent(u, "bad"));
  }
}

Http Header Helper

  
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public final class HttpHeaderHelper {
    public static final String ACCEPT_ENCODING = "Accept-Encoding";
    public static final String CONTENT_TYPE = "Content-Type";
    public static final String CONTENT_ID = "Content-ID";
    public static final String CONTENT_ENCODING = "Content-Encoding";
    public static final String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding";
    public static final String COOKIE = "Cookie";
    public static final String TRANSFER_ENCODING = "Transfer-Encoding";
    public static final String CHUNKED = "chunked";
    public static final String CONNECTION = "Connection";
    public static final String CLOSE = "close";
    public static final String AUTHORIZATION = "Authorization";
    private static final Charset UTF8 = Charset.forName("utf-8"); 
    
    private static Map<String, String> internalHeaders = new HashMap<String, String>();
    private static Map<String, String> encodings = new ConcurrentHashMap<String, String>();
    
    static {
        internalHeaders.put("Accept-Encoding", "accept-encoding");
        internalHeaders.put("Content-Encoding", "content-encoding");
        internalHeaders.put("Content-Type", "content-type");
        internalHeaders.put("Content-ID", "content-id");
        internalHeaders.put("Content-Transfer-Encoding", "content-transfer-encoding"); 
        internalHeaders.put("Transfer-Encoding", "transfer-encoding");
        internalHeaders.put("Connection", "connection");
        internalHeaders.put("authorization", "Authorization");
        internalHeaders.put("soapaction", "SOAPAction");
        internalHeaders.put("accept", "Accept");
    }
    
    private HttpHeaderHelper() {
        
    }
    
    public static List getHeader(Map<String, List<String>> headerMap, String key) {
        return headerMap.get(getHeaderKey(key));
    }
    
    public static String getHeaderKey(String key) {
        if (internalHeaders.containsKey(key)) {
            return internalHeaders.get(key);
        } else {
            return key;
        }
    }
    
    //helper to map the charsets that various things send in the http Content-Type header 
    //into something that is actually supported by Java and the Stax parsers and such.
    public static String mapCharset(String enc) {
        if (enc == null) {
            return UTF8.name();
        }
        //older versions of tomcat don"t properly parse ContentType headers with stuff
        //after charset="UTF-8"
        int idx = enc.indexOf(";");
        if (idx != -1) {
            enc = enc.substring(0, idx);
        }
        // Charsets can be quoted. But it"s quite certain that they can"t have escaped quoted or
        // anything like that.
        enc = enc.replace("\"", "").trim();
        enc = enc.replace(""", "");
        if ("".equals(enc)) {
            return UTF8.name();
        }
        String newenc = encodings.get(enc);
        if (newenc == null) {
            try {
                newenc = Charset.forName(enc).name();
            } catch (IllegalCharsetNameException icne) {
                return null;
            } catch (UnsupportedCharsetException uce) {
                return null;
            }
            encodings.put(enc, newenc);
        }
        return newenc;
    }
}

java.net.Authenticator can be used to send the credentials when needed

    

import java.io.DataInputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.util.Properties;
public class Main {
  public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");
    Authenticator.setDefault(new Authenticator() {
      protected PasswordAuthentication getPasswordAuthentication() {
        return new PasswordAuthentication("mydomain\\username", "password".toCharArray());
      }
    });
    URL u = new URL("http://www.google.ru");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
      System.out.print(new String(b));
    }
  }
}

Preventing Automatic Redirects in a HTTP Connection

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    HttpURLConnection.setFollowRedirects(false);
    URL url = new URL("http://hostname:80");
    URLConnection conn = url.openConnection();
    HttpURLConnection httpConn = (HttpURLConnection) conn;
    httpConn.setInstanceFollowRedirects(false);
    conn.connect();
  }
}

Reading from a URLConnection

    
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] args) throws Exception {
    URL yahoo = new URL("http://www.yahoo.ru/");
    URLConnection yc = yahoo.openConnection();
    BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
    String inputLine;
    while ((inputLine = in.readLine()) != null)
      System.out.println(inputLine);
    in.close();
  }
}

Sending a Cookie to an HTTP Server

    
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("http://hostname:80");
    URLConnection conn = url.openConnection();
    conn.setRequestProperty("Cookie", "name1=value1; name2=value2");
    conn.connect();
  }
}

Show the content length

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    int len = httpCon.getContentLength();
    if (len == -1)
      System.out.println("Content length unavailable.");
    else
      System.out.println("Content-Length: " + len);
 }
}

Show the content type

    
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    System.out.println("Content-Type: " + httpCon.getContentType());
 }
}

Use BufferedReader to read content from a URL

    
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("http://www.java.ru");
    URLConnection urlConnection = url.openConnection();
    HttpURLConnection connection = null;
    if (urlConnection instanceof HttpURLConnection) {
      connection = (HttpURLConnection) urlConnection;
    } else {
      System.out.println("Please enter an HTTP URL.");
      return;
    }
    BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
    String urlString = "";
    String current;
    while ((current = in.readLine()) != null) {
      urlString += current;
    }
    System.out.println(urlString);
  }
}