Java Tutorial/Network/HttpURLConnection

A utility class for parsing HTTP dates as used in cookies and other headers

/*
 * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/util/DateParser.java,v 1.11 2004/11/06 19:15:42 mbecke Exp $
 * $Revision: 480424 $
 * $Date: 2006-11-29 06:56:49 +0100 (Wed, 29 Nov 2006) $
 *
 * 
 *
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 * 
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 *
 */
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;
import java.util.TimeZone;
/**
 * A utility class for parsing HTTP dates as used in cookies and other headers.  
 * This class handles dates as defined by RFC 2616 section 3.3.1 as well as 
 * some other common non-standard formats.
 * 
 * @author Christopher Brown
 * @author Michael Becke
 * 
 */
public class DateParser {
    /**
     * Date format pattern used to parse HTTP date headers in RFC 1123 format.
     */
    public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss zzz";
    /**
     * Date format pattern used to parse HTTP date headers in RFC 1036 format.
     */
    public static final String PATTERN_RFC1036 = "EEEE, dd-MMM-yy HH:mm:ss zzz";
    /**
     * Date format pattern used to parse HTTP date headers in ANSI C 
     * <code>asctime()</code> format.
     */
    public static final String PATTERN_ASCTIME = "EEE MMM d HH:mm:ss yyyy";
    private static final Collection DEFAULT_PATTERNS = Arrays.asList(
            new String[] { PATTERN_ASCTIME, PATTERN_RFC1036, PATTERN_RFC1123 } );
    /**
     * Parses a date value.  The formats used for parsing the date value are retrieved from
     * the default http params.
     *
     * @param dateValue the date value to parse
     * 
     * @return the parsed date
     *
     * @throws DateParseException if the value could not be parsed using any of the 
     * supported date formats
     */
    public static Date parseDate(String dateValue){
        return parseDate(dateValue, null);
    }
    
    /**
     * Parses the date value using the given date formats.
     * 
     * @param dateValue the date value to parse
     * @param dateFormats the date formats to use
     * 
     * @return the parsed date
     * 
     * @throws DateParseException if none of the dataFormats could parse the dateValue
     */
    public static Date parseDate(
        String dateValue, 
        Collection dateFormats
    ) {
        
        if (dateValue == null) {
            throw new IllegalArgumentException("dateValue is null");
        }
        if (dateFormats == null) {
            dateFormats = DEFAULT_PATTERNS;
        }
        // trim single quotes around date if present
        // see issue #5279
        if (dateValue.length() > 1 
            && dateValue.startsWith(""") 
            && dateValue.endsWith(""")
        ) {
            dateValue = dateValue.substring (1, dateValue.length() - 1);
        }
        
        SimpleDateFormat dateParser = null;        
        Iterator formatIter = dateFormats.iterator();
        
        while (formatIter.hasNext()) {
            String format = (String) formatIter.next();            
            if (dateParser == null) {
                dateParser = new SimpleDateFormat(format, Locale.US);
                dateParser.setTimeZone(TimeZone.getTimeZone("GMT"));
            } else {
                dateParser.applyPattern(format);                    
            }
            try {
                return dateParser.parse(dateValue);
            } catch (ParseException pe) {
                // ignore this exception, we will try the next format
            }                
        }
        
        // we were unable to parse the date
        throw new RuntimeException("Unable to parse the date " + dateValue);        
    }
    /** This class should not be instantiated. */    
    private DateParser() { }
    
}

A Web Page Source Viewer

import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
public class Main extends JFrame {
  JButton go = new JButton("Get Source");
  JTextArea codeArea = new JTextArea();
  public Main() {
    JPanel inputPanel = new JPanel(new BorderLayout(3, 3));
    go.addActionListener(new ActionListener() {
      public void actionPerformed(ActionEvent e) {
        try {
          URL pageURL = new URL("http://www.google.ru");
          HttpURLConnection urlConnection = (HttpURLConnection) pageURL.openConnection();
          int respCode = urlConnection.getResponseCode();
          String response = urlConnection.getResponseMessage();
          codeArea.setText("HTTP/1.x " + respCode + " " + response + "\n");
          int count = 1;
          while (true) {
            String header = urlConnection.getHeaderField(count);
            String key = urlConnection.getHeaderFieldKey(count);
            if (header == null || key == null) {
              break;
            }
            codeArea.append(urlConnection.getHeaderFieldKey(count) + ": " + header + "\n");
            count++;
          }
          InputStream in = new BufferedInputStream(urlConnection.getInputStream());
          Reader r = new InputStreamReader(in);
          int c;
          while ((c = r.read()) != -1) {
            codeArea.append(String.valueOf((char) c));
          }
          codeArea.setCaretPosition(1);
        } catch (Exception ee) {
        }
      }
    });
    inputPanel.add(BorderLayout.EAST, go);
    JScrollPane codeScroller = new JScrollPane(codeArea);
    add(BorderLayout.NORTH, inputPanel);
    add(BorderLayout.CENTER, codeScroller);
    this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    this.setSize(700, 500);
    this.setVisible(true);
  }
  public static void main(String[] args) {
    JFrame webPageSourceViewer = new Main();
  }
}

Check if a page exists

import java.net.HttpURLConnection;
import java.net.URL;
public class Main {
  public static void main(String[] argv) throws Exception {
    HttpURLConnection.setFollowRedirects(false);
    HttpURLConnection con = (HttpURLConnection) new URL("http://www.google.coom").openConnection();
    con.setRequestMethod("HEAD");
    System.out.println(con.getResponseCode() == HttpURLConnection.HTTP_OK);
  }
}

Connect through a Proxy

import java.io.DataInputStream;
import java.io.FileOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Properties;
import sun.misc.BASE64Encoder;
public class Main {
  public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");
    URL u = new URL("http://www.google.ru");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    BASE64Encoder encoder = new BASE64Encoder();
    String encodedUserPwd = encoder.encode("mydomain\\MYUSER:MYPASSWORD".getBytes());
    con.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd);
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
      System.out.print(new String(b));
    }
  }
}

Display header information

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    Map<String, List<String>> hdrs = httpCon.getHeaderFields();
    Set<String> hdrKeys = hdrs.keySet();
    for (String k : hdrKeys)
      System.out.println("Key: " + k + "  Value: " + hdrs.get(k));
 }
}

Display request method

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    System.out.println("Request method is " + httpCon.getRequestMethod());
 }
}

Display response message

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    System.out.println("Response Message is " + httpCon.getResponseMessage());
 }
}

Download and display the content

import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
public class Main {
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    InputStream inStrm = httpCon.getInputStream();
    System.out.println("\nContent at " + url);
    int ch;
    while (((ch = inStrm.read()) != -1))
      System.out.print((char) ch);
    inStrm.close();
  }
}

Dump a page using the HTTPS protocol

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("https://www.server.ru");
    HttpURLConnection con = (HttpURLConnection) url.openConnection();
    BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
    String inputLine;
    while ((inputLine = in.readLine()) != null)
      System.out.println(inputLine);
    in.close();
  }
}

Get response code

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    System.out.println("Response code is " + httpCon.getResponseCode());
 }
}

Get the date of a url connection

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getDate();
    if (date == 0)
      System.out.println("No date information.");
    else
      System.out.println("Date: " + new Date(date));
 }
}

Get the document expiration date

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getExpiration();
    if (date == 0)
      System.out.println("No expiration information.");
    else
      System.out.println("Expires: " + new Date(date));
 }
}

Get the document Last-modified date

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
    long date = httpCon.getLastModified();
    if (date == 0)
      System.out.println("No last-modified information.");
    else
      System.out.println("Last-Modified: " + new Date(date));
 }
}

Http connection Utilities

/**********************************************************************************
 *
 * Copyright (c) 2003, 2004 The Regents of the University of Michigan, Trustees of Indiana University,
 *                  Board of Trustees of the Leland Stanford, Jr., University, and The MIT Corporation
 *
 * Licensed under the Educational Community License Version 1.0 (the "License");
 * By obtaining, using and/or copying this Original Work, you agree that you have read,
 * understand, and will comply with the terms and conditions of the Educational Community License.
 * You may obtain a copy of the License at:
 *
 *      http://cvs.sakaiproject.org/licenses/license_1_0.html
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE
 * AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **********************************************************************************/
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
/**
 * HTTP utilites
 */
public class HttpTransactionUtils {
  private HttpTransactionUtils() {
  }
  /**
   * Default HTTP character set
   */
  public static final String DEFAULTCS = "ISO-8859-1";
  /*
   * Parameter handling
   */
  /**
   * Format one HTTP parameter
   * 
   * @param name
   *          Parameter name
   * @param value
   *          Parameter value (URLEncoded using default chracter set)
   * @return Parameter text (ampersand+name=url-encoded-value)
   */
  public static String formatParameter(String name, String value) {
    return formatParameter(name, value, "&", DEFAULTCS);
  }
  /**
   * Format one HTTP parameter
   * 
   * @param name
   *          Parameter name
   * @param value
   *          Parameter value (will be URLEncoded)
   * @param separator
   *          Character to separate parameters
   * @param cs
   *          Character set specification (utf-8, etc)
   * @return Parameter text (separator+name=url-encoded-value)
   */
  public static String formatParameter(String name, String value, String separator, String cs) {
    StringBuilder parameter = new StringBuilder();
    parameter.append(separator);
    parameter.append(name);
    parameter.append("=");
    try {
      parameter.append(URLEncoder.encode(value, cs));
    } catch (UnsupportedEncodingException exception) {
      throw new IllegalArgumentException("Invalid character set: \"" + cs + "\"");
    }
    return parameter.toString();
  }
  /*
   * HTTP status values
   */
  /**
   * Informational status?
   * 
   * @return true if so
   */
  public static boolean isHttpInfo(int status) {
    return ((status / 100) == 1);
  }
  /**
   * HTTP redirect?
   * 
   * @return true if so
   */
  public static boolean isHttpRedirect(int status) {
    return ((status / 100) == 3);
  }
  /**
   * Success status?
   * 
   * @return true if so
   */
  public static boolean isHttpSuccess(int status) {
    return ((status / 100) == 2);
  }
  /**
   * Error in request?
   * 
   * @return true if so
   */
  public static boolean isHttpRequestError(int status) {
    return ((status / 100) == 4);
  }
  /**
   * Server error?
   * 
   * @return true if so
   */
  public static boolean isHttpServerError(int status) {
    return ((status / 100) == 5);
  }
  /**
   * General "did an error occur"?
   * 
   * @return true if so
   */
  public static boolean isHttpError(int status) {
    return isHttpRequestError(status) || isHttpServerError(status);
  }
  /**
   * Set up a simple Map of HTTP request parameters (assumes no duplicate names)
   * 
   * @param request
   *          HttpServletRequest object
   * @return Map of name=value pairs
   */
  public static Map getAttributesAsMap(HttpServletRequest request) {
    Enumeration enumeration = request.getParameterNames();
    HashMap map = new HashMap();
    while (enumeration.hasMoreElements()) {
      String name = (String) enumeration.nextElement();
      map.put(name, request.getParameter(name));
    }
    return map;
  }
  /**
   * Format a base URL string ( protocol://server[:port] )
   * 
   * @param url
   *          URL to format
   * @return URL string
   */
  public static String formatUrl(URL url) throws MalformedURLException {
    return formatUrl(url, false);
  }
  /**
   * Format a base URL string ( protocol://server[:port][/file-specification] )
   * 
   * @param url
   *          URL to format
   * @param preserveFile
   *          Keep the /directory/filename portion of the URL?
   * @return URL string
   */
  public static String formatUrl(URL url, boolean preserveFile) throws MalformedURLException {
    StringBuilder result;
    int port;
    result = new StringBuilder(url.getProtocol());
    result.append("://");
    result.append(url.getHost());
    if ((port = url.getPort()) != -1) {
      result.append(":");
      result.append(String.valueOf(port));
    }
    if (preserveFile) {
      String file = url.getFile();
      if (file != null) {
        result.append(file);
      }
    }
    return result.toString();
  }
  /**
   * Pull the server [and port] from a URL specification
   * 
   * @param url
   *          URL string
   * @return server[:port]
   */
  public static String getServer(String url) {
    String server = url;
    int protocol, slash;
    if ((protocol = server.indexOf("//")) != -1) {
      if ((slash = server.substring(protocol + 2).indexOf("/")) != -1) {
        server = server.substring(0, protocol + 2 + slash);
      }
    }
    return server;
  }
  /*
   * urlEncodeParameters(): URL component specifications
   */
  /**
   * protocol://server
   */
  public static final String SERVER = "server";
  /**
   * /file/specification
   */
  public static final String FILE = "file";
  /**
   * ?parameter1=value1&parameter2=value2
   */
  public static final String PARAMETERS = "parameters";
  /**
   * /file/specification?parameter1=value1&parameter2=value2
   */
  public static final String FILEANDPARAMS = "fileandparameters";
  /**
   * Fetch a component from a URL string
   * 
   * @param url
   *          URL String
   * @param component
   *          name (one of server, file, parameters, fileandparameters)
   * @return URL component string (null if none)
   */
  public static String getUrlComponent(String url, String component) throws MalformedURLException {
    String file;
    int index;
    if (component.equalsIgnoreCase(SERVER)) {
      return getServer(url);
    }
    if (!component.equalsIgnoreCase(FILE) && !component.equalsIgnoreCase(PARAMETERS)
        && !component.equalsIgnoreCase(FILEANDPARAMS)) {
      throw new IllegalArgumentException(component);
    }
    file = new URL(url).getFile();
    if (file == null) {
      return null;
    }
    /*
     * Fetch file and parameters?
     */
    if (component.equalsIgnoreCase(FILEANDPARAMS)) {
      return file;
    }
    /*
     * File portion only?
     */
    index = file.indexOf("?");
    if (component.equalsIgnoreCase(FILE)) {
      switch (index) {
      case -1: // No parameters
        return file;
      case 0: // Only parameters (no file)
        return null;
      default:
        return file.substring(0, index);
      }
    }
    /*
     * Isolate parameters
     */
    return (index == -1) ? null : file.substring(index);
  }
  /**
   * URLEncode parameter names and values
   * 
   * @param original
   *          Original parameter list (?a=b&c=d)
   * @return Possibly encoded parameter list
   */
  public static String urlEncodeParameters(String original) {
    StringBuilder encoded = new StringBuilder();
    for (int i = 0; i < original.length(); i++) {
      String c = original.substring(i, i + 1);
      if (!c.equals("&") && !c.equals("=") && !c.equals("?")) {
        c = URLEncoder.encode(c);
      }
      encoded.append(c);
    }
    return encoded.toString();
  }
  /*
   * Test
   */
  public static void main(String[] args) throws Exception {
    String u = "http://example.ru/dir1/dir2/file.html?parm1=1&param2=2";
    System.out.println("Server: " + getUrlComponent(u, "server"));
    System.out.println("File: " + getUrlComponent(u, "file"));
    System.out.println("Parameters: " + getUrlComponent(u, "parameters"));
    System.out.println("File & Parameters: " + getUrlComponent(u, "fileandparameters"));
    System.out.println("Bad: " + getUrlComponent(u, "bad"));
  }
}

Http Constants

/*
 * Copyright 2007 Outerthought bvba and Schaubroeck nv
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
public class HttpConstants {
    public final static String GET = "GET",
            POST = "POST",
            HEAD = "HEAD",
            PUT = "PUT",
            OPTIONS = "OPTIONS",
            DELETE = "DELETE";
    // These are copy-pasted from the Jetty codebase
    public final static int
            _100_Continue = 100,
            _101_Switching_Protocols = 101,
            _102_Processing = 102,
            _200_OK = 200,
            _201_Created = 201,
            _202_Accepted = 202,
            _203_Non_Authoritative_Information = 203,
            _204_No_Content = 204,
            _205_Reset_Content = 205,
            _206_Partial_Content = 206,
            _207_Multi_Status = 207,
            _300_Multiple_Choices = 300,
            _301_Moved_Permanently = 301,
            _302_Moved_Temporarily = 302,
            _302_Found = 302,
            _303_See_Other = 303,
            _304_Not_Modified = 304,
            _305_Use_Proxy = 305,
            _400_Bad_Request = 400,
            _401_Unauthorized = 401,
            _402_Payment_Required = 402,
            _403_Forbidden = 403,
            _404_Not_Found = 404,
            _405_Method_Not_Allowed = 405,
            _406_Not_Acceptable = 406,
            _407_Proxy_Authentication_Required = 407,
            _408_Request_Timeout = 408,
            _409_Conflict = 409,
            _410_Gone = 410,
            _411_Length_Required = 411,
            _412_Precondition_Failed = 412,
            _413_Request_Entity_Too_Large = 413,
            _414_Request_URI_Too_Large = 414,
            _415_Unsupported_Media_Type = 415,
            _416_Requested_Range_Not_Satisfiable = 416,
            _417_Expectation_Failed = 417,
            _422_Unprocessable_Entity = 422,
            _423_Locked = 423,
            _424_Failed_Dependency = 424,
            _500_Internal_Server_Error = 500,
            _501_Not_Implemented = 501,
            _502_Bad_Gateway = 502,
            _503_Service_Unavailable = 503,
            _504_Gateway_Timeout = 504,
            _505_HTTP_Version_Not_Supported = 505,
            _507_Insufficient_Storage = 507,
            _999_Unknown = 999;
    public final static String
            MIMETYPE_TEXT_HTML = "text/html",
            MIMETYPE_TEXT_PLAIN = "text/plain",
            MIMETYPE_TEXT_XML = "text/xml",
            MIMETYPE_TEXT_HTML_8859_1 = "text/html; charset=iso-8859-1",
            MIMETYPE_TEXT_PLAIN_8859_1 = "text/plain; charset=iso-8859-1",
            MIMETYPE_TEXT_XML_8859_1 = "text/xml; charset=iso-8859-1",
            MIMETYPE_TEXT_HTML_UTF_8 = "text/html; charset=utf-8",
            MIMETYPE_TEXT_PLAIN_UTF_8 = "text/plain; charset=utf-8",
            MIMETYPE_TEXT_XML_UTF_8 = "text/xml; charset=utf-8";
}

Http Header Helper

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public final class HttpHeaderHelper {
    public static final String ACCEPT_ENCODING = "Accept-Encoding";
    public static final String CONTENT_TYPE = "Content-Type";
    public static final String CONTENT_ID = "Content-ID";
    public static final String CONTENT_ENCODING = "Content-Encoding";
    public static final String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding";
    public static final String COOKIE = "Cookie";
    public static final String TRANSFER_ENCODING = "Transfer-Encoding";
    public static final String CHUNKED = "chunked";
    public static final String CONNECTION = "Connection";
    public static final String CLOSE = "close";
    public static final String AUTHORIZATION = "Authorization";
    private static final Charset UTF8 = Charset.forName("utf-8"); 
    
    private static Map<String, String> internalHeaders = new HashMap<String, String>();
    private static Map<String, String> encodings = new ConcurrentHashMap<String, String>();
    
    static {
        internalHeaders.put("Accept-Encoding", "accept-encoding");
        internalHeaders.put("Content-Encoding", "content-encoding");
        internalHeaders.put("Content-Type", "content-type");
        internalHeaders.put("Content-ID", "content-id");
        internalHeaders.put("Content-Transfer-Encoding", "content-transfer-encoding"); 
        internalHeaders.put("Transfer-Encoding", "transfer-encoding");
        internalHeaders.put("Connection", "connection");
        internalHeaders.put("authorization", "Authorization");
        internalHeaders.put("soapaction", "SOAPAction");
        internalHeaders.put("accept", "Accept");
    }
    
    private HttpHeaderHelper() {
        
    }
    
    public static List getHeader(Map<String, List<String>> headerMap, String key) {
        return headerMap.get(getHeaderKey(key));
    }
    
    public static String getHeaderKey(String key) {
        if (internalHeaders.containsKey(key)) {
            return internalHeaders.get(key);
        } else {
            return key;
        }
    }
    
    //helper to map the charsets that various things send in the http Content-Type header 
    //into something that is actually supported by Java and the Stax parsers and such.
    public static String mapCharset(String enc) {
        if (enc == null) {
            return UTF8.name();
        }
        //older versions of tomcat don"t properly parse ContentType headers with stuff
        //after charset="UTF-8"
        int idx = enc.indexOf(";");
        if (idx != -1) {
            enc = enc.substring(0, idx);
        }
        // Charsets can be quoted. But it"s quite certain that they can"t have escaped quoted or
        // anything like that.
        enc = enc.replace("\"", "").trim();
        enc = enc.replace(""", "");
        if ("".equals(enc)) {
            return UTF8.name();
        }
        String newenc = encodings.get(enc);
        if (newenc == null) {
            try {
                newenc = Charset.forName(enc).name();
            } catch (IllegalCharsetNameException icne) {
                return null;
            } catch (UnsupportedCharsetException uce) {
                return null;
            }
            encodings.put(enc, newenc);
        }
        return newenc;
    }
}

Identify ourself to a proxy

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Properties;
import sun.misc.BASE64Encoder;
public class Main {
  public static void main(String[] argv) throws Exception {
    Properties systemSettings = System.getProperties();
    systemSettings.put("proxySet", "true");
    systemSettings.put("http.proxyHost", "proxy.mycompany.local");
    systemSettings.put("http.proxyPort", "80");
    URL u = new URL("http://www.google.ru");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    BASE64Encoder encoder = new BASE64Encoder();
    String encodedUserPwd = encoder.encode("domain\\username:password".getBytes());
    con.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd);
    con.setRequestMethod("HEAD");
    System.out.println(con.getResponseCode() + " : " + con.getResponseMessage());
    System.out.println(con.getResponseCode() == HttpURLConnection.HTTP_OK);
  }
}

java.net.Authenticator can be used to send the credentials when needed

import java.io.DataInputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.util.Properties;
public class Main {
  public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");
    Authenticator.setDefault(new Authenticator() {
      protected PasswordAuthentication getPasswordAuthentication() {
        return new PasswordAuthentication("mydomain\\username", "password".toCharArray());
      }
    });
    URL u = new URL("http://www.google.ru");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
      System.out.print(new String(b));
    }
  }
}

Last Modified

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class MainClass {
  public static void main(String args[]) throws Exception {
    URL u = new URL("http://www.jexp.ru");
    HttpURLConnection http = (HttpURLConnection) u.openConnection();
    http.setRequestMethod("HEAD");
    System.out.println(u + "was last modified at " + new Date(http.getLastModified()));
  }
}

Parsing and formatting HTTP dates as used in cookies and other headers.

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.Locale;
import java.util.TimeZone;
/**
 * A utility class for parsing and formatting HTTP dates as used in cookies and
 * other headers.  This class handles dates as defined by RFC 2616 section
 * 3.3.1 as well as some other common non-standard formats.
 *
 * @author Christopher Brown
 * @author Michael Becke
 */
public final class DateUtil {
    /**
     * Date format pattern used to parse HTTP date headers in RFC 1123 format.
     */
    public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss zzz";
    /**
     * Date format pattern used to parse HTTP date headers in RFC 1036 format.
     */
    public static final String PATTERN_RFC1036 = "EEEE, dd-MMM-yy HH:mm:ss zzz";
    /**
     * Date format pattern used to parse HTTP date headers in ANSI C
     * <code>asctime()</code> format.
     */
    public static final String PATTERN_ASCTIME = "EEE MMM d HH:mm:ss yyyy";
    private static final Collection<String> DEFAULT_PATTERNS = Arrays.asList(
        new String[] {PATTERN_ASCTIME, PATTERN_RFC1036, PATTERN_RFC1123});
    private static final Date DEFAULT_TWO_DIGIT_YEAR_START;
    static {
        Calendar calendar = Calendar.getInstance();
        calendar.set(2000, Calendar.JANUARY, 1, 0, 0);
        DEFAULT_TWO_DIGIT_YEAR_START = calendar.getTime();
    }
    private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
    /**
     * This class should not be instantiated.
     */
    private DateUtil() {
    }

    /**
     * Parses a date value.  The formats used for parsing the date value are retrieved from
     * the default http params.
     *
     * @param dateValue the date value to parse
     * @return the parsed date
     * @throws DateParseException if the value could not be parsed using any of the
     *                            supported date formats
     */
    public static Date parseDate(String dateValue) {
        return parseDate(dateValue, null, null);
    }
    /**
     * Parses the date value using the given date formats.
     *
     * @param dateValue   the date value to parse
     * @param dateFormats the date formats to use
     * @return the parsed date
     * @throws DateParseException if none of the dataFormats could parse the dateValue
     */
    public static Date parseDate(String dateValue, Collection<String> dateFormats)
        {
        return parseDate(dateValue, dateFormats, null);
    }
    /**
     * Parses the date value using the given date formats.
     *
     * @param dateValue   the date value to parse
     * @param dateFormats the date formats to use
     * @param startDate   During parsing, two digit years will be placed in the range
     *                    <code>startDate</code> to <code>startDate + 100 years</code>. This value may
     *                    be <code>null</code>. When <code>null</code> is given as a parameter, year
     *                    <code>2000</code> will be used.
     * @return the parsed date
     * @throws DateParseException if none of the dataFormats could parse the dateValue
     */
    public static Date parseDate(
        String dateValue,
        Collection<String> dateFormats,
        Date startDate
    )  {
        if (dateValue == null) {
            throw new IllegalArgumentException("dateValue is null");
        }
        if (dateFormats == null) {
            dateFormats = DEFAULT_PATTERNS;
        }
        if (startDate == null) {
            startDate = DEFAULT_TWO_DIGIT_YEAR_START;
        }
        // trim single quotes around date if present
        // see issue #5279
        if (dateValue.length() > 1
            && dateValue.startsWith(""")
            && dateValue.endsWith(""")
            ) {
            dateValue = dateValue.substring(1, dateValue.length() - 1);
        }
        SimpleDateFormat dateParser = null;
        for (String format : dateFormats) {
            if (dateParser == null) {
                dateParser = new SimpleDateFormat(format, Locale.US);
                dateParser.setTimeZone(TimeZone.getTimeZone("GMT"));
                dateParser.set2DigitYearStart(startDate);
            } else {
                dateParser.applyPattern(format);
            }
            try {
                return dateParser.parse(dateValue);
            } catch (ParseException pe) {
                // ignore this exception, we will try the next format
            }
        }
        // we were unable to parse the date
        throw new RuntimeException("Unable to parse the date " + dateValue);
    }
    /**
     * Formats the given date according to the RFC 1123 pattern.
     *
     * @param date The date to format.
     * @return An RFC 1123 formatted date string.
     * @see #PATTERN_RFC1123
     */
    public static String formatDate(Date date) {
        return formatDate(date, PATTERN_RFC1123);
    }
    /**
     * Formats the given date according to the specified pattern.  The pattern
     * must conform to that used by the {@link SimpleDateFormat simple date
     * format} class.
     *
     * @param date    The date to format.
     * @param pattern The pattern to use for formatting the date.
     * @return A formatted date string.
     * @throws IllegalArgumentException If the given date pattern is invalid.
     * @see SimpleDateFormat
     */
    public static String formatDate(Date date, String pattern) {
        if (date == null) {
            throw new IllegalArgumentException("date is null");
        }
        if (pattern == null) {
            throw new IllegalArgumentException("pattern is null");
        }
        SimpleDateFormat formatter = new SimpleDateFormat(pattern, Locale.US);
        formatter.setTimeZone(GMT);
        return formatter.format(date);
    }
}

Read data from a URL

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import javax.swing.JFrame;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
public class WebReader extends JFrame {
  JTextArea box = new JTextArea("Getting data ...");
  public WebReader() {
    setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    setSize(600, 300);
    JScrollPane pane = new JScrollPane(box);
    add(pane);
    setVisible(true);
  }
  void getData(String address) throws Exception {
    setTitle(address);
    URL page = new URL(address);
    StringBuffer text = new StringBuffer();
    HttpURLConnection conn = (HttpURLConnection) page.openConnection();
    conn.connect();
    InputStreamReader in = new InputStreamReader((InputStream) conn.getContent());
    BufferedReader buff = new BufferedReader(in);
    box.setText("Getting data ...");
    String line;
    do {
      line = buff.readLine();
      text.append(line + "\n");
    } while (line != null);
    box.setText(text.toString());
  }
  public static void main(String[] arguments) throws Exception {
    WebReader app = new WebReader();
    app.getData("http://jexp.ru");
  }
}

Reading from a URLConnection

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] args) throws Exception {
    URL yahoo = new URL("http://www.yahoo.ru/");
    URLConnection yc = yahoo.openConnection();
    BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
    String inputLine;
    while ((inputLine = in.readLine()) != null)
      System.out.println(inputLine);
    in.close();
  }
}

Save URL contents to a file

/** 
 * 
 * The ObjectStyle Group Software License, version 1.1
 * ObjectStyle Group - http://objectstyle.org/
 * 
 * Copyright (c) 2002-2005, Andrei (Andrus) Adamchik and individual authors
 * of the software. All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 
 * 3. The end-user documentation included with the redistribution, if any,
 *    must include the following acknowlegement:
 *    "This product includes software developed by independent contributors
 *    and hosted on ObjectStyle Group web site (http://objectstyle.org/)."
 *    Alternately, this acknowlegement may appear in the software itself,
 *    if and wherever such third-party acknowlegements normally appear.
 * 
 * 4. The names "ObjectStyle Group" and "Cayenne" must not be used to endorse
 *    or promote products derived from this software without prior written
 *    permission. For written permission, email
 *    "andrus at objectstyle dot org".
 * 
 * 5. Products derived from this software may not be called "ObjectStyle"
 *    or "Cayenne", nor may "ObjectStyle" or "Cayenne" appear in their
 *    names without prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED ``AS IS"" AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE OBJECTSTYLE GROUP OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 * 
 * This software consists of voluntary contributions made by many
 * individuals and hosted on ObjectStyle Group web site.  For more
 * information on the ObjectStyle Group, please see
 * <http://objectstyle.org/>.
 */
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.lang.reflect.Member;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.ruparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
/**
 * Contains various unorganized static utility methods used across Cayenne.
 * 
 * @author Andrei Adamchik
 */
public class Util {
  /**
   * Copies file contents from source to destination. Makes up for the lack of file
   * copying utilities in Java
   */
  public static boolean copy(File source, File destination) {
      BufferedInputStream fin = null;
      BufferedOutputStream fout = null;
      try {
          int bufSize = 8 * 1024;
          fin = new BufferedInputStream(new FileInputStream(source), bufSize);
          fout = new BufferedOutputStream(new FileOutputStream(destination), bufSize);
          copyPipe(fin, fout, bufSize);
      }
      catch (IOException ioex) {
          return false;
      }
      catch (SecurityException sx) {
          return false;
      }
      finally {
          if (fin != null) {
              try {
                  fin.close();
              }
              catch (IOException cioex) {
              }
          }
          if (fout != null) {
              try {
                  fout.close();
              }
              catch (IOException cioex) {
              }
          }
      }
      return true;
  }
  /**
   * Save URL contents to a file.
   */
  public static boolean copy(URL from, File to) {
      BufferedInputStream urlin = null;
      BufferedOutputStream fout = null;
      try {
          int bufSize = 8 * 1024;
          urlin = new BufferedInputStream(
                  from.openConnection().getInputStream(),
                  bufSize);
          fout = new BufferedOutputStream(new FileOutputStream(to), bufSize);
          copyPipe(urlin, fout, bufSize);
      }
      catch (IOException ioex) {
          return false;
      }
      catch (SecurityException sx) {
          return false;
      }
      finally {
          if (urlin != null) {
              try {
                  urlin.close();
              }
              catch (IOException cioex) {
              }
          }
          if (fout != null) {
              try {
                  fout.close();
              }
              catch (IOException cioex) {
              }
          }
      }
      return true;
  }
  /**
   * Reads data from the input and writes it to the output, until the end of the input
   * stream.
   * 
   * @param in
   * @param out
   * @param bufSizeHint
   * @throws IOException
   */
  public static void copyPipe(InputStream in, OutputStream out, int bufSizeHint)
          throws IOException {
      int read = -1;
      byte[] buf = new byte[bufSizeHint];
      while ((read = in.read(buf, 0, bufSizeHint)) >= 0) {
          out.write(buf, 0, read);
      }
      out.flush();
  }
}

Show the content length

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    int len = httpCon.getContentLength();
    if (len == -1)
      System.out.println("Content length unavailable.");
    else
      System.out.println("Content-Length: " + len);
 }
}

Show the content type

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
public class Main{
  public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.ru");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    System.out.println("Content-Type: " + httpCon.getContentType());
 }
}

URL connection and proxy

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Properties;
public class Main{
  public static void main(String s[]) throws Exception{
    try {
      Properties systemSettings = System.getProperties();
      systemSettings.put("proxySet", "true");
      systemSettings.put("http.proxyHost", "proxy.mycompany.local");
      systemSettings.put("http.proxyPort", "80");
      URL u = new URL("http://www.java.ru");
      HttpURLConnection con = (HttpURLConnection) u.openConnection();
      sun.misc.BASE64Encoder encoder = new sun.misc.BASE64Encoder();
      String encodedUserPwd = encoder.encode("domain\\username:password".getBytes());
      con.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd);
      con.setRequestMethod("HEAD");
      System.out.println(con.getResponseCode() + " : " + con.getResponseMessage());
      System.out.println(con.getResponseCode() == HttpURLConnection.HTTP_OK);
    } catch (Exception e) {
      e.printStackTrace();
      System.out.println(false);
    }
  }
}

Use BufferedReader to read content from a URL

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
public class Main {
  public static void main(String[] argv) throws Exception {
    URL url = new URL("http://www.java.ru");
    URLConnection urlConnection = url.openConnection();
    HttpURLConnection connection = null;
    if (urlConnection instanceof HttpURLConnection) {
      connection = (HttpURLConnection) urlConnection;
    } else {
      System.out.println("Please enter an HTTP URL.");
      return;
    }
    BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
    String urlString = "";
    String current;
    while ((current = in.readLine()) != null) {
      urlString += current;
    }
    System.out.println(urlString);
  }
}