cdm/src/main/java/ucar/nc2/util/IO.java
static public long copyUrlB(String urlString, OutputStream out, int bufferSize) throws IOException {
    long count;
    URL url;

    try {
      url = new URL(urlString);
    } catch (MalformedURLException e) {
      throw new IOException("** MalformedURLException on URL <" + urlString + ">\n" + e.getMessage() + "\n");
    }

    try {
      java.net.URLConnection connection = url.openConnection();
      java.net.HttpURLConnection httpConnection = null;
      if (connection instanceof java.net.HttpURLConnection) {
        httpConnection = (java.net.HttpURLConnection) connection;
        httpConnection.addRequestProperty("Accept-Encoding", "gzip");
      }

      if (showHeaders) {
        showRequestHeaders(urlString, connection);
      }

      // get response
      if (httpConnection != null) {
        int responseCode = httpConnection.getResponseCode();
        if (responseCode / 100 != 2)
          throw new IOException("** Cant open URL <" + urlString + ">\n Response code = " + responseCode
              + "\n" + httpConnection.getResponseMessage() + "\n");
      }

      if (showHeaders && (httpConnection != null)) {
        int code = httpConnection.getResponseCode();
        String response = httpConnection.getResponseMessage();

        // response headers
        System.out.println("\nRESPONSE for " + urlString + ": ");
        System.out.println(" HTTP/1.x " + code + " " + response);
        System.out.println("Headers: ");

        for (int j = 1; ; j++) {
          String header = connection.getHeaderField(j);
          String key = connection.getHeaderFieldKey(j);
          if (header == null || key == null) break;
          System.out.println(" " + key + ": " + header);
        }
      }

      // read it
      try (InputStream is = connection.getInputStream()) {
        BufferedInputStream bis;

        // check if its gzipped
        if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
          bis = new BufferedInputStream(new GZIPInputStream(is), 8000);
        } else {
          bis = new BufferedInputStream(is, 8000);
        }

        if (out == null)
          count = IO.copy2null(bis, bufferSize);
        else
          count = IO.copyB(bis, out, bufferSize);
      }

     } catch (java.net.ConnectException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** ConnectException on URL: <" + urlString + ">\n" +
          e.getMessage() + "\nServer probably not running");

     } catch (java.net.UnknownHostException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** UnknownHostException on URL: <" + urlString + ">\n");

     } catch (Exception e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** Exception on URL: <" + urlString + ">\n" + e);
     }

    return count;
  }
static public long copyUrlB(String urlString, OutputStream out, int bufferSize) throws IOException {
    long count;
    URL url;

    try {
      url = new URL(urlString);
    } catch (MalformedURLException e) {
      throw new IOException("** MalformedURLException on URL <" + urlString + ">\n" + e.getMessage() + "\n");
    }

    try {
      java.net.URLConnection connection = url.openConnection();
      java.net.HttpURLConnection httpConnection = null;
      if (connection instanceof java.net.HttpURLConnection) {
        httpConnection = (java.net.HttpURLConnection) connection;
        httpConnection.addRequestProperty("Accept-Encoding", "gzip");
      }

      if (showHeaders) {
        showRequestHeaders(urlString, connection);
      }

      // get response
      if (httpConnection != null) {
        int responseCode = httpConnection.getResponseCode();
        if (responseCode / 100 != 2)
          throw new IOException("** Cant open URL <" + urlString + ">\n Response code = " + responseCode
              + "\n" + httpConnection.getResponseMessage() + "\n");
      }

      if (showHeaders && (httpConnection != null)) {
        int code = httpConnection.getResponseCode();
        String response = httpConnection.getResponseMessage();

        // response headers
        System.out.println("\nRESPONSE for " + urlString + ": ");
        System.out.println(" HTTP/1.x " + code + " " + response);
        System.out.println("Headers: ");

        for (int j = 1; ; j++) {
          String header = connection.getHeaderField(j);
          String key = connection.getHeaderFieldKey(j);
          if (header == null || key == null) break;
          System.out.println(" " + key + ": " + header);
        }
      }

      // read it
      try (InputStream is = connection.getInputStream()) {
        BufferedInputStream bis;

        // check if its gzipped
        if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
          bis = new BufferedInputStream(new GZIPInputStream(is), 8000);
        } else {
          bis = new BufferedInputStream(is, 8000);
        }

        if (out == null)
          count = IO.copy2null(bis, bufferSize);
        else
          count = IO.copyB(bis, out, bufferSize);
      }

     } catch (java.net.ConnectException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** ConnectException on URL: <" + urlString + ">\n" +
          e.getMessage() + "\nServer probably not running");

     } catch (java.net.UnknownHostException e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** UnknownHostException on URL: <" + urlString + ">\n");

     } catch (Exception e) {
       if (showStackTrace) e.printStackTrace();
       throw new IOException("** Exception on URL: <" + urlString + ">\n" + e);
     }

    return count;
  }
static public long copyUrlB ( String urlString , OutputStream out , int bufferSize ) throws IOException { long count ; URL url ; try { url = new URL ( urlString ) ; } catch ( MalformedURLException e ) { throw new IOException ( "** MalformedURLException on URL <" + urlString + ">\n" + e . getMessage ( ) + "\n" ) ; } try { java . net . URLConnection connection = url . openConnection ( ) ; java . net . HttpURLConnection httpConnection = null ; if ( connection instanceof java . net . HttpURLConnection ) { httpConnection = ( java . net . HttpURLConnection ) connection ; httpConnection . addRequestProperty ( "Accept-Encoding" , "gzip" ) ; } if ( showHeaders ) { showRequestHeaders ( urlString , connection ) ; } // get response
 if ( httpConnection != null ) { int responseCode = httpConnection . getResponseCode ( ) ; if ( responseCode / 100 != 2 ) throw new IOException ( "** Cant open URL <" + urlString + ">\n Response code = " + responseCode + "\n" + httpConnection . getResponseMessage ( ) + "\n" ) ; } if ( showHeaders && ( httpConnection != null ) ) { int code = httpConnection . getResponseCode ( ) ; String response = httpConnection . getResponseMessage ( ) ; // response headers
 System . out . println ( "\nRESPONSE for " + urlString + ": " ) ; System . out . println ( " HTTP/1.x " + code + " " + response ) ; System . out . println ( "Headers: " ) ; for ( int j = 1 ; ; j ++ ) { String header = connection . getHeaderField ( j ) ; String key = connection . getHeaderFieldKey ( j ) ; if ( header == null || key == null ) break ; System . out . println ( " " + key + ": " + header ) ; } } // read it
 try ( InputStream is = connection . getInputStream ( ) ) { BufferedInputStream bis ; // check if its gzipped
 if ( "gzip" . equalsIgnoreCase ( connection . getContentEncoding ( ) ) ) { bis = new BufferedInputStream ( new GZIPInputStream ( is ) , 8000 ) ; } else { bis = new BufferedInputStream ( is , 8000 ) ; } if ( out == null ) count = IO . copy2null ( bis , bufferSize ) ; else count = IO . copyB ( bis , out , bufferSize ) ; } } catch ( java . net . ConnectException e ) { if ( showStackTrace ) e . printStackTrace ( ) ; throw new IOException ( "** ConnectException on URL: <" + urlString + ">\n" + e . getMessage ( ) + "\nServer probably not running" ) ; } catch ( java . net . UnknownHostException e ) { if ( showStackTrace ) e . printStackTrace ( ) ; throw new IOException ( "** UnknownHostException on URL: <" + urlString + ">\n" ) ; } catch ( Exception e ) { if ( showStackTrace ) e . printStackTrace ( ) ; throw new IOException ( "** Exception on URL: <" + urlString + ">\n" + e ) ; } return count ; }
copy contents of URL to output stream, specify internal buffer size. request gzip encoding

@param urlString  copy the contents of this URL
@param out        copy to this stream. If null, throw bytes away
@param bufferSize internal buffer size.
@return number of bytes copied
@throws java.io.IOException on io error
copy contents of URL to output stream specify internal buffer size . request gzip encoding