core/src/main/java/com/digitalpebble/stormcrawler/util/CharsetIdentification.java
private static String getCharsetFromMeta(byte buffer[], int maxlength) {
        // convert to UTF-8 String -- which hopefully will not mess up the
        // characters we're interested in...
        int len = buffer.length;
        if (maxlength > 0 && maxlength < len) {
            len = maxlength;
        }
        String html = new String(buffer, 0, len, DEFAULT_CHARSET);

        Document doc = Parser.htmlParser().parseInput(html, "dummy");

        // look for <meta http-equiv="Content-Type"
        // content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        Elements metaElements = doc
                .select("meta[http-equiv=content-type], meta[charset]");
        String foundCharset = null;
        for (Element meta : metaElements) {
            if (meta.hasAttr("http-equiv"))
                foundCharset = getCharsetFromContentType(meta.attr("content"));
            if (foundCharset == null && meta.hasAttr("charset"))
                foundCharset = meta.attr("charset");
            if (foundCharset != null)
                return foundCharset;
        }

        return foundCharset;
    }
private static String getCharsetFromMeta(byte buffer[], int maxlength) {
        // convert to UTF-8 String -- which hopefully will not mess up the
        // characters we're interested in...
        int len = buffer.length;
        if (maxlength > 0 && maxlength < len) {
            len = maxlength;
        }
        String html = new String(buffer, 0, len, DEFAULT_CHARSET);

        Document doc = Parser.htmlParser().parseInput(html, "dummy");

        // look for <meta http-equiv="Content-Type"
        // content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        Elements metaElements = doc
                .select("meta[http-equiv=content-type], meta[charset]");
        String foundCharset = null;
        for (Element meta : metaElements) {
            if (meta.hasAttr("http-equiv"))
                foundCharset = getCharsetFromContentType(meta.attr("content"));
            if (foundCharset == null && meta.hasAttr("charset"))
                foundCharset = meta.attr("charset");
            if (foundCharset != null)
                return foundCharset;
        }

        return foundCharset;
    }
private static String getCharsetFromMeta ( byte buffer [ ] , int maxlength ) { // convert to UTF-8 String -- which hopefully will not mess up the // characters we're interested in... int len = buffer . length ; if ( maxlength > 0 && maxlength < len ) { len = maxlength ; } String html = new String ( buffer , 0 , len , DEFAULT_CHARSET ) ; Document doc = Parser . htmlParser ( ) . parseInput ( html , "dummy" ) ; // look for <meta http-equiv="Content-Type" // content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312"> Elements metaElements = doc . select ( "meta[http-equiv=content-type], meta[charset]" ) ; String foundCharset = null ; for ( Element meta : metaElements ) { if ( meta . hasAttr ( "http-equiv" ) ) foundCharset = getCharsetFromContentType ( meta . attr ( "content" ) ) ; if ( foundCharset == null && meta . hasAttr ( "charset" ) ) foundCharset = meta . attr ( "charset" ) ; if ( foundCharset != null ) return foundCharset ; } return foundCharset ; }
Attempt to find a META tag in the HTML that hints at the character set
used to write the document.
Attempt to find a META tag in the HTML that hints at the character set used to write the document .