external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchUtils.java
public static String getID(String url) {

        // the document needs an ID
        // see
        // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches
        // A unique ID for the document. A document ID can contain any
        // letter or number and the following characters: _ - = # ; : / ? @
        // &. Document IDs must be at least 1 and no more than 128
        // characters long.
        byte[] dig = digester.digest(url.getBytes(StandardCharsets.UTF_8));
        String ID = Hex.encodeHexString(dig);
        // is that even possible?
        if (ID.length() > 128) {
            throw new RuntimeException("ID larger than max 128 chars");
        }
        return ID;
    }
public static String getID(String url) {

        // the document needs an ID
        // see
        // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches
        // A unique ID for the document. A document ID can contain any
        // letter or number and the following characters: _ - = # ; : / ? @
        // &. Document IDs must be at least 1 and no more than 128
        // characters long.
        byte[] dig = digester.digest(url.getBytes(StandardCharsets.UTF_8));
        String ID = Hex.encodeHexString(dig);
        // is that even possible?
        if (ID.length() > 128) {
            throw new RuntimeException("ID larger than max 128 chars");
        }
        return ID;
    }
public static String getID ( String url ) { // the document needs an ID // see // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches // A unique ID for the document. A document ID can contain any // letter or number and the following characters: _ - = # ; : / ? @ // &. Document IDs must be at least 1 and no more than 128 // characters long. byte [ ] dig = digester . digest ( url . getBytes ( StandardCharsets . UTF_8 ) ) ; String ID = Hex . encodeHexString ( dig ) ; // is that even possible? if ( ID . length ( ) > 128 ) { throw new RuntimeException ( "ID larger than max 128 chars" ) ; } return ID ; }
Returns a normalised doc ID based on the URL of a document *
Returns a normalised doc ID based on the URL of a document *