core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
protected String valueForURL(Tuple tuple) {

        String url = tuple.getStringByField("url");
        Metadata metadata = (Metadata) tuple.getValueByField("metadata");

        // functionality deactivated
        if (StringUtils.isBlank(canonicalMetadataParamName)) {
            return url;
        }

        String canonicalValue = metadata.getFirstValue(canonicalMetadataName);

        // no value found?
        if (StringUtils.isBlank(canonicalValue)) {
            return url;
        }

        try {
            URL sURL = new URL(url);
            URL canonical = URLUtil.resolveURL(sURL, canonicalValue);

            String sDomain = PaidLevelDomain.getPLD(sURL.getHost());
            String canonicalDomain = PaidLevelDomain
                    .getPLD(canonical.getHost());

            // check that the domain is the same
            if (sDomain.equalsIgnoreCase(canonicalDomain)) {
                return canonical.toExternalForm();
            } else {
                LOG.info(
                        "Canonical URL references a different domain, ignoring in {} ",
                        url);
            }
        } catch (MalformedURLException e) {
            LOG.error("Malformed canonical URL {} was found in {} ",
                    canonicalValue, url);
        }

        return url;
    }
protected String valueForURL(Tuple tuple) {

        String url = tuple.getStringByField("url");
        Metadata metadata = (Metadata) tuple.getValueByField("metadata");

        // functionality deactivated
        if (StringUtils.isBlank(canonicalMetadataParamName)) {
            return url;
        }

        String canonicalValue = metadata.getFirstValue(canonicalMetadataName);

        // no value found?
        if (StringUtils.isBlank(canonicalValue)) {
            return url;
        }

        try {
            URL sURL = new URL(url);
            URL canonical = URLUtil.resolveURL(sURL, canonicalValue);

            String sDomain = PaidLevelDomain.getPLD(sURL.getHost());
            String canonicalDomain = PaidLevelDomain
                    .getPLD(canonical.getHost());

            // check that the domain is the same
            if (sDomain.equalsIgnoreCase(canonicalDomain)) {
                return canonical.toExternalForm();
            } else {
                LOG.info(
                        "Canonical URL references a different domain, ignoring in {} ",
                        url);
            }
        } catch (MalformedURLException e) {
            LOG.error("Malformed canonical URL {} was found in {} ",
                    canonicalValue, url);
        }

        return url;
    }
protected String valueForURL ( Tuple tuple ) { String url = tuple . getStringByField ( "url" ) ; Metadata metadata = ( Metadata ) tuple . getValueByField ( "metadata" ) ; // functionality deactivated if ( StringUtils . isBlank ( canonicalMetadataParamName ) ) { return url ; } String canonicalValue = metadata . getFirstValue ( canonicalMetadataName ) ; // no value found? if ( StringUtils . isBlank ( canonicalValue ) ) { return url ; } try { URL sURL = new URL ( url ) ; URL canonical = URLUtil . resolveURL ( sURL , canonicalValue ) ; String sDomain = PaidLevelDomain . getPLD ( sURL . getHost ( ) ) ; String canonicalDomain = PaidLevelDomain . getPLD ( canonical . getHost ( ) ) ; // check that the domain is the same if ( sDomain . equalsIgnoreCase ( canonicalDomain ) ) { return canonical . toExternalForm ( ) ; } else { LOG . info ( "Canonical URL references a different domain, ignoring in {} " , url ) ; } } catch ( MalformedURLException e ) { LOG . error ( "Malformed canonical URL {} was found in {} " , canonicalValue , url ) ; } return url ; }
Returns the value to be used as the URL for indexing purposes, if present
the canonical value is used instead
Returns the value to be used as the URL for indexing purposes if present the canonical value is used instead