core/src/main/java/com/digitalpebble/stormcrawler/util/URLPartitioner.java
public String getPartition(String url, Metadata metadata) {

        String partitionKey = null;
        String host = "";

        // IP in metadata?
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_IP)) {
            String ip_provided = metadata.getFirstValue("ip");
            if (StringUtils.isNotBlank(ip_provided)) {
                partitionKey = ip_provided;
            }
        }

        if (partitionKey == null) {
            URL u;
            try {
                u = new URL(url);
                host = u.getHost();
            } catch (MalformedURLException e1) {
                LOG.warn("Invalid URL: {}", url);
                return null;
            }
        }

        // partition by hostname
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_HOST))
            partitionKey = host;

        // partition by domain : needs fixing
        else if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_DOMAIN)) {
            partitionKey = PaidLevelDomain.getPLD(host);
        }

        // partition by IP
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_IP)
                && partitionKey == null) {
            try {
                long start = System.currentTimeMillis();
                final InetAddress addr = InetAddress.getByName(host);
                partitionKey = addr.getHostAddress();
                long end = System.currentTimeMillis();
                LOG.debug("Resolved IP {} in {} msec for : {}", partitionKey,
                        end - start, url);
            } catch (final Exception e) {
                LOG.warn("Unable to resolve IP for: {}", host);
                return null;
            }
        }

        LOG.debug("Partition Key for: {} > {}", url, partitionKey);

        return partitionKey;
    }
public String getPartition(String url, Metadata metadata) {

        String partitionKey = null;
        String host = "";

        // IP in metadata?
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_IP)) {
            String ip_provided = metadata.getFirstValue("ip");
            if (StringUtils.isNotBlank(ip_provided)) {
                partitionKey = ip_provided;
            }
        }

        if (partitionKey == null) {
            URL u;
            try {
                u = new URL(url);
                host = u.getHost();
            } catch (MalformedURLException e1) {
                LOG.warn("Invalid URL: {}", url);
                return null;
            }
        }

        // partition by hostname
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_HOST))
            partitionKey = host;

        // partition by domain : needs fixing
        else if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_DOMAIN)) {
            partitionKey = PaidLevelDomain.getPLD(host);
        }

        // partition by IP
        if (mode.equalsIgnoreCase(Constants.PARTITION_MODE_IP)
                && partitionKey == null) {
            try {
                long start = System.currentTimeMillis();
                final InetAddress addr = InetAddress.getByName(host);
                partitionKey = addr.getHostAddress();
                long end = System.currentTimeMillis();
                LOG.debug("Resolved IP {} in {} msec for : {}", partitionKey,
                        end - start, url);
            } catch (final Exception e) {
                LOG.warn("Unable to resolve IP for: {}", host);
                return null;
            }
        }

        LOG.debug("Partition Key for: {} > {}", url, partitionKey);

        return partitionKey;
    }
public String getPartition ( String url , Metadata metadata ) { String partitionKey = null ; String host = "" ; // IP in metadata? if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) ) { String ip_provided = metadata . getFirstValue ( "ip" ) ; if ( StringUtils . isNotBlank ( ip_provided ) ) { partitionKey = ip_provided ; } } if ( partitionKey == null ) { URL u ; try { u = new URL ( url ) ; host = u . getHost ( ) ; } catch ( MalformedURLException e1 ) { LOG . warn ( "Invalid URL: {}" , url ) ; return null ; } } // partition by hostname if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_HOST ) ) partitionKey = host ; // partition by domain : needs fixing else if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_DOMAIN ) ) { partitionKey = PaidLevelDomain . getPLD ( host ) ; } // partition by IP if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) && partitionKey == null ) { try { long start = System . currentTimeMillis ( ) ; final InetAddress addr = InetAddress . getByName ( host ) ; partitionKey = addr . getHostAddress ( ) ; long end = System . currentTimeMillis ( ) ; LOG . debug ( "Resolved IP {} in {} msec for : {}" , partitionKey , end - start , url ) ; } catch ( final Exception e ) { LOG . warn ( "Unable to resolve IP for: {}" , host ) ; return null ; } } LOG . debug ( "Partition Key for: {} > {}" , url , partitionKey ) ; return partitionKey ; }
Returns the host, domain, IP of a URL so that it can be partitioned for
politeness, depending on the value of the config
<i>partition.url.mode</i>.
Returns the host domain IP of a URL so that it can be partitioned for politeness depending on the value of the config <i > partition . url . mode< / i > .