core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
protected boolean filterDocument(Metadata meta) {
        String noindexVal = meta.getFirstValue(RobotsTags.ROBOTS_NO_INDEX);
        if ("true".equalsIgnoreCase(noindexVal))
            return false;

        if (filterKeyValue == null)
            return true;
        String[] values = meta.getValues(filterKeyValue[0]);
        // key not found
        if (values == null)
            return false;
        return ArrayUtils.contains(values, filterKeyValue[1]);
    }
protected boolean filterDocument(Metadata meta) {
        String noindexVal = meta.getFirstValue(RobotsTags.ROBOTS_NO_INDEX);
        if ("true".equalsIgnoreCase(noindexVal))
            return false;

        if (filterKeyValue == null)
            return true;
        String[] values = meta.getValues(filterKeyValue[0]);
        // key not found
        if (values == null)
            return false;
        return ArrayUtils.contains(values, filterKeyValue[1]);
    }
protected boolean filterDocument ( Metadata meta ) { String noindexVal = meta . getFirstValue ( RobotsTags . ROBOTS_NO_INDEX ) ; if ( "true" . equalsIgnoreCase ( noindexVal ) ) return false ; if ( filterKeyValue == null ) return true ; String [ ] values = meta . getValues ( filterKeyValue [ 0 ] ) ; // key not found if ( values == null ) return false ; return ArrayUtils . contains ( values , filterKeyValue [ 1 ] ) ; }
Determine whether a document should be indexed based on the presence of a
given key/value or the RobotsTags.ROBOTS_NO_INDEX directive.

@return true if the document should be kept.
Determine whether a document should be indexed based on the presence of a given key / value or the RobotsTags . ROBOTS_NO_INDEX directive .