core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRulesParser.java
public void setConf(Config conf) {

        // Grab the agent names we advertise to robots files.
        String agentName = ConfUtils.getString(conf, "http.agent.name");
        if (null == agentName) {
            throw new RuntimeException("Agent name not configured!");
        }

        String configuredAgentNames = ConfUtils.getString(conf,
                "http.robots.agents", "");
        StringTokenizer tok = new StringTokenizer(configuredAgentNames, ",");
        ArrayList<String> agents = new ArrayList<>();
        while (tok.hasMoreTokens()) {
            agents.add(tok.nextToken().trim());
        }

        /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */
        if (agents.isEmpty()) {
            LOG.info(
                    "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]",
                    agentName);
            this.agentNames = agentName;
        } else {
            int index = 0;
            if ((agents.get(0)).equalsIgnoreCase(agentName)) {
                index++;
            } else {
                LOG.info(
                        "Agent we advertise ({}) not listed first in 'http.robots.agents' property!",
                        agentName);
            }

            StringBuilder combinedAgentsString = new StringBuilder(agentName);
            // append all the agents from the http.robots.agents property
            for (; index < agents.size(); index++) {
                combinedAgentsString.append(", ").append(agents.get(index));
            }

            this.agentNames = combinedAgentsString.toString();
        }

        String spec = ConfUtils.getString(conf, cacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=6h");
        CACHE = CacheBuilder.from(spec).build();

        spec = ConfUtils.getString(conf, errorcacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=1h");
        ERRORCACHE = CacheBuilder.from(spec).build();
    }
public void setConf(Config conf) {

        // Grab the agent names we advertise to robots files.
        String agentName = ConfUtils.getString(conf, "http.agent.name");
        if (null == agentName) {
            throw new RuntimeException("Agent name not configured!");
        }

        String configuredAgentNames = ConfUtils.getString(conf,
                "http.robots.agents", "");
        StringTokenizer tok = new StringTokenizer(configuredAgentNames, ",");
        ArrayList<String> agents = new ArrayList<>();
        while (tok.hasMoreTokens()) {
            agents.add(tok.nextToken().trim());
        }

        /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */
        if (agents.isEmpty()) {
            LOG.info(
                    "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]",
                    agentName);
            this.agentNames = agentName;
        } else {
            int index = 0;
            if ((agents.get(0)).equalsIgnoreCase(agentName)) {
                index++;
            } else {
                LOG.info(
                        "Agent we advertise ({}) not listed first in 'http.robots.agents' property!",
                        agentName);
            }

            StringBuilder combinedAgentsString = new StringBuilder(agentName);
            // append all the agents from the http.robots.agents property
            for (; index < agents.size(); index++) {
                combinedAgentsString.append(", ").append(agents.get(index));
            }

            this.agentNames = combinedAgentsString.toString();
        }

        String spec = ConfUtils.getString(conf, cacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=6h");
        CACHE = CacheBuilder.from(spec).build();

        spec = ConfUtils.getString(conf, errorcacheConfigParamName,
                "maximumSize=10000,expireAfterWrite=1h");
        ERRORCACHE = CacheBuilder.from(spec).build();
    }
public void setConf ( Config conf ) { // Grab the agent names we advertise to robots files. String agentName = ConfUtils . getString ( conf , "http.agent.name" ) ; if ( null == agentName ) { throw new RuntimeException ( "Agent name not configured!" ) ; } String configuredAgentNames = ConfUtils . getString ( conf , "http.robots.agents" , "" ) ; StringTokenizer tok = new StringTokenizer ( configuredAgentNames , "," ) ; ArrayList < String > agents = new ArrayList <> ( ) ; while ( tok . hasMoreTokens ( ) ) { agents . add ( tok . nextToken ( ) . trim ( ) ) ; } /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */ if ( agents . isEmpty ( ) ) { LOG . info ( "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]" , agentName ) ; this . agentNames = agentName ; } else { int index = 0 ; if ( ( agents . get ( 0 ) ) . equalsIgnoreCase ( agentName ) ) { index ++ ; } else { LOG . info ( "Agent we advertise ({}) not listed first in 'http.robots.agents' property!" , agentName ) ; } StringBuilder combinedAgentsString = new StringBuilder ( agentName ) ; // append all the agents from the http.robots.agents property for ( ; index < agents . size ( ) ; index ++ ) { combinedAgentsString . append ( ", " ) . append ( agents . get ( index ) ) ; } this . agentNames = combinedAgentsString . toString ( ) ; } String spec = ConfUtils . getString ( conf , cacheConfigParamName , "maximumSize=10000,expireAfterWrite=6h" ) ; CACHE = CacheBuilder . from ( spec ) . build ( ) ; spec = ConfUtils . getString ( conf , errorcacheConfigParamName , "maximumSize=10000,expireAfterWrite=1h" ) ; ERRORCACHE = CacheBuilder . from ( spec ) . build ( ) ; }
Set the {@link Configuration} object
Set the {