core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java
protected static void main(AbstractHttpProtocol protocol, String args[])
            throws Exception {
        Config conf = new Config();

        // loads the default configuration file
        Map defaultSCConfig = Utils.findAndReadConfigFile(
                "crawler-default.yaml", false);
        conf.putAll(ConfUtils.extractConfigElement(defaultSCConfig));

        Options options = new Options();
        options.addOption("c", true, "configuration file");

        CommandLineParser parser = new DefaultParser();
        CommandLine cmd = parser.parse(options, args);

        if (cmd.hasOption("c")) {
            String confFile = cmd.getOptionValue("c");
            ConfUtils.loadConf(confFile, conf);
        }

        protocol.configure(conf);

        Set<Runnable> threads = new HashSet<>();

        class Fetchable implements Runnable {
            String url;
            Metadata md;

            Fetchable(String line) {
                StringTabScheme scheme = new StringTabScheme();
                List<Object> tuple = scheme.deserialize(ByteBuffer.wrap(line
                        .getBytes(StandardCharsets.UTF_8)));
                this.url = (String) tuple.get(0);
                this.md = (Metadata) tuple.get(1);
            }

            public void run() {

                StringBuilder stringB = new StringBuilder();
                stringB.append(url).append("\n");

                if (!protocol.skipRobots) {
                    BaseRobotRules rules = protocol.getRobotRules(url);
                    stringB.append("robots allowed: ")
                            .append(rules.isAllowed(url)).append("\n");
                    if (rules instanceof RobotRules) {
                        stringB.append("robots requests: ")
                                .append(((RobotRules) rules)
                                        .getContentLengthFetched().length)
                                .append("\n");
                    }
                    stringB.append("sitemaps identified: ")
                            .append(rules.getSitemaps().size()).append("\n");
                }

                long start = System.currentTimeMillis();
                ProtocolResponse response;
                try {
                    response = protocol.getProtocolOutput(url, md);
                    stringB.append(response.getMetadata()).append("\n");
                    stringB.append("status code: ")
                            .append(response.getStatusCode()).append("\n");
                    stringB.append("content length: ")
                            .append(response.getContent().length).append("\n");
                    long timeFetching = System.currentTimeMillis() - start;
                    stringB.append("fetched in : ").append(timeFetching)
                            .append(" msec");
                    System.out.println(stringB);
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    threads.remove(this);
                }
            }
        }

        for (String arg : cmd.getArgs()) {
            Fetchable p = new Fetchable(arg);
            threads.add(p);
            new Thread(p).start();
        }

        while (threads.size() > 0) {
            Thread.sleep(1000);
        }

        protocol.cleanup();
        System.exit(0);
    }
protected static void main(AbstractHttpProtocol protocol, String args[])
            throws Exception {
        Config conf = new Config();

        // loads the default configuration file
        Map defaultSCConfig = Utils.findAndReadConfigFile(
                "crawler-default.yaml", false);
        conf.putAll(ConfUtils.extractConfigElement(defaultSCConfig));

        Options options = new Options();
        options.addOption("c", true, "configuration file");

        CommandLineParser parser = new DefaultParser();
        CommandLine cmd = parser.parse(options, args);

        if (cmd.hasOption("c")) {
            String confFile = cmd.getOptionValue("c");
            ConfUtils.loadConf(confFile, conf);
        }

        protocol.configure(conf);

        Set<Runnable> threads = new HashSet<>();

        class Fetchable implements Runnable {
            String url;
            Metadata md;

            Fetchable(String line) {
                StringTabScheme scheme = new StringTabScheme();
                List<Object> tuple = scheme.deserialize(ByteBuffer.wrap(line
                        .getBytes(StandardCharsets.UTF_8)));
                this.url = (String) tuple.get(0);
                this.md = (Metadata) tuple.get(1);
            }

            public void run() {

                StringBuilder stringB = new StringBuilder();
                stringB.append(url).append("\n");

                if (!protocol.skipRobots) {
                    BaseRobotRules rules = protocol.getRobotRules(url);
                    stringB.append("robots allowed: ")
                            .append(rules.isAllowed(url)).append("\n");
                    if (rules instanceof RobotRules) {
                        stringB.append("robots requests: ")
                                .append(((RobotRules) rules)
                                        .getContentLengthFetched().length)
                                .append("\n");
                    }
                    stringB.append("sitemaps identified: ")
                            .append(rules.getSitemaps().size()).append("\n");
                }

                long start = System.currentTimeMillis();
                ProtocolResponse response;
                try {
                    response = protocol.getProtocolOutput(url, md);
                    stringB.append(response.getMetadata()).append("\n");
                    stringB.append("status code: ")
                            .append(response.getStatusCode()).append("\n");
                    stringB.append("content length: ")
                            .append(response.getContent().length).append("\n");
                    long timeFetching = System.currentTimeMillis() - start;
                    stringB.append("fetched in : ").append(timeFetching)
                            .append(" msec");
                    System.out.println(stringB);
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    threads.remove(this);
                }
            }
        }

        for (String arg : cmd.getArgs()) {
            Fetchable p = new Fetchable(arg);
            threads.add(p);
            new Thread(p).start();
        }

        while (threads.size() > 0) {
            Thread.sleep(1000);
        }

        protocol.cleanup();
        System.exit(0);
    }
protected static void main ( AbstractHttpProtocol protocol , String args [ ] ) throws Exception { Config conf = new Config ( ) ; // loads the default configuration file Map defaultSCConfig = Utils . findAndReadConfigFile ( "crawler-default.yaml" , false ) ; conf . putAll ( ConfUtils . extractConfigElement ( defaultSCConfig ) ) ; Options options = new Options ( ) ; options . addOption ( "c" , true , "configuration file" ) ; CommandLineParser parser = new DefaultParser ( ) ; CommandLine cmd = parser . parse ( options , args ) ; if ( cmd . hasOption ( "c" ) ) { String confFile = cmd . getOptionValue ( "c" ) ; ConfUtils . loadConf ( confFile , conf ) ; } protocol . configure ( conf ) ; Set < Runnable > threads = new HashSet <> ( ) ; class Fetchable implements Runnable { String url ; Metadata md ; Fetchable ( String line ) { StringTabScheme scheme = new StringTabScheme ( ) ; List < Object > tuple = scheme . deserialize ( ByteBuffer . wrap ( line . getBytes ( StandardCharsets . UTF_8 ) ) ) ; this . url = ( String ) tuple . get ( 0 ) ; this . md = ( Metadata ) tuple . get ( 1 ) ; } public void run ( ) { StringBuilder stringB = new StringBuilder ( ) ; stringB . append ( url ) . append ( "\n" ) ; if ( ! protocol . skipRobots ) { BaseRobotRules rules = protocol . getRobotRules ( url ) ; stringB . append ( "robots allowed: " ) . append ( rules . isAllowed ( url ) ) . append ( "\n" ) ; if ( rules instanceof RobotRules ) { stringB . append ( "robots requests: " ) . append ( ( ( RobotRules ) rules ) . getContentLengthFetched ( ) . length ) . append ( "\n" ) ; } stringB . append ( "sitemaps identified: " ) . append ( rules . getSitemaps ( ) . size ( ) ) . append ( "\n" ) ; } long start = System . currentTimeMillis ( ) ; ProtocolResponse response ; try { response = protocol . getProtocolOutput ( url , md ) ; stringB . append ( response . getMetadata ( ) ) . append ( "\n" ) ; stringB . append ( "status code: " ) . append ( response . getStatusCode ( ) ) . append ( "\n" ) ; stringB . append ( "content length: " ) . append ( response . getContent ( ) . length ) . append ( "\n" ) ; long timeFetching = System . currentTimeMillis ( ) - start ; stringB . append ( "fetched in : " ) . append ( timeFetching ) . append ( " msec" ) ; System . out . println ( stringB ) ; } catch ( Exception e ) { e . printStackTrace ( ) ; } finally { threads . remove ( this ) ; } } } for ( String arg : cmd . getArgs ( ) ) { Fetchable p = new Fetchable ( arg ) ; threads . add ( p ) ; new Thread ( p ) . start ( ) ; } while ( threads . size ( ) > 0 ) { Thread . sleep ( 1000 ) ; } protocol . cleanup ( ) ; System . exit ( 0 ) ; }
Called by extensions of this class *
Called by extensions of this class *