core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLNormalizer.java
private String processQueryElements(String urlToFilter) {
        try {
            // Handle illegal characters by making a url first
            // this will clean illegal characters like |
            URL url = new URL(urlToFilter);

            String query = url.getQuery();
            String path = url.getPath();

            // check if the last element of the path contains parameters
            // if so convert them to query elements
            if (path.contains(";")) {
                String[] pathElements = path.split("/");
                String last = pathElements[pathElements.length - 1];
                // replace last value by part without params
                int semicolon = last.indexOf(";");
                if (semicolon != -1) {
                    pathElements[pathElements.length - 1] = last.substring(0,
                            semicolon);
                    String params = last.substring(semicolon + 1).replaceAll(
                            ";", "&");
                    if (query == null) {
                        query = params;
                    } else {
                        query += "&" + params;
                    }
                    // rebuild the path
                    StringBuilder newPath = new StringBuilder();
                    for (String p : pathElements) {
                        if (StringUtils.isNotBlank(p)) {
                            newPath.append("/").append(p);
                        }
                    }
                    path = newPath.toString();
                }
            }

            if (StringUtils.isEmpty(query)) {
                return urlToFilter;
            }

            List<NameValuePair> pairs = URLEncodedUtils.parse(query,
                    StandardCharsets.UTF_8);
            Iterator<NameValuePair> pairsIterator = pairs.iterator();
            while (pairsIterator.hasNext()) {
                NameValuePair param = pairsIterator.next();
                if (queryElementsToRemove.contains(param.getName())) {
                    pairsIterator.remove();
                } else if (removeHashes && param.getValue() != null) {
                    Matcher m = thirtytwobithash.matcher(param.getValue());
                    if (m.matches()) {
                        pairsIterator.remove();
                    }
                }
            }

            StringBuilder newFile = new StringBuilder();
            if (StringUtils.isNotBlank(path)) {
                newFile.append(path);
            }
            if (!pairs.isEmpty()) {
                Collections.sort(pairs, comp);
                String newQueryString = URLEncodedUtils.format(pairs,
                        StandardCharsets.UTF_8);
                newFile.append('?').append(newQueryString);
            }
            if (url.getRef() != null) {
                newFile.append('#').append(url.getRef());
            }

            return new URL(url.getProtocol(), url.getHost(), url.getPort(),
                    newFile.toString()).toString();
        } catch (MalformedURLException e) {
            LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
            return null;
        }
    }
private String processQueryElements(String urlToFilter) {
        try {
            // Handle illegal characters by making a url first
            // this will clean illegal characters like |
            URL url = new URL(urlToFilter);

            String query = url.getQuery();
            String path = url.getPath();

            // check if the last element of the path contains parameters
            // if so convert them to query elements
            if (path.contains(";")) {
                String[] pathElements = path.split("/");
                String last = pathElements[pathElements.length - 1];
                // replace last value by part without params
                int semicolon = last.indexOf(";");
                if (semicolon != -1) {
                    pathElements[pathElements.length - 1] = last.substring(0,
                            semicolon);
                    String params = last.substring(semicolon + 1).replaceAll(
                            ";", "&");
                    if (query == null) {
                        query = params;
                    } else {
                        query += "&" + params;
                    }
                    // rebuild the path
                    StringBuilder newPath = new StringBuilder();
                    for (String p : pathElements) {
                        if (StringUtils.isNotBlank(p)) {
                            newPath.append("/").append(p);
                        }
                    }
                    path = newPath.toString();
                }
            }

            if (StringUtils.isEmpty(query)) {
                return urlToFilter;
            }

            List<NameValuePair> pairs = URLEncodedUtils.parse(query,
                    StandardCharsets.UTF_8);
            Iterator<NameValuePair> pairsIterator = pairs.iterator();
            while (pairsIterator.hasNext()) {
                NameValuePair param = pairsIterator.next();
                if (queryElementsToRemove.contains(param.getName())) {
                    pairsIterator.remove();
                } else if (removeHashes && param.getValue() != null) {
                    Matcher m = thirtytwobithash.matcher(param.getValue());
                    if (m.matches()) {
                        pairsIterator.remove();
                    }
                }
            }

            StringBuilder newFile = new StringBuilder();
            if (StringUtils.isNotBlank(path)) {
                newFile.append(path);
            }
            if (!pairs.isEmpty()) {
                Collections.sort(pairs, comp);
                String newQueryString = URLEncodedUtils.format(pairs,
                        StandardCharsets.UTF_8);
                newFile.append('?').append(newQueryString);
            }
            if (url.getRef() != null) {
                newFile.append('#').append(url.getRef());
            }

            return new URL(url.getProtocol(), url.getHost(), url.getPort(),
                    newFile.toString()).toString();
        } catch (MalformedURLException e) {
            LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
            return null;
        }
    }
private String processQueryElements ( String urlToFilter ) { try { // Handle illegal characters by making a url first // this will clean illegal characters like | URL url = new URL ( urlToFilter ) ; String query = url . getQuery ( ) ; String path = url . getPath ( ) ; // check if the last element of the path contains parameters // if so convert them to query elements if ( path . contains ( ";" ) ) { String [ ] pathElements = path . split ( "/" ) ; String last = pathElements [ pathElements . length - 1 ] ; // replace last value by part without params int semicolon = last . indexOf ( ";" ) ; if ( semicolon != - 1 ) { pathElements [ pathElements . length - 1 ] = last . substring ( 0 , semicolon ) ; String params = last . substring ( semicolon + 1 ) . replaceAll ( ";" , "&" ) ; if ( query == null ) { query = params ; } else { query += "&" + params ; } // rebuild the path StringBuilder newPath = new StringBuilder ( ) ; for ( String p : pathElements ) { if ( StringUtils . isNotBlank ( p ) ) { newPath . append ( "/" ) . append ( p ) ; } } path = newPath . toString ( ) ; } } if ( StringUtils . isEmpty ( query ) ) { return urlToFilter ; } List < NameValuePair > pairs = URLEncodedUtils . parse ( query , StandardCharsets . UTF_8 ) ; Iterator < NameValuePair > pairsIterator = pairs . iterator ( ) ; while ( pairsIterator . hasNext ( ) ) { NameValuePair param = pairsIterator . next ( ) ; if ( queryElementsToRemove . contains ( param . getName ( ) ) ) { pairsIterator . remove ( ) ; } else if ( removeHashes && param . getValue ( ) != null ) { Matcher m = thirtytwobithash . matcher ( param . getValue ( ) ) ; if ( m . matches ( ) ) { pairsIterator . remove ( ) ; } } } StringBuilder newFile = new StringBuilder ( ) ; if ( StringUtils . isNotBlank ( path ) ) { newFile . append ( path ) ; } if ( ! pairs . isEmpty ( ) ) { Collections . sort ( pairs , comp ) ; String newQueryString = URLEncodedUtils . format ( pairs , StandardCharsets . UTF_8 ) ; newFile . append ( ' ' ) . append ( newQueryString ) ; } if ( url . getRef ( ) != null ) { newFile . append ( ' ' ) . append ( url . getRef ( ) ) ; } return new URL ( url . getProtocol ( ) , url . getHost ( ) , url . getPort ( ) , newFile . toString ( ) ) . toString ( ) ; } catch ( MalformedURLException e ) { LOG . warn ( "Invalid urlToFilter {}. {}" , urlToFilter , e ) ; return null ; } }
Basic filter to remove query parameters from urls so parameters that
don't change the content of the page can be removed. An example would be
a google analytics query parameter like "utm_campaign" which might have
several different values for a url that points to the same content. This
is also called when removing attributes where the value is a hash.
Basic filter to remove query parameters from urls so parameters that don t change the content of the page can be removed . An example would be a google analytics query parameter like utm_campaign which might have several different values for a url that points to the same content . This is also called when removing attributes where the value is a hash .