Quadcap Embeddable Server

com/quadcap/http/client/LinkChecker.java

Go to the documentation of this file.

package com.quadcap.http.client;

/* Copyright 1998 - 2003 Quadcap Software.  All rights reserved.
 *
 * This software is distributed under the Quadcap Free Software License.
 * This software may be used or modified for any purpose, personal or
 * commercial.  Open Source redistributions are permitted.  Commercial
 * redistribution of larger works derived from, or works which bundle
 * this software requires a "Commercial Redistribution License"; see
 * http://www.quadcap.com/purchase.
 *
 * Redistributions qualify as "Open Source" under  one of the following terms:
 *   
 *    Redistributions are made at no charge beyond the reasonable cost of
 *    materials and delivery.
 *
 *    Redistributions are accompanied by a copy of the Source Code or by an
 *    irrevocable offer to provide a copy of the Source Code for up to three
 *    years at the cost of materials and delivery.  Such redistributions
 *    must allow further use, modification, and redistribution of the Source
 *    Code under substantially the same terms as this license.
 *
 * Redistributions of source code must retain the copyright notices as they
 * appear in each source code file, these license terms, and the
 * disclaimer/limitation of liability set forth as paragraph 6 below.
 *
 * Redistributions in binary form must reproduce this Copyright Notice,
 * these license terms, and the disclaimer/limitation of liability set
 * forth as paragraph 6 below, in the documentation and/or other materials
 * provided with the distribution.
 *
 * The Software is provided on an "AS IS" basis.  No warranty is
 * provided that the Software is free of defects, or fit for a
 * particular purpose.  
 *
 * Limitation of Liability. Quadcap Software shall not be liable
 * for any damages suffered by the Licensee or any third party resulting
 * from use of the Software.
 */

import java.io.*;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;

import org.xml.sax.AttributeList;
import org.xml.sax.DocumentHandler;
import org.xml.sax.DTDHandler;
import org.xml.sax.EntityResolver;
import org.xml.sax.ErrorHandler;
import org.xml.sax.HandlerBase;
import org.xml.sax.InputSource;
import org.xml.sax.Locator;
import org.xml.sax.SAXException;

import com.quadcap.text.sax.Parser;

import com.quadcap.http.util.HeaderParser;

import com.quadcap.util.collections.ArrayQueue;
import com.quadcap.util.collections.DiGraph;

import com.quadcap.util.text.OctetMap;
import com.quadcap.util.text.Scanner;

import com.quadcap.util.Debug;
import com.quadcap.util.Util;


/**
 * This class implements a simple link checker, following links
 * in the following tags:
 *
 * <ul>
 * <li><b>&lt;A HREF=""&gt;
 * <li><b>&lt;IMG SRC=""&gt;
 * <li><b>&lt;FRAME SRC=""&gt;
 * </ul>
 */
public class LinkChecker implements DocumentHandler { 
    /** uri of the document we're currently fetching and parsing   */
    String                      base;

    /** base uri of the current document for relative href resolution */
    String                      urlBase;

    /** uri of the document we're currently fetching and parsing   */
    String                      currentUrl;

    /** directed graph of all links found so far (even bad ones...) */
    DiGraph                     links = new DiGraph();

    /** queue of links to check */
    ArrayQueue                  linksToCheck = new ArrayQueue();

    /** uri -> status for all links */
    HashMap                   allLinks = new HashMap();

    /** uri -> status for completed links */
    HashMap                     linksChecked = new HashMap();

    Parser                      parser;
    String                      host;
        
    public LinkChecker(String url) {
        parser = new Parser();
        String s = url;
        if (s.startsWith("http://")) {
            s = url.substring("http://".length());
        }
        int idx = s.indexOf('/');
        if (idx > 0) s = s.substring(0, idx);
        host = "http://" + s;
        push(url, 0);
    }

    synchronized void push(String url, int line) {
        if (allLinks.get(url) == null && url.startsWith(host)) {
            System.out.println("PUSH " + trim(base) + " -> " + trim(url));
            if (currentUrl != null) {
                links.addArc(currentUrl + ":" + line, url);
            }
            allLinks.put(url, "queued");
            linksToCheck.push(url);
        }
    }

    String trim(String url) {
        if (url != null && url.startsWith(host)) {
            url = url.substring(host.length());
        }
        return url;
    }

    public void printBadLinks() {
        ArrayList k = new ArrayList();
        Iterator iter = linksChecked.keySet().iterator();
        while (iter.hasNext()) {
            String url = iter.next().toString();
            String val = linksChecked.get(url).toString();
            if (!val.equals("found")) {
                Iterator x = links.getParents(url);
                String ref = x.hasNext() ? x.next().toString() : "";
                k.add(trim(ref) + "\n    error: " +  trim(url));
            }
        }
        Collections.sort(k);
        iter = k.iterator();
        while (iter.hasNext()) {
            System.out.println(iter.next().toString());
        }
        System.out.println("--------------------\n");
        System.out.println("" + k.size() + " errors");
    }
    
    public void run() throws Exception {
        //HtmlParser parser = new HtmlParser();
        int cnt = 0;
        while (linksToCheck.size() > 0) {
            System.out.print("" + (linksChecked.size()+1) + " of " +
                             (linksToCheck.size() + linksChecked.size()) +
                             ": ");
            String url = linksToCheck.popBack().toString();
            if (linksChecked.get(url) != null) continue;
            System.out.println(trim(url));
            currentUrl = url;
            InputStream is = null;
            try {
                is = HttpFetcher.fetchStream(url);
                Scanner scanner = new Scanner(is);
                HashMap headers = new HashMap();
                scanner.skipUntil(OctetMap.wsChars);
                scanner.skipWhile(OctetMap.wsChars);
                String resp = scanner.parseUntil(OctetMap.crlfChars);
                HeaderParser.parseCRLF(scanner);
                HeaderParser.parseHeaders(scanner, headers);
                if (!resp.startsWith("200")) {
                    allLinks.put(url, "missing");
                    linksChecked.put(url, "missing");                
                    Iterator iter = links.getParents(url);
                    String referrer =
                        iter.hasNext()
                        ?  iter.next().toString()
                        :  "---";
                    System.err.println("*** " + trim(url) + "," +
                                       trim(referrer) + "," + resp);
                    continue;
                }
                String mimeType = (String)headers.get("content-type");
                if (mimeType == null || !mimeType.equals("text/html")) {
                    continue;
                }
                InputStreamReader r = new InputStreamReader(is);
                parser.setDocumentHandler(this);
                setBase(url);
                parser.parse(in);
                allLinks.put(url, "found");
                linksChecked.put(url, "found");                
            } catch (IOException e) {
                Debug.print(e);
                allLinks.put(url, "error");
                linksChecked.put(url, "error");
            } catch (Exception e3) {
                Debug.print(e3);
                allLinks.put(url, "exception");
                linksChecked.put(url, "exception");
            } catch (Throwable t) {
                Debug.print(t);
                allLinks.put(url, "exception");
                linksChecked.put(url, "exception");                
            } finally {
                if (is != null) is.close();
                //System.out.println("Result: " + allLinks.get(url));
            }
        }
    }
    
    public void setBase(String base) {
        this.base = base;
        this.urlBase = parent(base);
        if (base.endsWith("/")) urlBase = base;
    }

    public void startDocument() {
    }

    public void endDocument() {
    }

    public void ignorableWhitespace(char[] ch, int off, int cnt) 
    {
        characters(ch, off, cnt);
    }

    public void processingInstruction(String target, String data) {
    }

    }

    {
        try {
            if (tag.equalsIgnoreCase("a")) {
                String href = attrs.getValue("href");
                if (href != null) checkHref(href, parser.getLineNumber());
            } else if (tag.equalsIgnoreCase("img") || 
                       tag.equalsIgnoreCase("frame")) {
                String href = attrs.getValue("src");
                if (href != null) checkHref(href, parser.getLineNumber());
            }
        } catch (Throwable t) {
            t.printStackTrace(System.err);
            //System.err.println(t.toString());
            System.err.println("tag = " + tag);
            System.err.println("attrs = " + attrs);
            System.err.println("urlBase = " + urlBase);
        }
    }
   
    }

    }

    public void checkHref(String href, int line) {
        String tbase = urlBase;
        href = href.trim();
        if (href.length() > 0 && href.charAt(0) == '/') {
            href = href.substring(1);
            tbase = "";
        } else if (href.startsWith("http://")) {
            tbase = "";
        } else if (href.startsWith("ftp://") ||
            href.startsWith("mailto:")) {
            return;
        } else {
            while (href.startsWith("./") || href.startsWith("../")) {
                if (href.startsWith("./")) {
                    href = href.substring(2);
                } else if (href.startsWith("../")) {
                    href = href.substring(3);
                    tbase = parent(tbase);
                }
            }
        }
        String url = tbase + href;
        int idx = url.indexOf('#');
        if (idx >= 0) {
            url = url.substring(0, idx);
        }
        if (url.length() == 0) return;
        push(url, line);
    }

    static String parent(String s) {
        for (int i = s.length() - 2; i >= 0; i--) {
            if (s.charAt(i) == '/') return s.substring(0, i+1);
        }
        throw new RuntimeException("Bad parent: " + s);
    }
}