Quadcap Embeddable Server

com/quadcap/http/client/HttpFetcher.java

Go to the documentation of this file.

package com.quadcap.http.client;

/* Copyright 1998 - 2003 Quadcap Software.  All rights reserved.
 *
 * This software is distributed under the Quadcap Free Software License.
 * This software may be used or modified for any purpose, personal or
 * commercial.  Open Source redistributions are permitted.  Commercial
 * redistribution of larger works derived from, or works which bundle
 * this software requires a "Commercial Redistribution License"; see
 * http://www.quadcap.com/purchase.
 *
 * Redistributions qualify as "Open Source" under  one of the following terms:
 *   
 *    Redistributions are made at no charge beyond the reasonable cost of
 *    materials and delivery.
 *
 *    Redistributions are accompanied by a copy of the Source Code or by an
 *    irrevocable offer to provide a copy of the Source Code for up to three
 *    years at the cost of materials and delivery.  Such redistributions
 *    must allow further use, modification, and redistribution of the Source
 *    Code under substantially the same terms as this license.
 *
 * Redistributions of source code must retain the copyright notices as they
 * appear in each source code file, these license terms, and the
 * disclaimer/limitation of liability set forth as paragraph 6 below.
 *
 * Redistributions in binary form must reproduce this Copyright Notice,
 * these license terms, and the disclaimer/limitation of liability set
 * forth as paragraph 6 below, in the documentation and/or other materials
 * provided with the distribution.
 *
 * The Software is provided on an "AS IS" basis.  No warranty is
 * provided that the Software is free of defects, or fit for a
 * particular purpose.  
 *
 * Limitation of Liability. Quadcap Software shall not be liable
 * for any damages suffered by the Licensee or any third party resulting
 * from use of the Software.
 */

import java.io.*;
import java.util.*;

import java.net.Socket;
import java.net.URL;
import java.net.URLEncoder;
import java.net.URLConnection;

import org.xml.sax.InputSource;

import com.quadcap.http.util.HeaderParser;

import com.quadcap.util.collections.ArrayQueue;

import com.quadcap.util.text.OctetMap;
import com.quadcap.util.text.Scanner;

import com.quadcap.util.Debug;
import com.quadcap.util.Util;

import com.quadcap.io.IO;
import com.quadcap.io.LimitedInputStream;
import com.quadcap.io.NullOutputStream;

public class HttpFetcher {
    static boolean checkLinks            = false;
    static boolean showResponseHeaders               = false;

    static byte[] delims = { 0x0d, 0x0a, 0x0d, 0x0a };

    public static byte[] fetch(String url) throws Exception {
        return fetch(url, new ArrayList());
    }
    
    public static byte[] fetch(String url, List headers) throws Exception {
        InputStream is = fetchStream(url, headers);
        byte[] doc = readStream(is);
        is.close();
        return doc;
    }

    public static byte[] post(String url, String fileName,
                              List headers) throws Exception {
        InputStream is = postStream(url, fileName, headers);
        byte[] doc = readStream(is);
        is.close();
        return doc;
        
    }

    public static InputStream postStream(String url, String fileName,
                                         List headers) throws Exception {
        if (url.indexOf("http://") != 0) {
            System.err.println("Bad url (protocol): " + url);
            return null;
        }
        url = url.substring(7);
        int idx = url.indexOf('/');
        if (idx <= 0) {
            url = url + "/";
            idx = url.indexOf('/');
        }
        String host = url.substring(0, idx);
        String name = url.substring(idx);
        int port = 80;
        idx = host.indexOf(':');
        if (idx >= 0) {
            port = Integer.parseInt(host.substring(idx+1));
            host = host.substring(0, idx);
        }
        Socket s = new Socket(host, port);

        headers.add("Content-Length: " + 
                    String.valueOf(new File(fileName).length()));

        OutputStream sos = s.getOutputStream();
        BufferedOutputStream os = new BufferedOutputStream(sos);
        os.write(("POST " + name + " HTTP/1.0\r\n").getBytes());
        Iterator iter = headers.iterator();
        while (iter.hasNext()) {
            IO.write(os, iter.next().toString());
            os.write("\r\n".getBytes());
        }
        os.write("\r\n".getBytes());
                     
        FileInputStream fis = new FileInputStream(fileName);
        IO.copyStream(fis, os);
        os.flush();

        InputStream is = s.getInputStream();
        return new BufferedInputStream(is);
    }
    
    public static InputStream fetchStream(String url) throws Exception {
        return fetchStream(url, new ArrayList());
    }

    public static void ripPlayList(InputStream is, List headers)
        throws Exception
    {
        BufferedReader br = new BufferedReader(
            new InputStreamReader(is));
        String line;
        while ((line = br.readLine()) != null) {
            String[] v = line.split("=");
            if (v.length == 2 && v[0].equals("File1")) {
                is.close();
                ripStream(v[1], headers);
                return;
            }
        }
        is.close();
    }
    
    public static void ripStream(String url, List headers)
        throws Exception
    {
        InputStream is = fetchStream(url, headers);
        StringBuffer sb = new StringBuffer();
        for (int c = is.read(); c != '\n'; c = is.read()) {
            sb.append((char)c);
        }
        Debug.println("ripStream(" + url + "), Response: " + sb);
        Map hdrs = HeaderParser.parseHeaders(is);
        Debug.println("Headers = " + hdrs);
        if (sb.toString().indexOf("302") > 0) {
            is.close();
            Debug.println("redirecting to: " + hdrs.get("location"));
            ripStream(hdrs.get("location").toString(), headers);
            return;
        }
        String contentType = String.valueOf(hdrs.get("content-type"));
        if (contentType.equalsIgnoreCase("audio/x-scpls")) {
            ripPlayList(is, headers);
            return;
        }
        int metaInt = 0;
        try {
            metaInt = Integer.parseInt(String.valueOf(hdrs.get("icy-metaint")));
        } catch (Throwable t) {
        }
        byte[] buf = new byte[metaInt];
        String title = null;
        String lastTitle = "__INVALID_lastTitle__";
        Mp3FrameStream out = new Mp3FrameStream();
        FileOutputStream fout = null;
        while (true) {
            int cnt = is.read(buf);
            while (cnt < buf.length) {
                if (cnt < 0) {
                    if (out != null) {
                        out.close();
                    }
                    Debug.println("partial buffer, returning... (" + cnt + ")");
                    return;
                }
                cnt += is.read(buf, cnt, buf.length - cnt);
            }
            title = getTitle(is).replace('/', ' ');
            if (title.length() > 0) {
                if (!title.equals(lastTitle)) {
                    Debug.println("Title: " + title);
                    if (fout == null) {
                        // Start a new file
                        fout = new FileOutputStream(title);
                        out.init(fout, new NullOutputStream());
                        out.write(buf);
                    } else {
                        // split the difference
                        out.write(buf, 0, metaInt/2);
                        out.close();
                        fout = new FileOutputStream(title);
                        out.init(fout, new NullOutputStream());
                        out.write(buf, metaInt/2, metaInt/2);
                    }
                    lastTitle = title;
                }
            } else {
                if (fout != null) {
                    out.write(buf);
                }
            }
        }
    }

    public static String getTitle(InputStream in) throws IOException {
        byte[] buf = new byte[in.read() * 16];
        in.read(buf);
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < buf.length && buf[i] != 0; i++) {
            sb.append((char)(buf[i]));
        }
        String[] p = sb.toString().split(";");
        for (int i = 0; i < p.length; i++) {
            String[] v = p[i].trim().split("=");
            if (v.length == 2 && v[0].equalsIgnoreCase("StreamTitle")) {
                String s = v[1].substring(1, v[1].length()-1);
                while (s.toLowerCase().endsWith(".mp3")) {
                    s = s.substring(0, s.length() - 4);
                }
                s += ".mp3";
                return s;
            }
        }
        return "";
    }
    
    public static InputStream fetchStream(String url, List headers)
        throws IOException
    {
        //Debug.println(0, "Fetch: " + url);
        if (url.indexOf("http://") != 0) {
            System.err.println("Bad url (protocol): " + url);
            return null;
        }
        url = url.substring(7);
        int idx = url.indexOf('/');
        if (idx <= 0) {
            url = url + "/";
            idx = url.length() - 1;
        }
        String host = url.substring(0, idx);
        String name = url.substring(idx);
        int port = 80;
        idx = host.indexOf(':');
        if (idx >= 0) {
            port = Integer.parseInt(host.substring(idx+1));
            host = host.substring(0, idx);
        }
        Socket s = new Socket(host, port);

        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        Debug.println("GET " + name);
        bos.write(("GET " + name + " HTTP/1.0\r\n").getBytes());
        Iterator iter = headers.iterator();
        while (iter.hasNext()) {
            String hdr = iter.next().toString();
            IO.write(bos, hdr);
            Debug.println("  " + hdr);
            bos.write('\r');
            bos.write('\n');
        }
        bos.write("\r\n".getBytes());

        s.getOutputStream().write(bos.toByteArray());

        InputStream is = s.getInputStream();
        return new BufferedInputStream(is);
    }

    public static InputStream fetch2(String url) throws Exception {
        System.out.println("Fetch: " + url);
        URLConnection c = new URL(url).openConnection();
        c.connect();
        return c.getInputStream();
    }

    public static byte[] readStream(InputStream is) throws IOException {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();

        int state = 0;
        int cnt = 0;
        if (showResponseHeaders) state = 5;
        while (state < 4) {
            int c = is.read();
            if (c < 0) {
                throw new IOException("unexpected eof in message headers");
            }
            if (delims[state] == c) state++;
            else if (delims[0] == c) state = 1;
            else state = 0;
        }

        byte[] buf = new byte[1024];
        while ((cnt = is.read(buf)) > 0) {
            bos.write(buf, 0, cnt);
        }
        return bos.toByteArray();
    }

    public static Hashtable buildTable(String fname) throws Exception {
        BufferedReader r = new BufferedReader(new FileReader(fname));
        String turl;
        Hashtable t = new Hashtable();
        while ((turl = r.readLine()) != null) {
            url = turl;
            byte[] doc = fetch(url);
            System.err.println(url + ": " + Util.strBytes(doc));
            t.put(url, doc);
        }
        return t;
    }
        
    public static void checkTable(String fname, Hashtable t) throws Exception {
        BufferedReader r = new BufferedReader(new FileReader(fname));
        String url;
        while ((url = r.readLine()) != null) {
            try {
                byte[] doc = fetch(url);
                byte[] exp = (byte[])t.get(url);
                if (Util.compareBytes(doc, exp) != 0) {
                    System.err.println("Failed: " + url);
                    System.err.println("Doc: " + Util.strBytes(doc));
                }
            } catch (Exception e) {
                Debug.print(e);
            }
        }
    }

    public static void addAV(String fname) throws Exception {
        BufferedReader r = new BufferedReader(new FileReader(fname));
        String url;
        Hashtable t = new Hashtable();
        while ((url = r.readLine()) != null) {
            System.out.println("url: " + url);
            StringBuffer sb = new StringBuffer(
                "http://add-url.altavista.com/cgi-bin/newurl?ad=1&q=");
            sb.append(URLEncoder.encode(url));
            byte[] doc = fetch(sb.toString());
            String s = new String(doc);
            if (s.indexOf("The page was fetched") < 0) {
                System.out.println(s);
                break;
            }
            try { Thread.sleep(2000); } catch (Throwable dt) {}
        }
    }   
        
    public static void check(String name) throws Exception {
        LinkChecker lc = new LinkChecker(name);
        lc.run();
        lc.printBadLinks();
    }
    
    public static void main(String args[]) {
        Debug.debugMode = Debug.debugAll;
        Debug.debugStream = System.out;
        try {
            doit(args);
        } catch (Exception e) {
            System.out.println("Exception: " + e.toString());
            Debug.print(e);
        }
    }

    static String fname = null;
    static int repeat = 2;
    static int delay = 0;
    static Hashtable t;
    static boolean times = false;
    static boolean rip = false;
    static String url = null;
    static int limit = 0;

    public static void doit() throws Exception {
        if (times) {
            for (int i = 0; i < repeat; i++) {
                fetch(url);
            }
        } else {
            for (int i = 0; i < repeat; i++) {
                checkTable(fname, t);
                if (delay > 0) Thread.sleep(delay);
            }
        }
    }
    
    public static void doit (String args[]) throws Exception {
        int numThreads = 1;
        boolean av = false;
        String post = null;
        List headers = new ArrayList();

        int ac = 0;
        while (ac < args.length) {
            String arg = args[ac].trim();
            if (arg.charAt(0) != '-') break;
            ac++;
            if (arg.equals("-urls")) {
                fname = args[ac++];
            } else if (arg.equals("-count")) {
                repeat = Integer.parseInt(args[ac++]);
            } else if (arg.equals("-delay")) {
                delay = Integer.parseInt(args[ac++]);
            } else if (arg.equals("-threads")) {
                numThreads = Integer.parseInt(args[ac++]);
            } else if (arg.equals("-checklinks")) {
                checkLinks = true;
                fname = args[ac++];
            } else if (arg.equals("-headers")) {
                showResponseHeaders = true;
            } else if (arg.equals("-post")) {
                post = args[ac++];
            } else if (arg.equals("-header")) {
                String hName = args[ac++];
                String hVal = args[ac++];
                headers.add(hName + ": " + hVal);
            } else if (arg.equals("-altavista")) {
                av = true;
                fname = args[ac++];
            } else if (arg.equals("-times")) {
                times = true;
            } else if (arg.equals("-limit")) {
                limit = Integer.parseInt(args[ac++]);
            } else if (arg.equals("-rip")) {
                if (!rip) {
                    headers.add("Host: 192.168.1.8");
                    headers.add("User-Agent: WinampMPEG/2.8");
                    headers.add("Accept: */*");
                    headers.add("Icy-Metadata:1");
                    headers.add("Connection: close");
                }
                rip = true;
            } else {
                //throw new Exception("??");
            }
        }

        if (av) {
            addAV(fname);
        } else if (checkLinks) {
            check(fname);
            return;
        } else if (fname == null && !times) {
            url = args[ac];
            if (rip) {
                ripStream(url, headers);
                return;
            }
            if (post == null) {
                InputStream in = fetchStream(url, headers);
                if (limit > 0) {
                    in = new LimitedInputStream(in, limit);
                }
                try {
                    IO.copyStream(in, System.out);
                } finally {
                    in.close();
                }
            } else {
                byte[] doc = post(url, post, headers);
                System.out.write(doc);
            }
        } else {
            t = buildTable(fname);
            Thread[] threads = new Thread[numThreads];
            for (int i = 0; i < numThreads; i++) {
                threads[i] = new Thread() {
                    public void run() {
                        try {
                            doit();
                        } catch (Throwable t) {
                            Debug.print(t);
                        }
                    }
                };
            }
            long start = System.currentTimeMillis();
            for (int i = 0; i < numThreads; i++) {
                threads[i].start();
            }
            for (int i = 0; i < numThreads; i++) {
                try {
                    threads[i].join();
                } catch (Throwable t) {
                    Debug.print(t);
                }
            }
            long stop = System.currentTimeMillis();
            long elap = stop - start;
            int r_s = (int)((repeat * numThreads * 1000) / elap);
            if (times) {
                System.out.println("" + elap + " elapsed");
                System.out.println("" + r_s + " requests/second");
            }
        }
    }
}