at.molindo.webtools.crawler

Class Crawler

java.lang.Object

java.util.Observable

at.molindo.webtools.crawler.Crawler

public class Crawler
extends Observable

Field Summary

Fields

Modifier and Type	Field and Description
static Object	FINISH

Constructor Summary

Constructors

Constructor and Description
Crawler(String host, String start, int threads, int max, boolean tidy)
Crawler(String host, String username, String password, String start, int threads, int max, boolean tidy)

Method Summary

Modifier and Type	Method and Description
void	awaitTermination(long timeout, TimeUnit unit)
int	getDispatchedCount()
DTDMemoryCache	getDtdMemoryCache()
List<ICrawlerFilter>	getFilters()
String	getHost()
SAXParserFactory	getParserFactory()
String	getPassword()
int	getRetrievedCount()
String	getUsername()
Map<String,CrawlerResult>	getVisitedURLs()
static void	main(String[] args)
protected BlockingQueue<Runnable>	newBlockingQueue()
protected ICrawlerHistory	newCrawlerHistory()
protected CrawlerTask	newCrawlerTask(String url, CrawlerReferrer referrer, boolean tidy)
void	queue(String url, CrawlerReferrer referrer)
protected void	report(CrawlerResult result)
void	shutdown()

Methods inherited from class java.util.Observable

addObserver, clearChanged, countObservers, deleteObserver, deleteObservers, hasChanged, notifyObservers, notifyObservers, setChanged

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

FINISH

public static final Object FINISH

Constructor Detail

Crawler

public Crawler(String host,
               String start,
               int threads,
               int max,
               boolean tidy)

Crawler

public Crawler(String host,
               String username,
               String password,
               String start,
               int threads,
               int max,
               boolean tidy)

Method Detail

newBlockingQueue

protected BlockingQueue<Runnable> newBlockingQueue()

newCrawlerHistory

protected ICrawlerHistory newCrawlerHistory()

queue

public void queue(String url,
                  CrawlerReferrer referrer)

newCrawlerTask

protected CrawlerTask newCrawlerTask(String url,
                                     CrawlerReferrer referrer,
                                     boolean tidy)

report

protected void report(CrawlerResult result)

getVisitedURLs

public Map<String,CrawlerResult> getVisitedURLs()

getDispatchedCount

public int getDispatchedCount()

getRetrievedCount

public int getRetrievedCount()

getFilters

public List<ICrawlerFilter> getFilters()

getHost

public String getHost()

getParserFactory

public SAXParserFactory getParserFactory()

getDtdMemoryCache

public DTDMemoryCache getDtdMemoryCache()

getUsername

public String getUsername()

getPassword

public String getPassword()

shutdown

public void shutdown()

awaitTermination

public void awaitTermination(long timeout,
                             TimeUnit unit)
                      throws InterruptedException

Throws:

InterruptedException

main

public static void main(String[] args)
                 throws InterruptedException

Throws:

InterruptedException