/*** Abstract class for crawl-global registry of CrawlServer (host:port) and* CrawlHost (hostname) objects.*/
public abstract class ServerCache {public abstract CrawlHost getHostFor(String host);public abstract CrawlServer getServerFor(String serverKey);/*** Utility for performing an action on every CrawlHost. * * @param action 1-argument Closure to apply to each CrawlHost*/public abstract void forAllHostsDo(Closure action);private static Logger logger =Logger.getLogger(ServerCache.class.getName());/*** Get the {@link CrawlHost} associated with <code>curi</code>.* @param uuri CandidateURI we're to return Host for.* @return CandidateURI instance that matches the passed Host name.*/public CrawlHost getHostFor(UURI uuri) {CrawlHost h = null;try {if (uuri.getScheme().equals("dns")) {h = getHostFor("dns:");} else if (uuri.getScheme().equals("whois")) {h = getHostFor("whois:");} else {h = getHostFor(uuri.getReferencedHost());}} catch (URIException e) {logger.log(Level.SEVERE, uuri.toString(), e);}return h;}/*** Get the {@link CrawlServer} associated with <code>curi</code>.* @param uuri CandidateURI we're to get server from.* @return CrawlServer instance that matches the passed CandidateURI.*/public CrawlServer getServerFor(UURI uuri) {//System.out.println("classname:"+this.getClass().getName());CrawlServer cs = null;try {String key = CrawlServer.getServerKey(uuri);// TODOSOMEDAY: make this robust against those rare cases// where authority is not a hostname.if (key != null) {cs = getServerFor(key);}} catch (URIException e) {logger.log(Level.FINE, "No server key obtainable: "+uuri.toString(), e);} catch (NullPointerException npe) {logger.log(Level.FINE, "No server key obtainable: "+uuri.toString(), npe);}return cs;}abstract public Set<String> hostKeys();
}

*** Server and Host cache.* @author stack* @version $Date: 2011-01-20 23:28:38 +0000 (Thu, 20 Jan 2011) $, $Revision: 7067 $*/
public class DefaultServerCache extends ServerCache implements Closeable, Serializable {private static final long serialVersionUID = 1L;@SuppressWarnings("unused")private static Logger logger =Logger.getLogger(DefaultServerCache.class.getName());/*** hostname[:port] -> CrawlServer.* Set in the initialization.*/protected ObjectIdentityCache<CrawlServer> servers = null;/*** hostname -> CrawlHost.* Set in the initialization.*/protected ObjectIdentityCache<CrawlHost> hosts = null;/*** Constructor.*/public DefaultServerCache() {this(new ObjectIdentityMemCache<CrawlServer>(), new ObjectIdentityMemCache<CrawlHost>());}public DefaultServerCache(ObjectIdentityCache<CrawlServer> servers, ObjectIdentityCache<CrawlHost> hosts) {this.servers = servers;this.hosts = hosts;}/*** Get the {@link CrawlServer} associated with <code>name</code>.* @param serverKey Server name we're to return server for.* @return CrawlServer instance that matches the passed server name.*/public CrawlServer getServerFor(final String serverKey) {CrawlServer cserver = servers.getOrUse(serverKey,new Supplier<CrawlServer>() {public CrawlServer get() {String skey = new String(serverKey); // ensure private minimal keyreturn new CrawlServer(skey);}});return cserver;}/*** Get the {@link CrawlHost} associated with <code>name</code>.* @param hostname Host name we're to return Host for.* @return CrawlHost instance that matches the passed Host name.*/public CrawlHost getHostFor(final String hostname) {if (hostname == null || hostname.length() == 0) {return null;}CrawlHost host = hosts.getOrUse(hostname,new Supplier<CrawlHost>() {public CrawlHost get() {String hkey = new String(hostname); // ensure private minimal keyreturn new CrawlHost(hkey);}});return host;}/*** @param serverKey Key to use doing lookup.* @return True if a server instance exists.*/public boolean containsServer(String serverKey) {return (CrawlServer) servers.get(serverKey) != null; }/*** @param hostKey Key to use doing lookup.* @return True if a host instance exists.*/public boolean containsHost(String hostKey) {return (CrawlHost) hosts.get(hostKey) != null; }/*** Called when shutting down the cache so we can do clean up.*/public void close() {if (this.hosts != null) {// If we're using a bdb bigmap, the call to clear will// close down the bdb database.this.hosts.close();this.hosts = null;}if (this.servers != null) { this.servers.close();this.servers = null;}}/*** NOTE: Should not mutate the CrawlHost instance so retrieved; depending on* the hostscache implementation, the change may not be reliably persistent.  * * @see org.archive.modules.net.ServerCache#forAllHostsDo(org.apache.commons.collections.Closure)*/public void forAllHostsDo(Closure c) {for(String host : hosts.keySet()) {c.execute(hosts.get(host));}}public Set<String> hostKeys() {return hosts.keySet();}
}

/*** ServerCache backed by BDB big maps; the usual choice for crawls.* * @contributor pjack* @contributor gojomo*/
public class BdbServerCache extends DefaultServerCache
implements Lifecycle {private static final long serialVersionUID = 1L;protected BdbModule bdb;@Autowiredpublic void setBdbModule(BdbModule bdb) {this.bdb = bdb;}public BdbServerCache() {}public void start() {if(isRunning()) {return;}try {this.servers = bdb.getObjectCache("servers", false, CrawlServer.class, CrawlServer.class);this.hosts = bdb.getObjectCache("hosts", false, CrawlHost.class, CrawlHost.class);} catch (DatabaseException e) {throw new IllegalStateException(e);}isRunning = true;}boolean isRunning = false; public boolean isRunning() {return isRunning;}public void stop() {isRunning = false; // TODO: release bigmaps?
    }
}