/*
 * CSpider.js
 * $Revision: 1.4 $ $Date: 2003/07/07 18:35:24 $
 */

/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is Netscape code.
 *
 * The Initial Developer of the Original Code is
 * Netscape Corporation.
 * Portions created by the Initial Developer are Copyright (C) 2003
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s): Bob Clary <bclary@netscape.com>
 *
 * ***** END LICENSE BLOCK ***** */

/*
  CSpider

  aUrl     url of the first page to visit
  aDepth   depth in terms of pages to visit
  aPageLoader WDocumentLoader to load pages
  aRestrictUrl restrict spider to follow links that are prefixed by aUrl
*/

gCSpiderOnLoadCallbackDelay = 20;

function CSpider(/* String */ aUrl, 
                 /* Boolean */ aRestrictUrl,
                 /* Number */ aDepth, 
                 /* WDocumentLoader */ aPageLoader,
                 /* Seconds */ aOnLoadTimeoutInterval)
{
  this.mUrl    = aUrl; 
  this.mRestrictUrl = aRestrictUrl;
  this.mDepth  = aDepth;
  this.mPageLoader = aPageLoader;
  this.mCallWrapperOnLoadPage = null;
  this.mCallWrapperOnLoadPageTimeout = null;
  this.mCallWrapperLoadPage = null;
  this.mCallWrapperPause = null;
  this.mOnLoadTimeoutInterval = (aOnLoadTimeoutInterval || 60) * 1000;
  this.init(aUrl);

}

CSpider.handlePageLoad = function(/* CFormData */ loaderFormData)
{
  var spiderid = loaderFormData.getValue('spiderid');
  var callwrapper = CCallWrapper.mPendingCalls[spiderid];
  // if call wrapper is not defined, the page load was cancelled
  // and the WDocumentLoader is calling us
  if (callwrapper) 
  {
    callwrapper.execute();
  }
}

CSpider.prototype.init = function(aUrl)
{
  this.mPagesVisited  = [];
  this.mPagesPending  = [{ mDepth: 0, mUrl: aUrl }];
  this.mPageHash = {};
  this.mPageHash[aUrl] = true;
  this.mState = 'initialized';
};

CSpider.prototype.run =
function()
{
  if (this.mDepth > -1)
  {
    this.init(this.mUrl);
    this.mState = 'running';
    if (!this.mOnStart())
    {
      this.mState = 'initialized';
      return;
    }
    this.loadPage();
  }
};

CSpider.prototype.restart =
function()
{
  if (this.mState == 'paused')
  {
    this.mState = 'running';
    if(!this.mOnRestart())
    {
      this.mState = 'paused';
      return;
    }
    this.loadPage();
  }
};

CSpider.prototype.pause =
function()
{
  this.mState = 'pausing';

  if (this.mCallWrapperOnLoadPage || 
      this.mCallWrapperOnLoadPageTimeout)
  {
    this.mCallWrapperPause = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'pause');
    CCallWrapper.asyncExecute(this.mCallWrapperPause);
  }
  else
  {
    this.mCallWrapperPause = null;
    this.mState = 'paused';
    this.mOnPause();
  }
};

CSpider.prototype.stop =
function ()
{
  this.mState = 'stopped';

  if (this.mCallWrapperOnLoadPage && this.mCallWrapperOnLoadPage.mTimerId)
  {
    this.mCallWrapperOnLoadPage.cancel();
    this.mCallWrapperOnLoadPage = null;
  }
  
  if (this.mCallWrapperOnLoadPageTimeout && this.mCallWrapperOnLoadPageTimeout.mTimerId)
  {
    this.mCallWrapperOnLoadPageTimeout.cancel();
    this.mCallWrapperOnLoadPageTimeout = null;
  }
  
  if (this.mCallWrapperLoadPage && this.mCallWrapperLoadPage.mTimerId)
  {
    this.mCallWrapperLoadPage.cancel();
    this.mCallWrapperLoadPage = null;
  }
  
  this.mOnStop();
};

CSpider.prototype.mOnStart = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnBeforePage = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnAfterPage = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnPageTimeout = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnStop = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnPause = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnRestart = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.addPage =
function(href)
{
  if (!href)
  {
    return;
  }

  if (href.indexOf('http') != 0)
  {
    return;
  }

  if (this.mRestrictUrl && href.indexOf(this.mUrl) != 0)
  {
    return;
  }

  if (href.search(/\.zip$/) != -1 || href.search(/\.gz$/) != -1 || href.search(/\.tar$/) != -1)
  {
    return;
  }

  var hashIndex = href.indexOf('#');
  if (hashIndex != -1)
  {
    href = href.substr(0, hashIndex);
  }

  if (this.mPageHash[href])
  {
    return;
  }

  if (this.mCurrentUrl.mDepth + 1 <= this.mDepth)
  {
    this.mPageHash[href] = true;
    this.mPagesVisited.push(href);
    this.mPagesPending.push( { mDepth: this.mCurrentUrl.mDepth + 1, mUrl: href } );
  }
};

CSpider.prototype.loadPage = 
function ()
{
  if (this.mState != 'running')
  {
    this.mCallWrapperLoadPage.cancel();
    this.mCallWrapperLoadPage = null;
    return;
  }

  this.mCurrentUrl = this.mPagesPending.pop();
  
  if (!this.mCurrentUrl)
  {
    this.stop();
    return;
  }

  if (this.mCurrentUrl.mDepth > this.mDepth)
  {
    this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
    CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
    return;
  }

  if (!this.mOnBeforePage())
  {
    this.mState = 'paused';
    return;
  }

  this.mCallWrapperOnLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'onLoadPage');
  var loaderUrl = '?callbackfunction=CSpider.handlePageLoad&amp;spiderid=' + this.mCallWrapperOnLoadPage.mId + '&amp;file=' + escape(this.mCurrentUrl.mUrl);
  this.mPageLoader.location.search = loaderUrl;

  this.mCallWrapperOnLoadPageTimeout = new CCallWrapper(this, this.mOnLoadTimeoutInterval, 'onLoadPageTimeout');
  CCallWrapper.asyncExecute(this.mCallWrapperOnLoadPageTimeout);
};

CSpider.prototype.onLoadPageTimeout = 
function ()
{
  if (this.mCallWrapperOnLoadPage)
  {
    this.mCallWrapperOnLoadPage.cancel();
    this.mCallWrapperOnLoadPage = null;
  }

  this.mCallWrapperOnLoadPageTimeout = null;

  if (!this.mOnPageTimeout())
  {
    this.pause();
    return;
  }

  // this.loadPage();

  this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
  CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
};

CSpider.prototype.onLoadPage = 
function ()
{
  this.mCallWrapperOnLoadPage = null;

  if (this.mCallWrapperOnLoadPageTimeout)
  {
    this.mCallWrapperOnLoadPageTimeout.cancel();
    this.mCallWrapperOnLoadPageTimeout = null;
  }

  if (this.mState == 'stopped')
  {
    return;
  }

  if (navigator.product == 'Gecko' && this.mUrl.indexOf(document.location.host) == -1)
  {
    netscape.security.PrivilegeManager.enablePrivilege('UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite');
  }

  this.mDocument = this.mPageLoader.frames[0].document;

  var i;
  var links  = this.mDocument.links;
  if (links)
  {
    var length = links.length;
    var href;

    for (i = 0; i < length; ++i)
    {
      href = links[i].href;
      if (href)
      {
        this.addPage(href);
      }
    }

    links = this.mDocument.getElementsByTagName('frame');
    length = links.length;

    for (i = 0; i < length; ++i)
    {
      href = links[i].src;
      if (href)
      {
        this.addPage(href);
      }
    }

    links = this.mDocument.getElementsByTagName('iframe');
    length = links.length;

    for (i = 0; i < length; ++i)
    {
      href = links[i].src;
      if (href)
      {
        this.addPage(href);
      }
    }
  }

  if (!this.mOnAfterPage())
  {
    this.mState = 'paused';
    return;
  }

  //this.loadPage();

  this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
  CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
};