#original from Ben Hamilton benTHamilton@hotmail.com
#changed some portions as needed

import re,urllib2,getpass,MySQLdb

#now it seraches for sites (previously searched for media)
REFS="(www\.\S+(\.org|\.net|\.com|\.in|\.uk))(?=/)"  
#[(?=/)|(?=\')|(?=\")]{1}

availRefs=[]
tracedRefs=[]
#for keeping track of the sites traced in files
fTraced=None
fTracing=None
conCrawl=None
rstCrawl=None

#get the references
def getHrefs(lines=[]):
    global availRefs,tracedRefs,fTracing

    for line in lines:
        #search for reference
	getRef=re.search(REFS,line)

	if getRef is not None and getRef!="":
            #to get the main site name only
            
strTemp=str(getRef.string[getRef.start():getRef.end()].split('/')[0]).lower().strip()
            #check if already traced
    	    if strTemp not in tracedRefs and strTemp not in availRefs:
		availRefs.append(strTemp)
		#log to file
		fTracing.write(strTemp+"\n")
		fTracing.flush()
		print "Adding reference %s" %strTemp

#for adding proxy support
def initNet():
    proxy="proxyname:port"
    user=raw_input("Enter the user name for proxy : ")
    passw=getpass.getpass("Password for proxy :")
    proxySetting='http://'+user+':'+passw+'@'+proxy
    proxy_support=urllib2.ProxyHandler({"http":proxySetting})
    authinfo=urllib2.HTTPBasicAuthHandler()
    opener=urllib2.build_opener(proxy_support,authinfo,urllib2.HTTPHandler)
    urllib2.install_opener(opener)

#crawl the net
def crawlNet():
    global availRefs,tracedRefs,fTraced,rstCrawl
    #initialize the proxy
    #if direct connected then remove this and use urllib instead
    initNet()

    while len(availRefs)>0:
        #check if already traced
        if availRefs[0].strip() in tracedRefs:
            print "%s already traced..." %availRefs[0]
            del availRefs[0]
            continue

        else:
            try:
                print "Opening connection to %s" %(availRefs[0])
                siteSock=urllib2.urlopen("http://"+availRefs[0])
            except urllib2.HTTPError,e:
                print "Unable to connect to %s .Error occurred :%s" 
%(availRefs[0],str(e))
                #on error also it writes in the traced log
                fTraced.write(availRefs[0]+"\n")
                del availRefs[0]
                continue
            except urllib2.URLError,e:
                print "Unable to connect to %s .Error occurred :%s" 
%(availRefs[0],str(e))
                fTraced.write(availRefs[0]+"\n")
                del availRefs[0]
                continue
            except Exception,(num,e):
                if num==10061:
                    print "Proxy down..Trying to reconnect..."
                    continue
                else:
                    fTraced.write(availRefs[0]+"\n")
                    del availRefs[0]
                    continue
            else:
                htmlText=siteSock.readlines()
                siteSock.close()
                print "Tracing %s" %availRefs[0]
                fTraced.write(availRefs[0]+"\n")
                fTraced.flush()
                tracedRefs.append(availRefs[0])
                #just trying to insert html files in the database
                print "Insert data for %s..." %availRefs[0]
                txtHtml=""
                txtHtml=txtHtml.join(htmlText)
                strQry="insert into crawl_site(site,main_page) 
values('%s',%s)" %(availRefs[0],txtHtml)
                try:
                    rstCrawl.execute(strQry)
                except MySQLdb.OperationalError,(errno,errMsg):
                    print "Error occurred while inserting the record for 
%s..." %availRefs[0]
                    print errNo,":",errMsg
                    break
                getHrefs(htmlText)
                del availRefs[0]
    fTraced.close()
    fTracing.close()

def startCrawler():
    global tracedRefs,fTraced,fTracing,conCrawl,rstCrawl

    dbUser=raw_input("Please enter the user name for database : ")
    dbPass=getpass.getpass("Please enter the password for database : ")
    try:
        
conCrawl=MySQLdb.connect(host='localhost',db='spy',user=dbUser,passwd=dbPass)
    except MySQLdb.OperationalError,(errNo,errMsg):
        print "Error occurred while connecting with crawl database..."
        print errNo,":",errMsg
        return

    rstCrawl=conCrawl.cursor()

    tempSiteList=[]
    #now get already traced site list
    print "Getting already traced site list..."
    try:
        fTraced=open("tracedSite.log","r")
    except IOError,e:
        print "Unable to get the already traced site list..\nCreating a new 
file..."
        traceRefs=[]
    else:
        tracedRefs=fTraced.readlines()
        fTraced.close()
    #get already recieved links
    print "Getting already generated to be traced site list..."
    try:
        fTracing=open("traceInProgress.log","r")
    except IOError,e:
        print "Unable to get the already generated to be traced site 
list..\nCreating a new file...\n"
    else:
        tempSiteList=fTracing.readlines()
        fTracing.close()
    #file containing the initial list of sites to be traced
    print "Getting initial sites list...\n"
    fInit=open("initialSite.log","r")
    initSites=fInit.readlines()
    fInit.close()

    availRefs.extend(initSites)
    availRefs.extend(tempSiteList)

    siteCount=0
    for site in tracedRefs:
        tracedRefs[siteCount]=tracedRefs[siteCount].strip()
        siteCount+=1

    siteCount=0
    for site in availRefs:
        availRefs[siteCount]=availRefs[siteCount].strip()
        siteCount+=1

    siteCount=0
    for site in availRefs:
        if availRefs[siteCount] in tracedRefs:
           print "Already traced : ",availRefs[siteCount]
           del availRefs[siteCount]
        else:
            siteCount+=1

    fTraced=open("tracedSite.log","a")
    fTracing=open("traceInProgress.log","a")

    print "\t Starting web crawler...\n"


    crawlNet()

if __name__=="__main__":

    startCrawler()

