本文共 9033 字，大约阅读时间需要 30 分钟。

最近公司网站搞检查，发现了一些问题，一直在用的是xenu工具，速度快，小巧（写了这么多年的MFC,真的很亲切啊，呵呵）

刚好这2天才学习python，所以自己写了一个spider，逐渐也对python有了一些了解，下面把源码分享出来，大家可以玩玩看

文件是utf-8格式，但如果加了中文注释，竟然不能debug了。。。无论你是不是在文件头2行加了#encoding=utf-8等方法，有人解决了告诉我一下哦

用python实现的网站spider1.功能：检查死链和空title，链接目前只查属于站内的（包括主站和二级站,并且会过滤重复的URL，防止一个URL多次扫描）2.当页面中有一些错误时，也会提供出错原因和具体位置,所以也可以检查一下网站是否标准3.目前只查页面，不查资源，过滤了 .jpg|.jpeg|.avi|.png|.gif4.依赖：必须安装python-3.2rc1（将来也有可能是正式版，可以自己改test.bat中的python的路径，不过不能装低于这个版本的，3.1版本loggingRotating方式有bug）5.有些不能查出的死链：有时候死链会被重定向到别的页面，如果中国电信会直接跳到它的页面，此情况暂时无法处理，在公司测试没有此现象，adsl上网会这样6.检查网页是否有语法错误，spider.py中except HTMLParseError as e:# log.error("HTMLParseError : msg= %s, url= %s, pos= %s" % ( e.msg, url, lParser.getpos())) pass（前提是你已经了解了python的缩进语法）去掉第2行首的“#”即可，目前由于不少网页都有好些小问题，所以默认不开放7.这是方便检查公司网站的小工具，在公司的几个项目网站上测试可用，不过没有在其他大型网站做过性能和兼容性测试，如果你有兴趣，源码都在里面，自己动手玩玩吧8.如果有bug，请反馈给我，也可以自己debug找一下(我的开发环境eclipse + pydev)出错说明：1.URL出错，一般是死链接[2011-02-09 11:12:11,392] ERROR : URLError : http://Hardware.global-supplier.com[2011-02-09 15:37:57,148] ERROR : HTTPError : code=400, msg= Bad Request, url= http://www.fashionalgifts.com/../company_info.html2.页面解析时出错，下例是tag中内容有问题，pos代表页面的出错位置[2011-02-09 11:12:17,119] ERROR : msg= malformed start tag, HTMLParseError : http://fashion.global-supplier.com/cosmetic-bags/pu-cosmetic-bag-gb-c09.html pos= (238, 27)3.urlJoinError，一般是相对路径使用错误导致的，如下，第一个是页面URL，第二个是href中的内容，一级子目录竟然要返回上2级的目录UrlJoin Error:http://www.fashionalgifts.com/company-culture.html/, ../../company_info.html不过这样的URL在浏览器中是可以正常执行的。。。太强悍了，帮我们屏蔽错误4.空titletitle is null, url=*** 

spider.py(主程序)

#Author:Paul Wang#Date:2011-01-26#Description:from html.parser import HTMLParserfrom html.parser import HTMLParseErrorimport urllib.requestimport myLoggerimport myHtmlParseimport sysdebugMode = 1site = "http://www.geekzones.com/"charset = 'utf-8'logLever = 'debuG'try: def useage(): print(" python spider.py siteUrl charSet logLever") print(" example: python spider.py http://www.baidu.com utf-8 debug") def getInnerURL(url,parentUrl,siteName,log): global times global urlList lParser = myHtmlParse.MyParser(log) try: try: if(url[-1] != '/'): url += '/' if url in urlList or len(url) == 0: return times += 1 urlList.append(url) log.info('times = %d open url : %s' % (times , url)) lParser.currentLink = url; lParser.siteName = siteName; opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] htmlSource2 = opener.open(url).read() # req = urllib.request.Request(url) # htmlSource2 = urllib.request.urlopen(req).read() lParser.feed(htmlSource2.decode(charset)) lParser.filterRepeatLink() if(len(lParser.title) == 0): log.error('title is null, url= %s' % url) except urllib.error.HTTPError as e: log.error("HTTPError : code=%d, msg= %s, url= %s, parentUrl=%s" % ( e.code, e.msg, url, parentUrl)) except urllib.error.URLError as e: log.error("URLError : url= %s" % ( url)) except HTMLParseError as e: log.error("HTMLParseError : msg= %s, url= %s, pos= %s, parentUrl=%s" % ( e.msg, url, lParser.getpos(), parentUrl)) finally: lParser.close() for url1 in lParser.link: if url1 in urlList or len(url1) == 0: continue getInnerURL(url1, url, lParser.siteName,log) log = myLogger.myLogger('logging.config') if len(sys.argv) >= 3: site = sys.argv[1] charset = sys.argv[2] log.setLever(sys.argv[3]) else: useage() if debugMode != 1: sys.exit() if debugMode == 1: log.setLever("debUG") urlList = urllib.parse.urlparse(site) host = urlList.scheme + '://' + urlList.hostname siteName = host[ host.find('.') +1: ].rstrip('/') print(siteName)# sys.exit() times = 0; urlList = [] getInnerURL(site, '/',siteName,log) except KeyboardInterrupt: sys.exit()

mylogger.py

#Author:Paul Wang#Date:2011-01-26#Description:logging wrapperimport logging.configclass myLogger: LEVELS = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL} def __init__(self, configName): print('init logger...') logging.config.fileConfig(configName) self.logger = logging.getLogger("simpleExample") def setLever(self, leverName): if (len(leverName)==0 ): self.logger.setLevel(logging.NOTSET) else: self.logger.setLevel( self.LEVELS.get(leverName.lower(), logging.NOTSET)) print('set lever %s...' % logging.getLevelName(self.logger.level) ) def writeLog(self, msg): self.logger.log(self.logger.level, msg) def debug(self, msg): self.logger.debug(msg) def info(self, msg): self.logger.info(msg) def warn(self, msg): self.logger.warn(msg) def error(self, msg): self.logger.error(msg) def critical(self, msg): self.logger.critical(msg) def __del__(self): print('destructor logger...')  

myHtmlParse.py

#Author:Paul Wang#Date:2011-01-26from html.parser import HTMLParserimport urllibimport reimport myLoggerclass MyParser(HTMLParser): def __init__ (self,log): HTMLParser. __init__ (self) self.currentLink = '' self.link = [] self.exlink = [] self.host = '' self.siteName = '' #like 'baidu.com' self.title = '' self.titleFlag = 0 self.log = log def handle_starttag(self, tag, attrs): if tag == 'a' : if len(attrs)==0:pass for name, value in attrs: if name == 'href' : pattern = re.compile('(.jpg|.jpeg|.avi|.png|.gif)</p>,re.I) match = pattern.search(value) if match: continue if len(value) and value[0] != '#' and value.find('@') == -1: if value.find(self.siteName) != -1 / or value[0] == '/' / or value[0:2] == './' / or value[0:3] == '../': if(self.currentLink[-1] != '/'): self.currentLink += '/' url = urllib.request.urljoin(self.currentLink, value) if(url.find("..") != -1 ): self.log.error("UrlJoin Error:%s, %s" % (self.currentLink, value)) self.link.append( url) else: self.exlink.append(value.rstrip('/')) elif tag == 'title': if len(attrs)==0:pass self.titleFlag = 1 def handle_data(self, data): if self.titleFlag == 1 and len(self.title)== 0: self.title=data self.titleFlag = 0 def filterRepeatLink(self):# print('count1: %d' % len(self.link)) dict = {} for keys in self.link: dict.setdefault(keys, '111') # print('count2: %d' % len(dict)) self.link = dict.keys() # print('count2: %d' % len(self.link))  

logging.config(配置档，用过log4c等的一看就明白了)

[loggers]keys=root,example[handlers]keys=consoleHandler,timedRotatingFileHandler[formatters]keys=simpleFormatter[formatter_simpleFormatter]#format=[%(asctime)s]%(name)s : %(message)sformat=[%(asctime)s] %(levelname)s : %(message)s[logger_root]level=DEBUGhandlers=consoleHandler,timedRotatingFileHandler[logger_example]level=DEBUGhandlers=consoleHandler,timedRotatingFileHandlerqualname=examplepropagate=0[handler_consoleHandler]class=StreamHandlerlevel=DEBUGformatter=simpleFormatterargs=(sys.stdout,)[handler_rotateFileHandler]class=handlers.RotatingFileHandlerlevel=DEBUGformatter=simpleFormatterargs=('test.log', 'a', 1024*1024, 9)[handler_timedRotatingFileHandler]class=handlers.TimedRotatingFileHandlerlevel=ERRORformatter=simpleFormatterargs=('app.log','d') 

转载地址：http://oggdi.baihongyu.com/