mirror of
https://github.com/clearlinux/docker.git
synced 2026-10-04 07:48:44 +00:00
add Scott's link checker script, and fix what it finds
Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>
This commit is contained in:
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
""" I honestly don't even know how the hell this works, just use it. """
|
||||
__author__ = "Scott Stamp <scott@hypermine.com>"
|
||||
|
||||
from HTMLParser import HTMLParser
|
||||
from urlparse import urljoin
|
||||
from sys import setrecursionlimit
|
||||
import re
|
||||
import requests
|
||||
|
||||
setrecursionlimit(10000)
|
||||
root = 'http://localhost:8000'
|
||||
|
||||
|
||||
class DataHolder:
|
||||
|
||||
def __init__(self, value=None, attr_name='value'):
|
||||
self._attr_name = attr_name
|
||||
self.set(value)
|
||||
|
||||
def __call__(self, value):
|
||||
return self.set(value)
|
||||
|
||||
def set(self, value):
|
||||
setattr(self, self._attr_name, value)
|
||||
return value
|
||||
|
||||
def get(self):
|
||||
return getattr(self, self._attr_name)
|
||||
|
||||
|
||||
class Parser(HTMLParser):
|
||||
global root
|
||||
|
||||
ids = set()
|
||||
crawled = set()
|
||||
anchors = {}
|
||||
pages = set()
|
||||
save_match = DataHolder(attr_name='match')
|
||||
|
||||
def __init__(self, origin):
|
||||
self.origin = origin
|
||||
HTMLParser.__init__(self)
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if 'href' in attrs:
|
||||
href = attrs['href']
|
||||
|
||||
if re.match('^{0}|\/|\#[\S]{{1,}}'.format(root), href):
|
||||
if self.save_match(re.search('.*\#(.*?)$', href)):
|
||||
if self.origin not in self.anchors:
|
||||
self.anchors[self.origin] = set()
|
||||
self.anchors[self.origin].add(
|
||||
self.save_match.match.groups(1)[0])
|
||||
|
||||
url = urljoin(root, href)
|
||||
|
||||
if url not in self.crawled and not re.match('^\#', href):
|
||||
self.crawled.add(url)
|
||||
Parser(url).feed(requests.get(url).content)
|
||||
|
||||
if 'id' in attrs:
|
||||
self.ids.add(attrs['id'])
|
||||
# explicit <a name=""></a> references
|
||||
if 'name' in attrs:
|
||||
self.ids.add(attrs['name'])
|
||||
|
||||
|
||||
r = requests.get(root)
|
||||
parser = Parser(root)
|
||||
parser.feed(r.content)
|
||||
for anchor in sorted(parser.anchors):
|
||||
if not re.match('.*/\#.*', anchor):
|
||||
for anchor_name in parser.anchors[anchor]:
|
||||
if anchor_name not in parser.ids:
|
||||
print 'Missing - ({0}): #{1}'.format(
|
||||
anchor.replace(root, ''), anchor_name)
|
||||
Reference in New Issue
Block a user