mirror of
https://github.com/clearlinux/clear-linux-documentation.git
synced 2026-06-28 16:56:44 +00:00
renamed support directories by adding leading "_" to differentiate from content.
Signed-off-by: Kevin Putnam <kevin.putnam@intel.com>
This commit is contained in:
Kevin Putnam
@@ -0,0 +1,9 @@
|
||||
|
||||
py:
|
||||
python bundle_lister.py
|
||||
cp bundles.html.txt ../../clear-linux/reference/bundles
|
||||
rm -rf bundle_lister/cloned_repo/*
|
||||
rm bundles.html.txt
|
||||
@echo "Python script finished successfully!"
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
bundle_lister.py
|
||||
----------------
|
||||
|
||||
`bundle_lister.py` is a Python (3.6.0) web scraper and html file generator. First, it clones the
|
||||
[clr-bundles directory](https://github.com/clearlinux/clr-bundles). Second, it parses the content of all bundles in the clr-bundles/ directory and the `packages` file. Third, it uses Jinja2 template engine to output the result as: bundles.html.txt. This file is copied to reference/bundles location, and it is invoked in `bundles.rst`, which currently appears as [Available bundles](https://clearlinux.org/documentation/clear-linux/reference/bundles).
|
||||
|
||||
`bundle_lister.py` automates clear linux documentation so it reflects
|
||||
current bundles and packages per developer updates to the
|
||||
[clr-bundles GitHub repository](https://github.com/clearlinux/clr-bundles). Therefore, it increases efficiency, automatically aligns documentation with Clear Linux Engineering development, and it eliminates potential for human error, and saves labor hours in contrast to the previous manual method.
|
||||
|
||||
`bundle_lister.py` will be invoked in a bash script in the `source/Makefile` of clear-linux documentation. Therefore, `bundle_lister.py` will automatically create newly scraped and parsed data upon each build of the
|
||||
[website](https://clearlinux.org) and output an accurate, up-to-date table showing all bundles and packages for interested developers and admins.
|
||||
|
||||
See `requirements.txt` for dependencies necessary to run this application.
|
||||
|
||||
Built in:`Python==3.6.0`
|
||||
|
||||
To run `bundle_lister.py` in the terminal, enter: `python bundle_lister.py`.
|
||||
|
||||
Note: The `cloned_repo` directory must remain in the parent directory in order for this code to work; the template.html must remain as
|
||||
well.
|
||||
|
||||
Note: A successful build will produce a file `bundles.html.txt` showing a table of current bundles and pundles (packages) alphabetized, with a (UTC) time and date stamp in the upper right corner.
|
||||
|
||||
|
||||
`~$~`
|
||||
@@ -0,0 +1,96 @@
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import urllib
|
||||
import jinja2
|
||||
from jinja2 import Environment, FileSystemLoader, Template
|
||||
import git
|
||||
from operator import itemgetter
|
||||
from datetime import datetime
|
||||
|
||||
GITHUB_BASE = "https://github.com/clearlinux/clr-bundles/tree/master/bundles/"
|
||||
PUNDLES = "https://github.com/clearlinux/clr-bundles/blob/master/packages"
|
||||
|
||||
PATTERN1 = re.compile(r"#\s?\[TITLE]:\w?(.*)")
|
||||
PATTERN2 = re.compile(r"#\s?\[DESCRIPTION]:\w?(.*)")
|
||||
PATTERN3 = re.compile(r"\(([^()]*|include)\)", re.MULTILINE)
|
||||
PATTERN4 = re.compile(r"^((?:(?!#)\w+[^-\s][-])\w+|\w+[^\s-])", re.MULTILINE)
|
||||
PATTERN5 = re.compile(r"^\w.+\s[#]\s(\w?.*)?", re.MULTILINE)
|
||||
|
||||
def extractor(lines):
|
||||
bundle_title = "title"
|
||||
data_desc = "description"
|
||||
url = "url"
|
||||
include_list = []
|
||||
include_unique = []
|
||||
|
||||
for i in lines:
|
||||
title = PATTERN1.match(i)
|
||||
desc = PATTERN2.match(i)
|
||||
includes = PATTERN3.findall(i)
|
||||
|
||||
if title:
|
||||
bundle_title = title.groups(0)[0].strip()
|
||||
if desc:
|
||||
data_desc = desc.groups(0)[0].strip()
|
||||
if url:
|
||||
url = os.path.join(GITHUB_BASE, bundle_title)
|
||||
if includes:
|
||||
include_text = includes[0].strip("()")
|
||||
include_list.append(include_text)
|
||||
include_unique = set(include_list)
|
||||
return {"title": bundle_title, "data_desc": data_desc, "include_list": include_unique, "url": url}
|
||||
|
||||
def pundler():
|
||||
with io.open("./cloned_repo/clr-bundles/packages") as file_obj:
|
||||
lines = file_obj.readlines()
|
||||
pundle_title = "pundle_title"
|
||||
pundle_desc = "pundle_desc"
|
||||
purl = "purl"
|
||||
pundle_list = []
|
||||
pun_desc = []
|
||||
pundle_master = []
|
||||
|
||||
for i in lines:
|
||||
pundle = PATTERN4.findall(i)
|
||||
pundle_plus = PATTERN5.findall(i)
|
||||
|
||||
if pundle:
|
||||
pundle_title = pundle[0]
|
||||
pundle_list.append(pundle_title)
|
||||
|
||||
if pundle_plus:
|
||||
pundle_desc = pundle_plus[0].strip("[]")
|
||||
pun_desc.append(pundle_desc)
|
||||
|
||||
for pun, desc in zip(pundle_list, pun_desc):
|
||||
pundle_master.append({"title": pun, "pun_desc": desc, "purl": PUNDLES})
|
||||
return pundle_master
|
||||
|
||||
def bundler():
|
||||
data = []
|
||||
try:
|
||||
git.Git("./cloned_repo/").clone("https://github.com/clearlinux/clr-bundles.git")
|
||||
except:
|
||||
pass
|
||||
for root, dirs, files in os.walk("./cloned_repo/clr-bundles/bundles", topdown=False):
|
||||
for name in files:
|
||||
with open(os.path.join(root, name)) as file_obj:
|
||||
lines = file_obj.readlines()
|
||||
data.append(extractor(lines))
|
||||
|
||||
pundle_master = pundler()
|
||||
data = data + pundle_master
|
||||
filtered = list(filter(lambda x: x.get('title'), data))
|
||||
sortedData = sorted(filtered, key=lambda x:x['title'].lower())
|
||||
#ALT sortedData2 = sorted(sortedData, key=itemgetter('title'))
|
||||
loader = jinja2.FileSystemLoader(searchpath='./')
|
||||
env = jinja2.Environment(loader=loader)
|
||||
template = env.get_template('template.html')
|
||||
template.globals['now'] = datetime.utcnow
|
||||
|
||||
output = template.render(data=sortedData, now=datetime.utcnow())
|
||||
with io.open('bundles.html.txt', 'w') as file:
|
||||
file.write(output)
|
||||
|
||||
bundler()
|
||||
@@ -0,0 +1,12 @@
|
||||
http://www.intel.com/content/www/us/en/virtualization/virtualization-technology/intel-virtualization-technology.html
|
||||
https://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices
|
||||
http://www.intel.com/content/www/us/en/virtualization/virtualization-technology/intel-virtualization-technology.html
|
||||
https://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices
|
||||
https://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices
|
||||
https://www.intel.com/content/www/us/en/virtualization/virtualization-technology/intel-virtualization-technology.html
|
||||
https://www.intel.com/content/www/us/en/virtualization/virtualization-technology/intel-virtualization-technology.html
|
||||
https://www.intel.com/content/www/us/en/privacy/intel-privacy-notice.html
|
||||
http://ark.intel.com
|
||||
https://software.intel.com/en-us/articles/OpenVINO-ModelOptimizer
|
||||
https://www.intel.com/content/www/us/en/privacy/intel-privacy-notice.html
|
||||
http://www.intel.com/content/www/us/en/nuc/nuc-kit-nuc6i5syh.html
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#***********************************************************
|
||||
#
|
||||
# parse-link-check.py
|
||||
#
|
||||
# Arguments:
|
||||
# 1. working directory: path to directory containing output
|
||||
# of sphinx-build linkcheck (output.txt). Output of this
|
||||
# script (broken_links.html) will also be saved in this
|
||||
# location.
|
||||
#
|
||||
# External file dependencies:
|
||||
# 1. output.txt - the output of sphinx-build
|
||||
# 2. link-whitelist.txt - broken links that should be ignored.
|
||||
# This file should be in the same location as this script.
|
||||
#
|
||||
# Output:
|
||||
# 1. broken_links.html - provides count of broken and whitelist
|
||||
# matches. Also provides links to all flagged links. Will
|
||||
# appear in the same directory as output.txt
|
||||
# 2. Error code 255 if unexpected broken links are found
|
||||
#
|
||||
# Purpose:
|
||||
# This script supplements the built-in link checking of
|
||||
# sphinx-build. In practice, sphinx-build link checking
|
||||
# will produce a variety of false negatives. This script,
|
||||
# using the white list, will skip known false negatives and
|
||||
# anchors, providing an HTML digest of suspected actual
|
||||
# broken links if there are any.
|
||||
#
|
||||
#***********************************************************
|
||||
|
||||
|
||||
import sys
|
||||
import re
|
||||
import os
|
||||
|
||||
fileName = "output.txt"
|
||||
outFile = "broken_links.html"
|
||||
whitelistFile = "link-whitelist.txt"
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print ("Enter path of input directory")
|
||||
sys.exit()
|
||||
|
||||
scriptPath = sys.argv[0]
|
||||
outputPath = sys.argv[1]
|
||||
fileNamePath = outputPath + "/" + fileName
|
||||
outFilePath = outputPath + "/" + outFile
|
||||
|
||||
whitelistFilePath = os.path.dirname(scriptPath) + "/" + whitelistFile
|
||||
|
||||
with open (whitelistFilePath) as w:
|
||||
whLines = w.readlines()
|
||||
|
||||
whitelist = []
|
||||
for line in whLines:
|
||||
link = line.rstrip()
|
||||
whitelist.append(link)
|
||||
|
||||
with open (fileNamePath) as f:
|
||||
lines = f.readlines()
|
||||
|
||||
numBrokenLinks = 0
|
||||
numWhiteListMatches = 0
|
||||
numAnchors = 0
|
||||
newLines = ["<!DOCTYPE html><html><head><style>body {font-family: sans-serif;}</style></head><body>"]
|
||||
whiteListLines = []
|
||||
anchorLines = []
|
||||
|
||||
for line in lines:
|
||||
if "[broken]" in line:
|
||||
strings = line.split(" ")
|
||||
link = strings[2][:-1]
|
||||
link = link.strip()
|
||||
if link in whitelist:
|
||||
whiteListLines.append("<b>" + strings[0] + "</b>\n<blockquote><a href=\"" + link + "\">[whitelist] " + link + "</a></blockquote>\n")
|
||||
numWhiteListMatches += 1
|
||||
elif "Anchor '" in line:
|
||||
anchorLines.append("<b>" + strings[0] + "</b>\n<blockquote><a href=\"" + link + "\">[anchor] " + link + "</a></blockquote>\n")
|
||||
numAnchors += 1
|
||||
else:
|
||||
newLines.append("<b>" + strings[0] + "</b>\n<blockquote><a href=\"" + link + "\">[broken] " + link + "</a></blockquote>\n")
|
||||
numBrokenLinks += 1
|
||||
|
||||
newLines.insert(0,"<h1>" + str(numBrokenLinks + numWhiteListMatches) + " broken links found in Sphinx link check</h1>\n")
|
||||
newLines.insert(1,"<h2>" + str(numBrokenLinks) + " unmatched broken links</h2>\n")
|
||||
newLines.append("<h2>" + str(numAnchors) + " links did not find anchors</h2>\n")
|
||||
for line in anchorLines:
|
||||
newLines.append(line)
|
||||
|
||||
newLines.append("<h2>" + str(numWhiteListMatches) + " links matched whitelist</h2>\n")
|
||||
for line in whiteListLines:
|
||||
newLines.append(line)
|
||||
newLines.append("</body></html>")
|
||||
|
||||
with open (outFilePath, "w") as outF:
|
||||
for line in newLines:
|
||||
outF.write(line)
|
||||
|
||||
print("See ./" + outFilePath + " for a detailed breakdown of broken links.")
|
||||
|
||||
if numBrokenLinks != 0:
|
||||
print (str(numBrokenLinks) + " detected. Exiting with error code 255.")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
print ("No unexpected broken links detected.")
|
||||
@@ -0,0 +1,2 @@
|
||||
Jinja2==2.11.dev0
|
||||
GitPython==2.1.11
|
||||
@@ -0,0 +1,48 @@
|
||||
table {
|
||||
margin: 32px;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-collapse: collapse;
|
||||
width: auto;
|
||||
}
|
||||
th {
|
||||
font-family: IntelClear-Regular,Helvetica,Arial,sans-serif;
|
||||
align-content: center;
|
||||
padding: 5px;
|
||||
border: #ccc solid 1px;
|
||||
background-color: #555;
|
||||
color: #fff;
|
||||
text-transform: uppercase;
|
||||
font-size: 18px;
|
||||
}
|
||||
tr {
|
||||
padding-top: 20px ;
|
||||
padding-bottom: 10px;
|
||||
}
|
||||
tbody tr:nth-child(odd) {
|
||||
background-color: #e0e0e0;
|
||||
}
|
||||
.bundlename {
|
||||
font-family: IntelClear-Regular,Helvetica,Arial,sans-serif;
|
||||
font-size: 16px;
|
||||
font-weight: bolder;
|
||||
padding-left: 6px;
|
||||
line-height: 18px;
|
||||
padding-top:7px ;
|
||||
padding-bottom: 5px;
|
||||
}
|
||||
.bundledesc {
|
||||
font-family: IntelClear-Regular,Helvetica,Arial,sans-serif;
|
||||
font: italic;
|
||||
font-size: 16px;
|
||||
padding-left: 6px;
|
||||
line-height: 18px;
|
||||
padding-top: 7px ;
|
||||
padding-bottom: 5px;
|
||||
}
|
||||
ul, li {
|
||||
margin-left: 8px;
|
||||
/* padding: 0; */
|
||||
padding-left: 5px;
|
||||
padding-top: 2px;
|
||||
line-height: 16px;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Bundles in Clear Linux* OS</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<table id="bundletable">
|
||||
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th style="text-align:right; font-family:IntelClear-Regular,Helvetica,Arial; font-style:italic">
|
||||
Updated: {{ now.strftime('%x %H:%M') }} UTC
|
||||
</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<th> Name</th>
|
||||
<th> Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
{% for d in data %}
|
||||
{% if d.url %}
|
||||
<tr id="bundle">
|
||||
<td class="bundlename" id="bundle"><a href="{{d.url}}">{{d.title}}</a></td>
|
||||
<td class="bundledesc">{{d.data_desc}} <br />
|
||||
{% if d.include_list %}
|
||||
<p>Includes bundle(s):
|
||||
{% for include in d.include_list %}
|
||||
<li>{{include}}</li>
|
||||
{% endfor %}
|
||||
</p>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr id="pundle">
|
||||
<td class="bundlename"><a href="{{d.purl}}">{{d.title}}</a></td>
|
||||
<td class="bundledesc"> {{d.pun_desc}} </td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,40 @@
|
||||
'''
|
||||
Usage: python code-blocks.py path/to/file.rst
|
||||
|
||||
Default output is to stdout
|
||||
|
||||
Parses all code found in blocks listed in blockTypes out of rst file for instruction testing.
|
||||
'''
|
||||
import sys
|
||||
|
||||
blockTypes = ["bash","console"]
|
||||
|
||||
def code_blocks(filename):
|
||||
with open(filename, 'r') as fd:
|
||||
c_indent = ''
|
||||
in_section = []
|
||||
for line in fd:
|
||||
blockFound = False
|
||||
indent = line[:len(line) - len(line.lstrip())]
|
||||
for blockType in blockTypes:
|
||||
if 'code-block:: ' + blockType in line:
|
||||
blockFound = True
|
||||
if blockFound:
|
||||
in_section = [line.strip()]
|
||||
c_indent = ''
|
||||
blockFound = False
|
||||
elif in_section:
|
||||
if not c_indent and line.strip():
|
||||
c_indent = indent
|
||||
if not (len(indent) >= len(c_indent)) and line.strip():
|
||||
yield in_section[2:]
|
||||
in_section = []
|
||||
else:
|
||||
in_section.append(line[len(c_indent):].rstrip())
|
||||
|
||||
def main():
|
||||
for code_block in code_blocks(sys.argv[1]):
|
||||
print('\n'.join(code_block) + '\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user