diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5ad76c5e..e2315974 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -13,11 +13,12 @@ jobs: - name: Install dependencies run: | export PATH="$HOME/.local/bin:$PATH" - sudo apt-get install -y python3-setuptools + sudo apt-get install -y python3-setuptools pandoc pip3 install --user -r requirements.txt - name: Build the docs run: | export PATH="$HOME/.local/bin:$PATH" make py + make man make htmlall \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2831e86e..b4cf2743 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,12 +15,13 @@ jobs: - name: Install dependencies run: | export PATH="$HOME/.local/bin:$PATH" - sudo apt-get install -y python3-setuptools + sudo apt-get install -y python3-setuptools pandoc pip3 install --user -r requirements.txt - name: Build the docs run: | export PATH="$HOME/.local/bin:$PATH" make py + make man make htmlall - name: Pre-deploy run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e02ebb06..5d1bff7c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,12 +15,13 @@ jobs: - name: Install dependencies run: | export PATH="$HOME/.local/bin:$PATH" - sudo apt-get install -y python3-setuptools + sudo apt-get install -y python3-setuptools pandoc pip3 install --user -r requirements.txt - name: Build the docs run: | export PATH="$HOME/.local/bin:$PATH" make py + make man make htmlall - name: Pre-deploy run: | diff --git a/.gitignore b/.gitignore index ce9ec15f..14c2dd61 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,9 @@ bundles.html.txt # ignore the venv, used for running make py venv + +# ignore artifacts of man page generation +source/_scripts/_python/manpages/*.rst +source/reference/manpages +source/_scripts/_python/manpages/*/ +source/reference/man-pages.rst diff --git a/Makefile b/Makefile index bb3a8c22..6f6bfd3b 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,12 @@ linkcheck: py: make -C source py +man: + make -C source man + +clean-man: + make -C source clean-man + help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" diff --git a/README.rst b/README.rst index a76ae312..e97f2144 100644 --- a/README.rst +++ b/README.rst @@ -63,8 +63,13 @@ running ``make html``: Open one of the HTML pages found in ``source/_build/html`` in a web browser to view the rendered documentation. -If you want to build the documentation exactly as seen on the website, use -``make py`` followed by ``make htmlall``. This builds some +This build will generate several warnings as there are two other optional make commands required to build the full documentation. + +1. ``make py`` to generate the bundle reference material. +2. ``make man`` to generate man page reference material. + +To build the documentation exactly as seen on the website, use +``make man``, ``make py``, and ``make htmlall``. This builds both external dependencies and all supported languages. Use virtualenv @@ -163,6 +168,9 @@ build before building again by running ``make clean``: This will completely remove the previous build output, including artifacts from the `make venv` target when done outside an active venv. +Before running ``make man``, please run ``make clean-man`` to clear out any +previous attempts. + Convenience script ================== diff --git a/make.bat b/make.bat index bac6aebb..2bc255b1 100644 --- a/make.bat +++ b/make.bat @@ -5,7 +5,7 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set BUILDDIR=build +set BUILDDIR=source\_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source set I18NSPHINXOPTS=%SPHINXOPTS% source if NOT "%PAPER%" == "" ( @@ -30,6 +30,7 @@ if "%1" == "help" ( echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages + echo. clean-man to clean up after man page generation echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items @@ -186,10 +187,14 @@ if "%1" == "text" ( ) if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. + cd source/_scripts/_python/manpages + man.bat man + goto end +) + +if "%1" == "clean-man" ( + cd source/_scripts/_python/manpages + man.bat clean-man goto end ) diff --git a/source/Makefile b/source/Makefile index 8be0a724..480400e2 100644 --- a/source/Makefile +++ b/source/Makefile @@ -50,6 +50,8 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" @echo " py to trigger an update of bundle content" + @echo " man to pull and create manpage rst files in reference section" + @echo " clean-man to clean up manpage generation" clean: rm -rf $(BUILDDIR)/* @@ -210,3 +212,9 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +man: + $(MAKE) -C $(SCRIPTDIR)/manpages man + +clean-man: + $(MAKE) -C $(SCRIPTDIR)/manpages clean-man \ No newline at end of file diff --git a/source/_scripts/_python/manpages/Makefile b/source/_scripts/_python/manpages/Makefile new file mode 100644 index 00000000..84addf28 --- /dev/null +++ b/source/_scripts/_python/manpages/Makefile @@ -0,0 +1,25 @@ +man: + git clone https://github.com/clearlinux/clr-man-pages.git + git clone https://github.com/clearlinux/clr-power-tweaks.git + git clone https://github.com/clearlinux/clrtrust.git + git clone https://github.com/clearlinux/mixer-tools.git + git clone https://github.com/clearlinux/swupd-client.git + git clone https://github.com/clearlinux/telemetrics-client.git + git clone https://github.com/clearlinux/tallow.git + git clone https://github.com/clearlinux/micro-config-drive.git + python3 manpages.py + - mkdir ../../../reference/manpages + cp *.rst ../../../reference/manpages + +clean-man: + - rm -rf clr-man-pages + - rm -rf clr-power-tweaks + - rm -rf clrtrust + - rm -rf mixer-tools + - rm -rf swupd-client + - rm -rf telemetrics-client + - rm -rf tallow + - rm -rf micro-config-drive + - rm *.rst + - rm ../../../reference/manpages/*.rst + - rm ../../../reference/man-pages.rst \ No newline at end of file diff --git a/source/_scripts/_python/manpages/man.bat b/source/_scripts/_python/manpages/man.bat new file mode 100644 index 00000000..88c05b8e --- /dev/null +++ b/source/_scripts/_python/manpages/man.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +REM Command file for building man pages + +if "%1" == "man" ( + git clone https://github.com/clearlinux/clr-man-pages.git + git clone https://github.com/clearlinux/clr-power-tweaks.git + git clone https://github.com/clearlinux/clrtrust.git + git clone https://github.com/clearlinux/mixer-tools.git + git clone https://github.com/clearlinux/swupd-client.git + git clone https://github.com/clearlinux/telemetrics-client.git + git clone https://github.com/clearlinux/tallow.git + git clone https://github.com/clearlinux/micro-config-drive.git + python.exe manpages.py + mkdir ..\..\..\reference\manpages + copy *.rst ..\..\..\reference\manpages + goto end +) + +if "%1" == "clean-man" ( + rmdir /q /s clr-man-pages + rmdir /q /s clr-power-tweaks + rmdir /q /s clrtrust + rmdir /q /s mixer-tools + rmdir /q /s swupd-client + rmdir /q /s telemetrics-client + rmdir /q /s tallow + rmdir /q /s micro-config-drive + del *.rst + del ..\..\..\reference\manpages\*.rst + del ..\..\..\reference\man-pages.rst + goto end +) + +:end \ No newline at end of file diff --git a/source/_scripts/_python/manpages/manpages.py b/source/_scripts/_python/manpages/manpages.py new file mode 100644 index 00000000..a9d14ab6 --- /dev/null +++ b/source/_scripts/_python/manpages/manpages.py @@ -0,0 +1,196 @@ +# +# manpages.py +# +# maintainer: intelkevinputnam +# +# usage: python3 manpages.py +# +# dependencies: 1. clone of Clear Linux documentation https://github.com/clearlinux/clear-linux-documentation +# 2. Makefile or make.bat to create directories and clone repositories (``make man`` to generate. ``make clean-man`` to clean up.) +# +# This script does 4 things: +# +# 1. Find and move reStructuredText versions of man pages (looks for manName.sectionNumber.fileExtension: mixer.1.rst) to reference/manpages directory of Clear Linux docs. +# 2. Massage markdown and reStructuredText man pages into a normalized format. +# 3. Add cross reference links wherever man pages reference each other. +# 4. Create manpages.rst in reference directory (already included in toctree of reference/index.rst) +# +# + +import sys +import re +import subprocess +from os import listdir +from os.path import join, isfile +from shutil import copyfile + +paths = ["clr-man-pages","clr-power-tweaks/man","clrtrust/man","micro-config-drive/docs/","mixer-tools/docs","swupd-client/docs","tallow/man","telemetrics-client/docs/man"] +pathToRefSection = "../../../reference/" +manPageRegex = '.[0-9]+.rst' +mdManPageRegex = '.[0-9]+.md' +mdBoldItalicRegex = '\\*\\*\\`[a-z-._]+\\`\\*\\*' +manFiles = [] +manGroups = {} +manNamePerms = {} +seeAlsoRefs = {} +gitHubGroup = "https://github.com/clearlinux/" +TOC = "\n.. contents::\n :local:\n" +manPageRest = """.. _man-pages: + +Man pages +######### + +These pages are generated from `Clear Linux\\* tool +repositories `__. Updated |today|. + + +""" +manTOC = """.. toctree:: + :maxdepth: 1 + +""" + +def getPages(paths): + for path in paths: + manGroups[path] = [] + files = listdir(path) + for file in files: + if file.endswith(".rst"): + regex_found = re.search(manPageRegex,file) + if regex_found: + filePath = join(path,file) + copyfile(filePath,file) + manFiles.append(file) + manGroups[path].append(file) + addManNamePermutations(file) + elif file.endswith(".md"): + regex_found = re.search(mdManPageRegex,file) + if regex_found: + filePath = join(path,file) + rstFileName = processMDFile(file,filePath) + manFiles.append(rstFileName) + manGroups[path].append(rstFileName) + addManNamePermutations(rstFileName) + +def processMDFile(fileName,filePath): + manName = fileName.rstrip('.md') + manSection = manName.split('.')[-1] + manName = manName.rstrip('.' + manSection) + lines = [] + headerInsert = "" + with open(filePath,'r') as m: + lines = m.readlines() + if "SYNOPSIS" in lines[0]: + header='='*len(manName) + lines.insert(0,"# " + manName + "\n\n:Manual section: "+manSection+"\n\n") + index = 0 + first = True + for line in lines: + newLine = line.replace("**`","`") #Fix some markdown formatting weirdness that doesn't translate to reST + newLine = newLine.replace("`**","`") + if newLine.startswith("#") and not newLine.startswith("##"): #Fix if all headers are first level + if first: + first = False + else: + newLine = newLine.replace("#","##") + if "===" in newLine: #fix rst style header that is actually description in some docs + newLine = "" + headerInsert = manName + "\n" + '='*len(manName) + "\n\n" + lines[index] = newLine + index += 1 + lines.insert(0,headerInsert) + + with open(filePath,'w') as md: + md.writelines(lines) + rstFilePath = filePath.replace(".md",".rst") + rstFileName = fileName.replace(".md",".rst") + + command = "pandoc " + filePath + " -o " + rstFilePath + subprocess.run(command, shell=True) + if isfile(rstFilePath): + copyfile(rstFilePath,rstFileName) + return rstFileName + +def addManNamePermutations(fileName): #Based on all the syntactical permutations of references to man pages in the documents. + (manName,subsection) = getNameAndSubsection(fileName) + manNames = [] + manNames.append("``" + manName + "``\\(" + subsection +")") # ``mixer.init``\(1) + manNames.append("``" + manName + "``\\ (" + subsection +")") # ``mixer.init``\ (1) + manNames.append("**``" + manName + "(" + subsection + ")``**") # **`mixer.init(1)`** + manNames.append("``" + manName + "(" + subsection + ")``") # ``mixer.init(1)`` + manNames.append("**" + manName + "(" + subsection + ")**") # **mixer.init(1)** + manNames.append("`" + manName + "(" + subsection + ")`") # `mixer.init(1)` + manNamePerms[(manName,subsection)] = manNames + +def getNameAndSubsection(manFileName): + manName = manFileName.rstrip('.rst') + manSection = manName.split('.')[-1] + manName = manName.rstrip('.'+manSection) + return (manName,manSection) + +def linkToMan(manName,manSection): + return "`" + manName + "(" + manSection + ") <" + manName + "." + manSection + ".html>`__" + +def buildManName(name,section): + return name + "(" + section + ")" + +def updateManPages(): + #makeSeeAlsoReplacements() + for file in manFiles: + manFile = "" + with open(file,'r') as f: + manFile = f.read() + #manFile = addTOC(manFile,file) # Not convinced adding TOC adds value. + # + # Add linked cross referencing for all manpages discovered. + # 1. the manNamePerms dictionary is created once all of the man page source files are discovered + # 2. Each document is checked for each permutation. + # 3. When a permuation is found it is replaced with a normalized version + # 4. Once normalized it is turned into a reStructuredText link (def linkToMan) + # + for nameAndSection, listOfPerms in manNamePerms.items(): + for perm in listOfPerms: + manFile = manFile.replace(perm,buildManName(nameAndSection[0],nameAndSection[1])) + for manName, doNotUse in manNamePerms.items(): + manFile = manFile.replace(buildManName(manName[0],manName[1]),linkToMan(manName[0],manName[1])) + with open(file,'w') as w: + w.write(manFile) + +def addTOC(manContent,file): +# Find the first instance of the manpage Name +# Skip the underline of the header +# Add a new line +# Add contents directive with local modifier +# Add a new line + manSectionMeta = ":Manual section:" + manContentLines = manContent.split('\n') + index = 0 + for line in manContentLines: + if manSectionMeta in line: + manContentLines.insert(index + 2,TOC) + break + index += 1 + output = "" + for line in manContentLines: + output = output + line + "\n" + return output + +def createManpagesRST(): + filePath = join(pathToRefSection,"man-pages.rst") + with open(filePath,'w') as f: + manGrouping = manPageRest + for path, fileList in manGroups.items(): + repoName = path.split("/")[0] + repoLink = gitHubGroup + repoName + repoReST = "`" + repoName + " <" + repoLink + ">`__" + manGrouping += repoReST + "\n" + manGrouping += "="*len(repoReST) + "\n\n" + manGrouping += manTOC + for file in fileList: + manGrouping += " manpages/" + file + "\n" + manGrouping += "\n" + f.write(manGrouping) + +getPages(paths) +updateManPages() +createManpagesRST() \ No newline at end of file diff --git a/source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css b/source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css index f252efae..71205dc9 100644 --- a/source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css +++ b/source/_themes/otc_tcs_sphinx_theme/static/tcs_theme.css @@ -306,9 +306,9 @@ button:hover a.headerlink:after { /*End support for custom Clear Linux header*/ /*Adds a bit of spacing after the last paragraph in a bulleted list*/ -.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child { +/*.wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child { margin-bottom: 10px; -} +}*/ div.admonition ul { margin-top: 20px; diff --git a/source/conf.py b/source/conf.py index 310646ff..0b25023f 100644 --- a/source/conf.py +++ b/source/conf.py @@ -81,7 +81,7 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build','_themes'] +exclude_patterns = ['_build','_themes','_scripts'] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/source/reference/index.rst b/source/reference/index.rst index 554a8c86..12f2ec71 100644 --- a/source/reference/index.rst +++ b/source/reference/index.rst @@ -13,3 +13,4 @@ features. bundles/bundles system-requirements image-types + man-pages \ No newline at end of file