nixpkgs/nixos/doc/manual/md-to-db.sh

#! /usr/bin/env nix-shell
#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/tarball/21.11 -i bash -p pandoc

# This script is temporarily needed while we transition the manual to
# CommonMark. It converts the .md files in the regular manual folder
# into DocBook files in the from_md folder.

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
pushd "$DIR"

# NOTE: Keep in sync with Nixpkgs manual (/doc/Makefile).
# TODO: Remove raw-attribute when we can get rid of DocBook altogether.
pandoc_commonmark_enabled_extensions=+attributes+fenced_divs+footnotes+bracketed_spans+definition_lists+pipe_tables+raw_attribute
pandoc_flags=(
  # Not needed:
  # - diagram-generator.lua (we do not support that in NixOS manual to limit dependencies)
  # - media extraction (was only required for diagram generator)
  # - docbook-reader/citerefentry-to-rst-role.lua (only relevant for DocBook → MarkDown/rST/MyST)
  "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/myst-reader/roles.lua"
  "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/link-unix-man-references.lua"
  "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/rst-roles.lua"
  "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/labelless-link-is-xref.lua"
  -f "commonmark${pandoc_commonmark_enabled_extensions}+smart"
  -t docbook
)

OUT="$DIR/from_md"
mapfile -t MD_FILES < <(find . -type f -regex '.*\.md$')

for mf in ${MD_FILES[*]}; do
  if [ "${mf: -11}" == ".section.md" ]; then
    mkdir -p "$(dirname "$OUT/$mf")"
    OUTFILE="$OUT/${mf%".section.md"}.section.xml"
    pandoc "$mf" "${pandoc_flags[@]}" \
      -o "$OUTFILE"
    grep -q -m 1 "xi:include" "$OUTFILE" && sed -i 's|xmlns:xlink="http://www.w3.org/1999/xlink"| xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude"|' "$OUTFILE"
  fi

  if [ "${mf: -11}" == ".chapter.md" ]; then
    mkdir -p "$(dirname "$OUT/$mf")"
    OUTFILE="$OUT/${mf%".chapter.md"}.chapter.xml"
    pandoc "$mf" "${pandoc_flags[@]}" \
      --top-level-division=chapter \
      -o "$OUTFILE"
    grep -q -m 1 "xi:include" "$OUTFILE" && sed -i 's|xmlns:xlink="http://www.w3.org/1999/xlink"| xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude"|' "$OUTFILE"
  fi
done

popd
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00			`#! /usr/bin/env nix-shell`
nixos/doc/md-to-db.sh: consistent pandoc version Until now, this script has used the version of pandoc from unstable. This means that running the script on the same version of Nixpkgs could produce different results, and meant that when Pandoc's output was changed, random PRs were changing the whole manual when they ran the script to regenerate docs[1][2]. Here I've changed the manual to use a consistent version of pandoc — the one from the latest release tag, which will avoid this problem in future. This will avoid this problem in future. The only time we'll need to worry about pandoc output changes is when we bump the version used in this script. I also considered using the version from the current Nixpkgs branch, but decided against it as it's unlikely that e.g. the person bumping Pandoc will remember to regenerate the manual. [1]: https://github.com/NixOS/nixpkgs/pull/162550 [2]: https://github.com/NixOS/nixpkgs/pull/168535 2022-04-14 07:48:02 +00:00			`#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/tarball/21.11 -i bash -p pandoc`
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00
			`# This script is temporarily needed while we transition the manual to`
			`# CommonMark. It converts the .md files in the regular manual folder`
			`# into DocBook files in the from_md folder.`

			`DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"`
nixos/doc/md-to-db.sh: handle path to nixpkgs with spaces Without this change, the script will fail if the path to nixpkgs contains a space. 2021-10-09 17:08:25 +00:00			`pushd "$DIR"`
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00
nixos/doc: Synchronize the Markdown generator with Nixpkgs Switch to CommonMark with our extensions. 2021-06-17 22:07:00 +00:00			`# NOTE: Keep in sync with Nixpkgs manual (/doc/Makefile).`
			`# TODO: Remove raw-attribute when we can get rid of DocBook altogether.`
			`pandoc_commonmark_enabled_extensions=+attributes+fenced_divs+footnotes+bracketed_spans+definition_lists+pipe_tables+raw_attribute`
			`pandoc_flags=(`
doc: Add support for MyST roles Officially, only the manpage role is supported at the moment. Unlike in rST, the syntax uses braces instead of colons: {manpage}`nix.conf(5)` 2021-07-12 13:54:14 +00:00			`# Not needed:`
			`# - diagram-generator.lua (we do not support that in NixOS manual to limit dependencies)`
			`# - media extraction (was only required for diagram generator)`
			`# - docbook-reader/citerefentry-to-rst-role.lua (only relevant for DocBook → MarkDown/rST/MyST)`
			`"--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/myst-reader/roles.lua"`
doc: Linkify man page references 2021-07-12 16:40:33 +00:00			`"--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/link-unix-man-references.lua"`
doc: Add support for MyST roles Officially, only the manpage role is supported at the moment. Unlike in rST, the syntax uses braces instead of colons: {manpage}`nix.conf(5)` 2021-07-12 13:54:14 +00:00			`"--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/rst-roles.lua"`
doc: Move lua filters to subdirectory 2021-07-12 02:38:41 +00:00			`"--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/labelless-link-is-xref.lua"`
nixos/doc: Synchronize the Markdown generator with Nixpkgs Switch to CommonMark with our extensions. 2021-06-17 22:07:00 +00:00			`-f "commonmark${pandoc_commonmark_enabled_extensions}+smart"`
			`-t docbook`
			`)`

nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00			`OUT="$DIR/from_md"`
			`mapfile -t MD_FILES < <(find . -type f -regex '.*\.md$')`

			`for mf in ${MD_FILES[*]}; do`
			`if [ "${mf: -11}" == ".section.md" ]; then`
nixos/doc/md-to-db.sh: handle path to nixpkgs with spaces Without this change, the script will fail if the path to nixpkgs contains a space. 2021-10-09 17:08:25 +00:00			`mkdir -p "$(dirname "$OUT/$mf")"`
nixos/doc/md-to-db.sh: handle Docbook inclues in CommonMark You can do includes like this: ```{=docbook} <xi:include href="rl-2111.section.xml" /> ``` 2021-09-07 13:29:21 +00:00			`OUTFILE="$OUT/${mf%".section.md"}.section.xml"`
nixos/doc: Synchronize the Markdown generator with Nixpkgs Switch to CommonMark with our extensions. 2021-06-17 22:07:00 +00:00			`pandoc "$mf" "${pandoc_flags[@]}" \`
nixos/doc/md-to-db.sh: handle Docbook inclues in CommonMark You can do includes like this: ```{=docbook} <xi:include href="rl-2111.section.xml" /> ``` 2021-09-07 13:29:21 +00:00			`-o "$OUTFILE"`
			`grep -q -m 1 "xi:include" "$OUTFILE" && sed -i 's\|xmlns:xlink="http://www.w3.org/1999/xlink"\| xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude"\|' "$OUTFILE"`
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00			`fi`

			`if [ "${mf: -11}" == ".chapter.md" ]; then`
nixos/doc/md-to-db.sh: handle path to nixpkgs with spaces Without this change, the script will fail if the path to nixpkgs contains a space. 2021-10-09 17:08:25 +00:00			`mkdir -p "$(dirname "$OUT/$mf")"`
nixos/doc/md-to-db.sh: handle Docbook inclues in CommonMark You can do includes like this: ```{=docbook} <xi:include href="rl-2111.section.xml" /> ``` 2021-09-07 13:29:21 +00:00			`OUTFILE="$OUT/${mf%".chapter.md"}.chapter.xml"`
nixos/doc: Synchronize the Markdown generator with Nixpkgs Switch to CommonMark with our extensions. 2021-06-17 22:07:00 +00:00			`pandoc "$mf" "${pandoc_flags[@]}" \`
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00			`--top-level-division=chapter \`
nixos/doc/md-to-db.sh: handle Docbook inclues in CommonMark You can do includes like this: ```{=docbook} <xi:include href="rl-2111.section.xml" /> ``` 2021-09-07 13:29:21 +00:00			`-o "$OUTFILE"`
			`grep -q -m 1 "xi:include" "$OUTFILE" && sed -i 's\|xmlns:xlink="http://www.w3.org/1999/xlink"\| xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude"\|' "$OUTFILE"`
nixos/doc: add md-to-db.sh, convert "Building Your Own NixOS CD" to CommonMark 2020-12-23 05:56:46 +00:00			`fi`
			`done`

			`popd`