Org mode publishing with code evaluation

readme

My setup is a bunch of org files that I use a Makefile and publish.el to publish (as opposed to publishing directly from emacs).

My goal was to include files at publishing time but be able to filter secrets or other information I don't want included.

To this end I'm creating a code block that exports only the results.

The #+NAME: for the src block becomes the #+CAPTION of the results.
The results syntax highlighting can be controlled independently.

I've included my build files in their entirety.

The only relevant part for the code evaluation is the section marked "Code evaluation" in publish.el at the beginning and the example org file.

The rest is mostly about customizing org's export output to suit my pdf output.

Directory structure of the project is:
/--+-Makefile
   |-publish.el
   |-system.tex (uses subfiles system to include org->tex files)
   |-html (html output)
   |-tex  (tex output)
   +-src/--+-dir1/--org files
           +-dir2/--org files
           +-dir3/--org files
           +-images/ (static files to be copied during publishing)
           +-org file
           +-org file  
   

example.org

Example Document

Check the raw file (dunno why gists handle org files, but ok)

example.tex

% Created 2020-06-27 Sat 20:31
% Intended LaTeX compiler: xelatex
\documentclass[../../system.tex]{subfiles}
\author{Author}
\date{\today}
\title{Example Document}
\begin{document}

Check the raw file (dunno why gists handle org files, but ok)

\begin{SourceBox}[ini]{\href{run:/path/to/file}{/path/to/file}}
# set up password and their access
password <password>
default_permissions     ""
\end{SourceBox}
\end{document}

Makefile

LMAKE = latexmk
LMAKE_OPTIONS = -xelatex -pdfxe
LMAKE_PREVIEW = -xelatex -interaction=nonstopmode -pdfxe -pvc

ORG_FILES = $(shell find src/ -type f -name '*.org')
IMAGES = src/images/network.png src/images/backup.png

all: html system.pdf

.PHONY: clean html tex preview

%.png: %.graph
	dot -Tpng $< -o $@

html: $(ORG_FILES) $(IMAGES) publish.el
	emacs -l publish.el -batch --eval '(org-publish-project "org-static-html" t)'
	emacs -l publish.el -batch --eval '(org-publish-project "org-my-system-html" t)'

tex: $(ORG_FILES) $(IMAGES) publish.el
	emacs -l publish.el -batch --eval '(org-publish-project "org-static-tex" t)'
	emacs -l publish.el -batch --eval '(org-publish-project "org-my-system-tex" t)'

tex/section_list.tex: tex
	emacs -l publish.el -batch --eval '(reformat-sitemap "tex/index.tex" "tex/section_list.tex")'

system.pdf: tex system.tex tex/filedates.tex tex/section_list.tex $(wildcard tex/**/*.tex)
	$(LMAKE) $(LMAKE_OPTIONS) system.tex

preview: $(ORG_FILES) system.tex
	$(LMAKE) $(LMAKE_PREVIEW) system.tex &
	while true; do make tex; inotifywait -qr -e modify -e create -e delete -e move publish.el src; done;

# generate_dates doesn't get re-run during preview
# this can cause modification dates to be incorrect
tex/filedates.tex: tex $(wildcard src/**/*.org)
	find src -type d -printf "\\\\declare{%p/}\n" > filedates.tex
	find src -type f -name "*.org" -printf "\\\\setvalue{%p = %TY-%Tm-%Td}\n" >> filedates.tex

clean:
	- rm -f system.xdv system.aux system.fdb_latexmk system.fls system.log system.toc system.out system.listing system.pyg system.pdf
	- rm -f test.xdv test.aux test.fdb_latexmk test.fls test.log test.toc test.out test.listing test.pyg test.pdf
	- rm -f missfont.log filedates.tex
	- rm -f src/index.org
	- rm -f src/images/network.png src/images/backup.png
	- rm -rf _minted-system _minted-test tex/* html/* org/*
	emacs -l publish.el -batch --eval '(clear-org-cache)'

publish.el

(setq make-backup-files nil) ;; backup files aren't necessary for publishing
(require 'org)
(setq org-log-done t)

;; Code evaluation
;; Use CAPTION for the keyword because then source blocks with a NAME
;; become code blocks with a caption
(setq org-babel-results-keyword "CAPTION")

;; enable automatic evaluation
(defun my-org-confirm-babel-evaluate (lang body)
  (not (string= lang "shell")))  ;don't ask for shell
(setq org-confirm-babel-evaluate #'my-org-confirm-babel-evaluate)

;; set languages for babel to evaluate
(org-babel-do-load-languages
 'org-babel-load-languages
 '((shell . t)))
;; End Code evaluation

(require 'ox-latex)
(unless (boundp 'org-latex-classes)
  (setq org-latex-classes nil))
(add-to-list 'org-latex-classes
             '("subfile"
               "\\documentclass[../../system.tex]{subfiles}
               [NO-DEFAULT-PACKAGES]
               [NO-PACKAGES]
               [NO-EXTRA]"
               ("\\subsubsection{%s}" . "\\subsubsection*{%s}")
               ("\\paragraph{%s}" . "\\paragraph*{%s}")
               ("\\subparagraph{%s}" . "\\subparagraph*{%s}"))
               )

;; Don't upsize small images
(setq org-latex-image-default-width "")
(setq org-latex-image-default-option "max width=.9\\linewidth")

(setq org-latex-hyperref-template "")
(setq org-latex-default-figure-position "!htb")
(setq org-export-with-sub-superscripts nil)

(setq org-latex-listings 'minted)

(defun better-pdf-links (text backend info)
  "Export filter to use 'run:' instead of
  'file://' to prefix file links"
  (when (org-export-derived-backend-p backend 'latex)
    (with-temp-buffer
      (insert text)
      (goto-char (point-min))
      (while (search-forward "\\href{file://" nil t)
             (replace-match "\\\\href{run:"))
      (buffer-substring-no-properties (point-min) (point-max)))))
(add-to-list 'org-export-filter-final-output-functions 'better-pdf-links)

(defun org-latex-verse-block (verse-block contents info)
  "Override default latex verse block"
  (setq caption (org-export-data (org-export-get-caption verse-block) info))
  (org-latex--wrap-label
    verse-block
    (format "\\begin{VerseBox}[%s]\n%s\\end{VerseBox}" caption contents) info))

(defun org-latex-example-block (example-block contents info)
  "Override default latex example block"
  (setq caption (org-export-data (org-export-get-caption example-block) info))
  (org-latex--wrap-label
    example-block
    (format "\\begin{ExampleBox}[%s]\n%s\\end{ExampleBox}" caption (org-export-format-code-default example-block info)) info))

(defun org-latex-quote-block (quote-block contents info)
  "Override default latex quote block"
  (setq caption (org-export-data (org-export-get-caption quote-block) info))
  (org-latex--wrap-label
    quote-block
    (format "\\begin{QuoteBox}[%s]\n%s\\end{QuoteBox}" caption contents) info))

(defun unescape-underscore-filter (text backend info)
  "Let quote/verse block have unescaped underscores
  since we're using an appropriate environment"
  (when (org-export-derived-backend-p backend 'latex)
    (with-temp-buffer
      (insert text)
      (goto-char (point-min))
      (while (re-search-forward "\\\\_" nil t)
             (replace-match "_"))
      (buffer-substring-no-properties (point-min) (point-max)))))
(add-to-list 'org-export-filter-quote-block-functions 'unescape-underscore-filter)
(add-to-list 'org-export-filter-verse-block-functions 'unescape-underscore-filter)

(require 's)
(defun src-tcolorbox-env (text backend info)
  "Export filter to put src blocks in a SourceBox"
  (when (org-export-derived-backend-p backend 'latex)
    (with-temp-buffer
      (insert text)
      ;; grab the lang and caption before we delete these lines
      (goto-char (point-min))
      (if (re-search-forward "^\\\\begin{minted}\\\[]{\\(.*\\)}" nil t)
        (setq lang (match-string 1))
        (setq lang "text"))
      (goto-char (point-min))
      (if (re-search-forward "^\\\\caption{\\(.*\\)}$" nil t)
        (setq caption (s-replace "file://" "run:" (s-replace "\\" "\\\\" (match-string 1))))
        (setq caption ""))

      ;; replace environments with SourceBox
      (goto-char (point-min))
      (if (re-search-forward  "^\\\\begin{listing}.*$" nil t)
        (kill-whole-line))
      (goto-char (point-min))
      (if (re-search-forward "^\\\\begin{minted}.*$" nil t)
        (replace-match
          (concat "\\\\begin{SourceBox}" "[" lang "]" "{" caption "}")))
      (goto-char (point-max))
      (if (re-search-backward "^\\\\end{listing}$" nil t)
        (kill-whole-line))
      (goto-char (point-max))
      (if (re-search-backward "^\\\\caption{.*$" nil t)
        (kill-whole-line))
      (goto-char (point-max))
      (if (search-backward "\\end{minted}") (replace-match "\\\\end{SourceBox}"))

      (buffer-substring-no-properties (point-min) (point-max)))))
(add-to-list 'org-export-filter-src-block-functions 'src-tcolorbox-env)

(setq org-html-link-home "/")
(setq org-html-link-up "index.html")
(setq org-html-preamble t)
(setq org-html-postamble nil)

(defun my-system-sitemap-entry (entry style project)
  "Customize sitemap entries
  replace directory names with section names"
  (setq section_names
        '(("apps" . "Applications")
          ("data" . "Data")
          ("devices" . "Devices")
          ("machines" . "Machines")
          ("network" . "Network")
          ("user_apps" . "User Applications")
          ("website" . "Website")))
  (cond ((not (directory-name-p entry))
         (format "[[file:%s][%s]]" entry (org-publish-find-title entry project)))
        ((eq style 'tree)
         ;; Return only last subdir.
         (cond ((assoc-string (file-name-nondirectory (directory-file-name entry)) section_names)
                (cdr (assoc-string (file-name-nondirectory (directory-file-name entry)) section_names)))
               (t (file-name-nondirectory (directory-file-name entry)))))
        (t (cond ((assoc-string entry section_names)
                  (cdr (assoc-string entry section_names)))
                 (t entry)))))

;; for tex/section_list.tex target
(defun reformat-sitemap (input_filename output_filename)
  "Convert a sitemap to my own section, subsection,
  style for LaTeX inclusion"
  (insert-file-contents input_filename)
  (goto-char (point-min))
  ;; Delete headings
  (search-forward "\\begin{itemize}" nil nil)
  (kill-region (point-min) (search-backward "\\begin{itemize}" nil nil))

  (setq section-levels
        '((1 . "section")
          (2 . "subsection")
          (3 . "subsubsection")
          (4 . "paragraph")
          (5 . "subparagraph")))
  ;; Loop over list
  (setq level 0)
  (while (not (s-contains? "\\end{document}"
                           (setq line (buffer-substring-no-properties
                                        (line-beginning-position)
                                        (line-end-position)))))
         ;; if begin itemize increment level, end itemize decrement level
         (if (s-contains? "\\begin{itemize}" line)
           (progn (setf level (+ 1 level)) (kill-whole-line))
           (if (s-contains? "\\end{itemize}" line)
             (progn (setf level (- level 1)) (kill-whole-line))
             ;; if item reformat (handle href or plain text) (level determines section/sub/subsub)
             (if (s-contains? "\\item" line) 
               (if (s-contains? "\\href" line)
                 (progn
                   (goto-char (line-beginning-position))
                   (re-search-forward "\\\\href{\\(.*\\)}{\\(.*\\)}")
                   (setq file_name (match-string 1))
                   (setq section_name (match-string 2))
                   (goto-char (line-beginning-position)) (kill-line)
                   (insert "\\" (cdr (assoc level section-levels)) "{" section_name
                           "} \\hfill \\moddate{src/" file_name "} \\subfile{tex/"
                           (s-chop-suffix ".org" file_name) "}")
                   (next-line))
                 (progn
                   (goto-char (line-beginning-position))
                   (re-search-forward "\\\\item \\(.*\\)")
                   (setq section_name (match-string 1))
                   (goto-char (line-beginning-position)) (kill-line)
                   (insert "\\" (cdr (assoc level section-levels)) "{" section_name "}")
                   (next-line)))))))
  ;; stop and delete end{document}
  (kill-whole-line)
  (write-file output_filename))

(defun clear-org-cache ()
  (org-publish-remove-all-timestamps))

(setq org-publish-project-alist
      '(
        ("org-my-system-html"
         :base-directory "src"
         :publishing-directory "html"
         :base-extension "org"
         :auto-sitemap t
         :sitemap-format-entry my-system-sitemap-entry
         :sitemap-filename "index.org"
         :sitemap-title "My Computing System"
         :headline-levels 3
         :recursive t
         :publishing-function org-html-publish-to-html
         :auto-preamble t
         :timestamp nil
         :author nil
         :date nil
         )
        ("org-static-html"
         :base-directory "src"
         :base-extension "css\\|js\\|png\\|jpg\\|gif\\|pdf\\|mp3\\|ogg\\|swf"
         :publishing-directory "html"
         :recursive t
         :publishing-function org-publish-attachment
         )
        ("org-my-system-tex"
         :base-directory "src"
         :publishing-directory "tex"
         :base-extension "org"
         :auto-sitemap t
         :sitemap-format-entry my-system-sitemap-entry
         :sitemap-filename "index.org"
         :sitemap-title "My Computing System"
         :headline-levels 3
         :recursive t
         :publishing-function org-latex-publish-to-latex
         :latex-compiler "xelatex"
         :latex-class "subfile"
         :latex-title-command ""
         :latex-toc-command ""
         )
        ("org-static-tex"
         :base-directory "src"
         :base-extension "png\\|jpg\\|gif\\|pdf"
         :publishing-directory "tex"
         :recursive t
         :publishing-function org-publish-attachment
         )
        ("org" :components ("org-my-system-html"
                            "org-static-html"
                            "org-my-system-tex"
                            "org-static-tex"))
        ))