diff --git a/tools/build-website.sh b/tools/build-website.sh index 2f8ceb9..6e2842d 100755 --- a/tools/build-website.sh +++ b/tools/build-website.sh @@ -1,334 +1,335 @@ #!/bin/sh # Build the gnupg.org website from a git working directory. # # This script requires two users # # webbuilder - the user to run this script # webbuild-x - the user used by this script to run emacs # webbuild-y - the user used by this script to run emacs (for preview) # # A certain directory layout is required with permissions setup # so that the webbuild-x has only write access to the stage area # and to its own home directory. The script checks the permissions. # # The trigger-website-build scripts is expected to be installed # as git post-merge hook. # # These cronjobs are required for user webbuilder: # --8<---------------cut here---------------start------------->8--- # # Pull the master branch of the web pages # */20 * * * * cd /home/webbuilder/gnupg-doc && git pull -q origin master # */18 * * * * cd /home/webbuilder/gnupg-doc-preview && git pull -q origin preview # # # In case of race conditions we try to build every few ours again. # 35 */7 * * * /home/webbuilder/bin/build-website.sh --cron # --8<---------------cut here---------------end--------------->8--- # # /etc/sudoers needs this: # --8<---------------cut here---------------start------------->8--- # # Let webbuilder run any command as user webbuild-x # webbuilder ALL = (webbuild-x,webbuild-y) NOPASSWD: ALL # --8<---------------cut here---------------end--------------->8--- # set -e pgm=build-website.sh mainuser=webbuilder workuser=webbuild-x workuser_pv=webbuild-y # We use a fixed HOME so that this script can be run here from other # accounts. HOME=$(awk &2; exit 1 fi reponame=gnupg-doc htdocs_web="/var/www/www/www.gnupg.org/htdocs" htdocs_preview="/var/www/www/preview.gnupg.org/htdocs" htdocs_blog="/var/www/www/www.gnupg.org/misc/blog" workuser_dir=$HOME/${workuser} workuser_pv_dir=$HOME/${workuser_pv} log_dir="$HOME/log" root_dir="$HOME/${reponame}" root_dir_pv="$HOME/${reponame}-preview" stage_dir="$HOME/${reponame}-stage" stage_dir_pv="$HOME/${reponame}-preview-stage" LOCKFILE="${log_dir}/${reponame}.lock" if [ x"$1" = x"--git" ]; then shift exec >>${log_dir}/"$reponame".log 2>&1 echo "$(date -u -Iseconds) gpgweb site build was git triggered" elif [ x"$1" = x"--cron" ]; then shift exec >>${log_dir}/"$reponame".log 2>&1 echo "$(date -u -Iseconds) gpgweb site build was cron triggered" fi if ! id $workuser >/dev/null 2>&1 ; then echo "$pgm: sudo user '${workuser}' not available" >&2; exit 1 fi if ! id $workuser_pv >/dev/null 2>&1 ; then echo "$pgm: sudo user '${workuser_pv}' not available" >&2; exit 1 fi # Check directories for f in "${workuser_dir}" "${root_dir}" "${stage_dir}" \ "${workuser_pv_dir}" "${root_dir_pv}" "${stage_dir_pv}"; do if [ ! -d "$f" ]; then echo "$pgm: directory '$f' missing" >&2; exit 1 fi done want="2775:${workuser}:${mainuser}" for f in "${workuser_dir}" "${stage_dir}"; do x=$(stat -c '%a:%U:%G' "$f") if [ x"$x" != x"$want" ]; then echo "$pgm: directory '$f' has wrong permissions" >&2 echo "$pgm: want: $want" >&2 echo "$pgm: have: $x" >&2 exit 1 fi done want="2775:${workuser_pv}:${mainuser}" for f in "${workuser_pv_dir}" "${stage_dir_pv}"; do x=$(stat -c '%a:%U:%G' "$f") if [ x"$x" != x"$want" ]; then echo "$pgm: directory '$f' has wrong permissions" >&2 echo "$pgm: want: $want" >&2 echo "$pgm: have: $x" >&2 exit 1 fi done cd "${root_dir}" # # Take a lock so that only one instance of this script runs. # if ! lockfile -l 7200 -r 2 $LOCKFILE; then echo "$pgm: another instance is still running" >&2 exit 0 fi trap "rm -f $LOCKFILE" 0 # These flags are set to the stage directory if a sync is required sync_web= sync_blog= sync_preview= # # Build main part # subdir=web revlastfile="${log_dir}/${reponame}.$(echo $subdir | tr / _).revlast" buildlog="${log_dir}/${reponame}.$(echo $subdir | tr / _).log" rev="$(git rev-parse --verify HEAD:$subdir)" if [ -z "$rev" ]; then echo "$pgm: No git revision found" >&2; exit 1 fi revlast="$(head -1 ${revlastfile} 2>/dev/null || true)" if [ x"$rev" = x"$revlast" ]; then echo "$pgm: No need to build $subdir" >&2; else echo "$(date -u -Iseconds) build started for $subdir" | tee ${buildlog} if [ ! -d ${stage_dir}/${subdir} ]; then sudo -u webbuild-x mkdir ${stage_dir}/${subdir} fi sudo 2>>${buildlog} -u webbuild-x emacs24 -q --batch \ --eval "(require 'assoc)" \ --eval "(require 'org)" \ --eval "(setq make-backup-files nil)" \ --eval "(setq gpgweb-root-dir \"${root_dir}/${subdir}/\")" \ --eval "(setq gpgweb-stage-dir \"${stage_dir}/${subdir}/\")" \ --eval "(require 'gpgweb (concat gpgweb-root-dir \"share/gpgweb.el\"))" \ --eval "(setq org-publish-use-timestamps-flag nil)" \ --eval "(setq org-export-html-toplevel-hlevel 1)" \ --eval "(setq org-export-html-coding-system 'utf-8)" \ --eval "(gpgweb-setup-project)" \ --eval "(org-publish-initialize-cache \"gpgweb\")" \ --eval "(message \"root=(%s)\" gpgweb-root-dir)" \ - --eval "(org-publish \"gpgweb\" t nil)" + --eval "(org-publish \"gpgweb\" t nil)" \ + --eval "(gpgweb-faq-to-txt \"gnupg-faq.org\")" echo "$rev" > ${revlastfile} sync_web=${stage_dir}/${subdir} echo "$(date -u -Iseconds) build finished for $subdir" | tee -a ${buildlog} fi # # Build blogs # subdir=misc/blog.gnupg.org revlastfile="${log_dir}/${reponame}.$(echo $subdir | tr / _).revlast" buildlog="${log_dir}/${reponame}.$(echo $subdir | tr / _).log" rev="$(git rev-parse --verify HEAD:$subdir)" if [ -z "$rev" ]; then echo "$pgm: No git revision found" >&2; exit 1 fi revlast="$(head -1 ${revlastfile} 2>/dev/null || true)" if [ x"$rev" = x"$revlast" ]; then echo "$pgm: No need to build $subdir" >&2; else echo "$(date -u -Iseconds) build started for $subdir" | tee ${buildlog} if [ ! -d ${stage_dir}/${subdir} ]; then sudo -u webbuild-x mkdir -p ${stage_dir}/${subdir} fi cd ${stage_dir}/${subdir} # We need to initialize that org cache to use our own publish function # despite that we do not use any org-publish feature echo "$pgm: Rendering blogs" >&2 sudo 2>>${buildlog} -u webbuild-x emacs24 -q --batch \ --eval "(require 'assoc)" \ --eval "(require 'org)" \ --eval "(setq gpgweb-root-dir \"${root_dir}/web/\")" \ --eval "(setq gpgweb-blog-dir \"${root_dir}/${subdir}/\")" \ --eval "(setq gpgweb-stage-dir \"${stage_dir}/${subdir}/\")" \ --eval "(require 'gpgweb (concat gpgweb-root-dir \"share/gpgweb.el\"))" \ --eval "(setq org-publish-use-timestamps-flag nil)" \ --eval "(setq org-export-html-toplevel-hlevel 1)" \ --eval "(setq org-export-html-coding-system 'utf-8)" \ --eval "(gpgweb-setup-project)" \ --eval "(org-publish-initialize-cache \"gpgweb\")" \ --eval "(message \"root=(%s)\" gpgweb-root-dir)" \ --eval "(gpgweb-publish-blogs)" echo "$pgm: Updating blog index" >&2 indexcreator="${root_dir}/${subdir}/update-index.sh" if [ ! -f $indexcreator ]; then echo "$pgm: $indexcreator not found" >&2 exit 1 fi sudo -u webbuild-x ${indexcreator} echo "$rev" > ${revlastfile} sync_blog=${stage_dir}/${subdir} echo "$(date -u -Iseconds) build finished for $subdir" | tee -a ${buildlog} fi # # Build the preview site (w/o blogs) # branch=preview subdir=web cd "${root_dir_pv}" revlastfile="${log_dir}/${reponame}.$branch.$(echo $subdir | tr / _).revlast" buildlog="${log_dir}/${reponame}.$branch.$(echo $subdir | tr / _).log" rev="$(git rev-parse --verify $branch:$subdir)" if [ -z "$rev" ]; then echo "$pgm: No git revision found" >&2; exit 1 fi revlast="$(head -1 ${revlastfile} 2>/dev/null || true)" if [ x"$rev" = x"$revlast" ]; then echo "$pgm: No need to build $branch:$subdir" >&2; else echo "$(date -u -Iseconds) build started for $branch:$subdir" | tee ${buildlog} if [ ! -d ${stage_dir_pv}/${subdir} ]; then sudo -u webbuild-y mkdir ${stage_dir_pv}/${subdir} fi sudo 2>>${buildlog} -u webbuild-y emacs24 -q --batch \ --eval "(require 'assoc)" \ --eval "(require 'org)" \ --eval "(setq make-backup-files nil)" \ --eval "(setq gpgweb-root-dir \"${root_dir_pv}/${subdir}/\")" \ --eval "(setq gpgweb-stage-dir \"${stage_dir_pv}/${subdir}/\")" \ --eval "(require 'gpgweb (concat gpgweb-root-dir \"share/gpgweb.el\"))" \ --eval "(setq org-publish-use-timestamps-flag nil)" \ --eval "(setq org-export-html-toplevel-hlevel 1)" \ --eval "(setq org-export-html-coding-system 'utf-8)" \ --eval "(gpgweb-setup-project)" \ --eval "(org-publish-initialize-cache \"gpgweb\")" \ --eval "(message \"root=(%s)\" gpgweb-root-dir)" \ --eval "(org-publish \"gpgweb\" t nil)" echo "$rev" > ${revlastfile} sync_preview=${stage_dir_pv}/${subdir} echo "$(date -u -Iseconds) build finished for $branch:$subdir" | tee -a ${buildlog} fi cd "${root_dir}" # # Sync to the webspace # cd "${root_dir}" any_sync= if [ -n "$sync_web" ]; then cd "$sync_web" rsync -rlt --exclude '*~' --exclude '*.tmp' \ . ${htdocs_web}/ touch ${htdocs_web}/donate/donors.dat cd ${htdocs_web} ln -sf ../../howtos.gnupg.org/htdocs howtos ln -sf software related_software ln -sf software/index.html features.html cd "$sync_web" any_sync=yes fi if [ -n "$sync_blog" ]; then cd "$sync_blog" rsync -rt --links --exclude '*~' --exclude '*.sh' \ --exclude '*tmp' --exclude '*.org' . ${htdocs_blog}/ cd "$root_dir/misc/blog.gnupg.org" rsync -rt --links --exclude '*~' --exclude '*.sh' \ --exclude '*tmp' --exclude '*.org' img data ${htdocs_blog}/ any_sync=yes fi if [ -n "$sync_preview" ]; then cd "$sync_preview" rsync -rlt --exclude '*~' --exclude '*.tmp' \ . ${htdocs_preview}/ $HOME/bin/mkkudos.sh --verbose --force --test fi cd "${root_dir}" if [ "$any_sync" = yes ]; then $HOME/bin/mkkudos.sh --verbose --force fi # # Print warnings when the scripts are out of date # (For security reasons the scripts need to be installed manually.) # for f in trigger-website-build build-website.sh mkkudos.sh \ append-to-donors.sh ; do if ! cmp -s ${HOME}/bin/$f tools/$f ; then echo "$pgm: Warning: A newer version of $f is available" >&2; fi done exit 0 diff --git a/web/share/gpgweb.el b/web/share/gpgweb.el index 6539206..2651c35 100644 --- a/web/share/gpgweb.el +++ b/web/share/gpgweb.el @@ -1,527 +1,543 @@ ;;; gpgweb.el --- elisp helper code for the GnuPG web pages (if (< (string-to-number emacs-version) 24) (require 'org-exp)) ;; makeindex disabled because the generated file is created in the ;; source directory. (defun gpgweb-setup-project () "Set up an org-publish project for the gnupg.org website." (progn (require 'ox-gpgweb (concat gpgweb-root-dir "share/ox-gpgweb.el")) (aput 'org-publish-project-alist "gpgweb-org" `(:base-directory ,gpgweb-root-dir :base-extension "org" :language "en" :html-extension "html" :recursive t :publishing-directory ,gpgweb-stage-dir :publishing-function gpgweb-org-to-html :body-only t :section-numbers nil :tags nil :with-toc nil :makeindex nil :auto-sitemap nil :sitemap-title "GnuPG - Sitemap" :sitemap-sort-folders "last" :sitemap-file-entry-format "%t @@html:@@(%d)@@html:@@" :style-include-default nil :timestamp-file nil :html-head "" :html-head-include-scripts nil)) (aput 'org-publish-project-alist "gpgweb-other" `(:base-directory ,gpgweb-root-dir :base-extension "jpg\\|png\\|css\\|txt\\|rss\\|lst\\|sig\\|js\\|map\\|eot\\|ttf\\|woff\\|woff2\\|svg" :recursive t :publishing-directory ,gpgweb-stage-dir :publishing-function org-publish-attachment :completion-function gpgweb-upload)) (aput 'org-publish-project-alist "gpgweb" '(:components ("gpgweb-org" "gpgweb-other"))) (add-hook 'org-export-before-processing-hook 'gpgweb-preprocess))) (defun gpgweb-preprocess (backend) "Insert certain stuff before processing." (let () (goto-char (point-min)) (when (re-search-forward "^#\\+GPGWEB-NEED-SWDB\\b" 2048 t) (beginning-of-line) (kill-line 1) (insert (org-file-contents (concat gpgweb-root-dir "swdb.mac") 'noerror))))) (defun gpgweb-insert-header (title committed-at custom) "Insert the header. COMMITTED-AT is the commit date string of the source file or nil if not available. If CUSTOM is true only a minimal header is set." (goto-char (point-min)) (insert " " title " ") (when (and committed-at (>= (length committed-at) 10)) (insert "\n")) (insert " ") (unless custom (insert " "))) (defconst gpgweb-gnupg-menu-alist '(("/index.html" "Home" (("/index.html" "Home") ("/news.html" "News") ("/people/index.html" "People") ("/verein/index.html" "Verein") ("/documentation/sites.html" "Sites"))) ("/donate/index.html" "Donate" (("/donate/index.html" "Donate") ("/donate/kudos.html" "List of Donors"))) ("/software/index.html" "Software" (("/software/index.html" "GnuPG") ("/software/frontends.html" "Frontends") ("/software/tools.html" "Tools") ("/software/libraries.html" "Libraries") ("/software/swlist.html" "All"))) ("/download/index.html" "Download" (("/download/index.html" "Download") ("/download/integrity_check.html" "Integrity Check") ("/download/supported_systems.html" "Supported Systems") ("/download/release_notes.html" "Release Notes") ("/download/mirrors.html" "Mirrors") ("/download/git.html" "GIT"))) ("/documentation/index.html" "Documentation" (("/documentation/howtos.html" "HOWTOs") ("/documentation/manuals.html" "Manuals") ("/documentation/guides.html" "Guides") ("/documentation/faqs.html" "FAQs") ("/documentation/mailing-lists.html" "Mailing Lists") ("/service.html" "3rd Party Support") ("/documentation/bts.html" "Bug Tracker") ("/documentation/security.html" "Security"))) ("/blog/index.html" "Blog")) "The definition of the gnupg.org menu structure.") (defconst gpgweb-gnupg-bottom-menu-alist '(("/privacy-policy.html" "Privacy Policy" ()) ("/imprint.html" "Imprint" ()) ("/misc/index.html" "Archive" ()) ("/sitemap.html" "Sitemap" ()) ("/blog/index.html" "Blog" ()) ("/ftp/index.html" "Files" ())) "The definition of the gnupg.org bottom menu structure.") (defun gpgweb--any-selected-menu-p (menu selected-file) "Return t if any item in MENU has been selected." (let ((item (car menu)) res) (when menu (when item (when (string= (car item) selected-file) (setq res t)) (when (caddr item) (when (gpgweb--any-selected-menu-p (caddr item) selected-file) (setq res t)))) (when (gpgweb--any-selected-menu-p (cdr menu) selected-file) (setq res t))) res)) (defun gpgweb--selected-top-menu (menu selected-file) "Return the selected top menu or nil." (when menu (let ((item (car menu))) (if (and item (or (string= (car item) selected-file) (gpgweb--any-selected-menu-p (caddr item) selected-file))) menu (gpgweb--selected-top-menu (cdr menu) selected-file))))) (defun gpgweb--insert-menu (menu lvl selected-file) "Helper function to insert the menu." (when menu (let ((item (car menu))) (when item (dotimes (i (1+ lvl)) (insert " ")) (insert "
  • " (cadr item) "\n") (when (caddr item) (dotimes (i (1+ lvl)) (insert " ")) (insert " \n")) (dotimes (i (1+ lvl)) (insert " ")) (insert "
    • \n"))) (gpgweb--insert-menu (cdr menu) lvl selected-file))) (defun gpgweb--insert-submenu (menu selected-file) "Helper function to insert the sub-menu." (when menu (let ((item (car menu))) (when item (insert "
  • " (cadr item) "
    • \n"))) (gpgweb--insert-submenu (cdr menu) selected-file))) (defun gpgweb-insert-menu (selected-file) "Insert the menu structure into the HTML file." (goto-char (point-min)) (when (re-search-forward "^\n" nil t) (insert "
     
    ") (let ((m (caddr (car (gpgweb--selected-top-menu gpgweb-gnupg-menu-alist selected-file))))) (when m (insert "\n"))) (insert "
    "))) (defun gpgweb-blog-index (orgfile filelist) "Return the index of ORGFILE in FILELIST or nil if not found." (let (found (i 0)) (while (and filelist (not found)) (if (string= orgfile (car filelist)) (setq found i)) (setq i (1+ i)) (setq filelist (cdr filelist))) found)) (defun gpgweb-blog-prev (fileidx filelist) "Return the chronological previous file at FILEIDX from FILELIST with the suffixed replaced by \"html\"." (if (> fileidx 1) (concat (file-name-sans-extension (nth (1- fileidx) filelist)) ".html"))) (defun gpgweb-blog-next (orgfile filelist) "Return the chronological next file at FILEIDX from FILELIST with the suffixed replaced by \"html\"." (if (< fileidx (1- (length filelist))) (concat (file-name-sans-extension (nth (1+ fileidx) filelist)) ".html"))) (defun gpgweb-fixup-blog (info orgfile filelist) "Insert the blog specific content. INFO is the usual plist. ORGFILE is the name of the current source file without the directory part. If FILELIST is a list it has an ordered list of org filenames." (let ((authorstr (car (plist-get info :author))) (datestr (car (plist-get info :date)))) (goto-char (point-min)) (if (re-search-forward "^
    " nil t) (let* ((indexp (string= orgfile "index.org")) (fileidx (if (listp filelist) (if indexp (1- (length filelist)) (gpgweb-blog-index orgfile filelist)))) (prevfile (if fileidx (gpgweb-blog-prev fileidx filelist))) (nextfile (if (and fileidx (not indexp)) (gpgweb-blog-next fileidx filelist)))) (move-beginning-of-line nil) (insert "\n"))) (if (and datestr authorstr) (if (re-search-forward "^

    Posted " datestr " by " authorstr "

    \n"))))) (defun gpgweb-insert-footer (htmlfile committed-at blogmode) "Insert the footer. HTMLFILE is HTML file name and COMMITTED-AT is the commit date string of the source file or nil if not available." (let ((srcfile (concat "https://git.gnupg.org/cgi-bin/gitweb.cgi?" "p=gnupg-doc.git;a=blob;f=" (if blogmode "misc/blog.gnupg.org" "web/") ;; The replace below is a hack to cope with ;; blogmode where HTMLFILE is like "./foo.html". (replace-regexp-in-string "^\\./" "/" (file-name-sans-extension htmlfile) t) ".org")) (changed (if (and committed-at (>= (length committed-at) 10)) (substring committed-at 0 10) "[unknown]"))) (goto-char (point-max)) (insert "

      ") (gpgweb--insert-menu gpgweb-gnupg-bottom-menu-alist 0 nil) (insert "
    ") (insert "
    \"Traueranzeige:

    ") (goto-char (point-min)) (unless (search-forward "" nil t) (goto-char (point-max)) (if (string-prefix-p "verein/" htmlfile) (insert "
    \"CC  This web page is Copyright 2017 GnuPG e.V. and licensed under a Creative Commons Attribution-ShareAlike 4.0 International License. See copying for details. Page source last changed on " changed ".
    \n") (insert "
    \"CC  These web pages are Copyright 1998--2017 The GnuPG Project and licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License. See copying for details. Page source last changed on " changed ".
    \n"))) (goto-char (point-max)) (insert "
    "))) (defun gpgweb-publish-find-title (file &optional reset) "Find the title of FILE in project. This is a copy of org-publish-find-title which switches the buffer into read-write mode so that it works with read-only files." (or (and (not reset) (org-publish-cache-get-file-property file :title nil t)) (let* ((org-inhibit-startup t) (visiting (find-buffer-visiting file)) (buffer (or visiting (find-file-noselect file)))) (with-current-buffer buffer (toggle-read-only 0) (let ((title (let ((property (plist-get ;; protect local variables in open buffers (if visiting (org-export-with-buffer-copy (org-export-get-environment)) (org-export-get-environment)) :title))) (if property (org-no-properties (org-element-interpret-data property)) (file-name-nondirectory (file-name-sans-extension file)))))) (unless visiting (kill-buffer buffer)) (org-publish-cache-set-file-property file :title title) title))))) (defun gpgweb-want-custom-page-p () "Return true if the current buffer indicated that it wants to be a custom page." (let ((savepoint (point)) (result)) (goto-char (point-min)) (setq result (not (not (search-forward "" nil t)))) (goto-char savepoint) result)) (defun gpgweb-postprocess-html (plist orgfile htmlfile blogmode) "Post-process the generated HTML file - Insert header and footer - Insert \"class=selected\" into the active menu entry - Fixup sitemap. If blogmode is not nil the output is rendered as a blog. BLOGMODE may then contain an ordered list of org file names which are used to create the previous and Next links for an entry." (let* ((visitingp (find-buffer-visiting htmlfile)) (work-buffer (or visitingp (find-file-noselect htmlfile))) (committed-at (shell-command-to-string (concat "git" (if blogmode (concat " -C " gpgweb-blog-dir)) " log -1 --format='%ci' -- " orgfile)))) (prog1 (with-current-buffer work-buffer (let ((fname (file-name-nondirectory htmlfile)) (fname-2 (replace-regexp-in-string ".*/gnupg-doc-stage/web/\\(.*\\)$" "\\1" htmlfile t)) (title (gpgweb-publish-find-title orgfile)) (custom (gpgweb-want-custom-page-p))) ;; Insert header, menu, and footer. (gpgweb-insert-header title committed-at custom) (unless custom (goto-char (point-min)) (unless (search-forward "" nil t) (gpgweb-insert-menu fname-2)) (if blogmode (gpgweb-fixup-blog plist (file-name-nondirectory orgfile) blogmode)) (gpgweb-insert-footer fname-2 committed-at blogmode)) ; Fixup the sitemap (when (string-equal fname "sitemap.html") (goto-char (point-min)) (while (re-search-forward "^.*
  • .*>\\(GnuPG - \\).* and ; attributes. (goto-char (point-min)) (when (search-forward "" nil t) (goto-char (point-min)) (while (re-search-forward "^