From mboxrd@z Thu Jan 1 00:00:00 1970 From: stefan at sevenbyte.org (Stefan Tatschner) Date: Thu, 9 Jan 2014 21:28:15 +0100 Subject: [PATCH] filters: Improved syntax-highlighting.py Message-ID: <1389299295-4518-1-git-send-email-stefan@sevenbyte.org> - Switched back to python2 according to a problem in pygments with python3. With the next release of pygments this problem should be fixed. Issue see here: https://bitbucket.org/birkenfeld/pygments-main/issue/901/problems-with-python3 - Just read the stdin, decode it to utf-8 and ignore unknown signs. This ensures that even destroyed files do not cause any errors in the filter. - As a workaround I added a mapping dict for missing filename mappings in pygments. - Improved language guessing: -> At first use guess_lexer_for_filename for a better detection of the used programming languages (even mixed cases will be detected, e.g. php + html). -> If nothing was found look if there is a shebang and use guess_lexer. -> As default/fallback choose TextLexer. Signed-off-by: Stefan Tatschner --- filters/syntax-highlighting.py | 71 ++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/filters/syntax-highlighting.py b/filters/syntax-highlighting.py index 72d9097..ac38e28 100755 --- a/filters/syntax-highlighting.py +++ b/filters/syntax-highlighting.py @@ -1,13 +1,16 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python2 -# This script uses Pygments and Python3. You must have both installed for this to work. +# This script uses Pygments and Python2. You must have both installed +# for this to work. +# # http://pygments.org/ # http://python.org/ # -# It may be used with the source-filter or repo.source-filter settings in cgitrc. +# It may be used with the source-filter or repo.source-filter settings +# in cgitrc. # -# The following environment variables can be used to retrieve the configuration -# of the repository for which this script is called: +# The following environment variables can be used to retrieve the +# configuration of the repository for which this script is called: # CGIT_REPO_URL ( = repo.url setting ) # CGIT_REPO_NAME ( = repo.name setting ) # CGIT_REPO_PATH ( = repo.path setting ) @@ -18,22 +21,42 @@ import sys -import cgi -import codecs -from pygments.lexers import get_lexer_for_filename -from pygments import highlight -from pygments.formatters import HtmlFormatter - -sys.stdin = codecs.getreader("utf-8")(sys.stdin.detach()) -sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) -doc = sys.stdin.read() -try: - lexer = get_lexer_for_filename(sys.argv[1]) - formatter = HtmlFormatter(style='pastie') - sys.stdout.write("") - - highlight(doc, lexer, formatter, sys.stdout) -except: - sys.stdout.write(str(cgi.escape(doc).encode("ascii", "xmlcharrefreplace"), "ascii")) +import pygments +import pygments.lexers as lexers +import pygments.formatters as formatters +from pygments.util import ClassNotFound +from pygments.lexers import guess_lexer +from pygments.lexers import guess_lexer_for_filename + + +# read stdin and decode to utf-8. ignore any unkown signs. +data = sys.stdin.read().decode(encoding='utf-8', errors='ignore') +filename = sys.argv[1].lower() +formatter = formatters.HtmlFormatter(encoding='utf-8') + +# WORKAROUND: mapping dict for lexers +# just add the expected filename as the key; the corresponding +# pygments lexer instance as the value. +LEXERS = {'cmakelists.txt': lexers.CMakeLexer(encoding='utf-8'), + 'pkgbuild': lexers.BashLexer(encoding='utf-8')} + +# map known filenames to an explicit lexer +if filename in LEXERS: + lexer = LEXERS[filename] +else: + try: + lexer = guess_lexer_for_filename(filename, data, encoding='utf-8') + except ClassNotFound: + # check if there is any shebang + if data[0:2] == '#!': + lexer = guess_lexer(data, encoding='utf-8') + else: + lexer = lexers.TextLexer(encoding='utf-8') + except TypeError: + lexer = lexers.TextLexer(encoding='utf-8') + +# highlight! :-) +sys.stdout.write('') +pygments.highlight(data, lexer, formatter, outfile=sys.stdout) -- 1.8.5.2