[RFC/PATCH 0/5] Add render filter

[RFC/PATCH 0/5] Add render filter

[john]

This is an initial attempt at implementing a mechanism for rendering
text documents as HTML in the tree view (Markdown, ReStructuredText,
etc.).  Filters are specified by file extension in the same way as MIME
type mappings, but I have not added support for overriding these in
per-repository configuration.  This makes it like MIME type mapping but
unlike the source filter, which can be specific per-repository.

A new "source" view is introduced to get back to the source, as it is
currently displayed, when a renderer is available and is being used by
default.

The tree view changes so that if a render filter is specified in the
configuration, the rendered content is used.  As an additional
enhancement, if a MIME type is specified, the content is included inline
served via the "plain" page.  This allows displaying images inline
without needing to fork a filter process just to write an HTML wrapper.

The stylesheet probably needs some work, any suggestions or follow up
patches will be gratefully received.

John Keeping (5):
  Use string list strdup_strings for mimetypes
  Add "source" page
  Parse render filters from the config
  ui-tree: split out buffer printing
  ui-tree: use render filters to display content

 cgit.c       |  19 ++++++++-
 cgit.css     |   5 +++
 cgit.h       |   4 +-
 cgitrc.5.txt |  18 +++++++++
 cmd.c        |   8 +++-
 filter.c     |   1 +
 shared.c     |  21 ++++++++++
 ui-shared.c  |  10 +++++
 ui-shared.h  |   3 ++
 ui-tree.c    | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 ui-tree.h    |   2 +-
 11 files changed, 196 insertions(+), 19 deletions(-)

-- 
2.10.0.rc0.142.g1e9f63b

[RFC/PATCH 1/5] Use string list strdup_strings for mimetypes

[john]

There's no need to do this manually with the string list API will do it
for us.

Signed-off-by: John Keeping <john at keeping.me.uk>
---
This is a preliminary cleanup I noticed while in the area.

 cgit.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cgit.c b/cgit.c
index 9427c4a..de0bdb8 100644
--- a/cgit.c
+++ b/cgit.c
@@ -23,7 +23,7 @@ static void add_mimetype(const char *name, const char *value)
 {
 	struct string_list_item *item;
 
-	item = string_list_insert(&ctx.cfg.mimetypes, xstrdup(name));
+	item = string_list_insert(&ctx.cfg.mimetypes, name);
 	item->util = xstrdup(value);
 }
 
@@ -414,7 +414,7 @@ static void prepare_context(void)
 	ctx.page.modified = time(NULL);
 	ctx.page.expires = ctx.page.modified;
 	ctx.page.etag = NULL;
-	memset(&ctx.cfg.mimetypes, 0, sizeof(struct string_list));
+	string_list_init(&ctx.cfg.mimetypes, 1);
 	if (ctx.env.script_name)
 		ctx.cfg.script_name = xstrdup(ctx.env.script_name);
 	if (ctx.env.query_string)
-- 
2.10.0.rc0.142.g1e9f63b

[RFC/PATCH 2/5] Add "source" page

[john]

We are about to introduce rendering of content for the tree view.  This
source page will allow bypassing the renderer and accessing the content
of the current tree view.

Signed-off-by: John Keeping <john at keeping.me.uk>
---
 cmd.c       |  6 ++++++
 ui-shared.c | 10 ++++++++++
 ui-shared.h |  3 +++
 3 files changed, 19 insertions(+)

diff --git a/cmd.c b/cmd.c
index d280e95..18cc980 100644
--- a/cmd.c
+++ b/cmd.c
@@ -129,6 +129,11 @@ static void refs_fn(void)
 	cgit_print_refs();
 }
 
+static void source_fn(void)
+{
+	cgit_print_tree(ctx.qry.sha1, ctx.qry.path);
+}
+
 static void snapshot_fn(void)
 {
 	cgit_print_snapshot(ctx.qry.head, ctx.qry.sha1, ctx.qry.path,
@@ -177,6 +182,7 @@ struct cgit_cmd *cgit_get_cmd(void)
 		def_cmd(refs, 1, 0, 0),
 		def_cmd(repolist, 0, 0, 0),
 		def_cmd(snapshot, 1, 0, 0),
+		def_cmd(source, 1, 1, 0),
 		def_cmd(stats, 1, 1, 0),
 		def_cmd(summary, 1, 0, 0),
 		def_cmd(tag, 1, 0, 0),
diff --git a/ui-shared.c b/ui-shared.c
index e39d004..36b6108 100644
--- a/ui-shared.c
+++ b/ui-shared.c
@@ -297,6 +297,12 @@ void cgit_tree_link(const char *name, const char *title, const char *class,
 	reporevlink("tree", name, title, class, head, rev, path);
 }
 
+void cgit_source_link(const char *name, const char *title, const char *class,
+		    const char *head, const char *rev, const char *path)
+{
+	reporevlink("source", name, title, class, head, rev, path);
+}
+
 void cgit_plain_link(const char *name, const char *title, const char *class,
 		     const char *head, const char *rev, const char *path)
 {
@@ -476,6 +482,10 @@ static void cgit_self_link(char *name, const char *title, const char *class)
 		cgit_tree_link(name, title, class, ctx.qry.head,
 			       ctx.qry.has_sha1 ? ctx.qry.sha1 : NULL,
 			       ctx.qry.path);
+	else if (!strcmp(ctx.qry.page, "source"))
+		cgit_source_link(name, title, class, ctx.qry.head,
+				 ctx.qry.has_sha1 ? ctx.qry.sha1 : NULL,
+				 ctx.qry.path);
 	else if (!strcmp(ctx.qry.page, "plain"))
 		cgit_plain_link(name, title, class, ctx.qry.head,
 				ctx.qry.has_sha1 ? ctx.qry.sha1 : NULL,
diff --git a/ui-shared.h b/ui-shared.h
index 87799f1..0b6b23d 100644
--- a/ui-shared.h
+++ b/ui-shared.h
@@ -23,6 +23,9 @@ extern void cgit_tag_link(const char *name, const char *title,
 extern void cgit_tree_link(const char *name, const char *title,
 			   const char *class, const char *head,
 			   const char *rev, const char *path);
+extern void cgit_source_link(const char *name, const char *title,
+			     const char *class, const char *head,
+			     const char *rev, const char *path);
 extern void cgit_plain_link(const char *name, const char *title,
 			    const char *class, const char *head,
 			    const char *rev, const char *path);
-- 
2.10.0.rc0.142.g1e9f63b

[RFC/PATCH 3/5] Parse render filters from the config

[john]

Render filters will be used to present rendered content in the tree
view, for example to display Markdown source rendered as HTML.

We will add support for using these from the tree view in the following
commits.

Signed-off-by: John Keeping <john at keeping.me.uk>
---
 cgit.c       | 15 +++++++++++++++
 cgit.h       |  4 +++-
 cgitrc.5.txt | 18 ++++++++++++++++++
 filter.c     |  1 +
 shared.c     | 21 +++++++++++++++++++++
 5 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/cgit.c b/cgit.c
index de0bdb8..211cfd7 100644
--- a/cgit.c
+++ b/cgit.c
@@ -27,6 +27,18 @@ static void add_mimetype(const char *name, const char *value)
 	item->util = xstrdup(value);
 }
 
+static void add_render_filter(const char *name, const char *cmd)
+{
+	struct string_list_item *item;
+	struct cgit_filter *filter = cgit_new_filter(cmd, RENDER);
+
+	if (!filter)
+		return;
+
+	item = string_list_insert(&ctx.cfg.render_filters, name);
+	item->util = filter;
+}
+
 static void process_cached_repolist(const char *path);
 
 static void repo_config(struct cgit_repo *repo, const char *name, const char *value)
@@ -282,6 +294,8 @@ static void config_cb(const char *name, const char *value)
 			ctx.cfg.branch_sort = 0;
 	} else if (starts_with(name, "mimetype."))
 		add_mimetype(name + 9, value);
+	else if (starts_with(name, "render."))
+		add_render_filter(name + 7, value);
 	else if (!strcmp(name, "include"))
 		parse_configfile(expand_macros(value), config_cb);
 }
@@ -415,6 +429,7 @@ static void prepare_context(void)
 	ctx.page.expires = ctx.page.modified;
 	ctx.page.etag = NULL;
 	string_list_init(&ctx.cfg.mimetypes, 1);
+	string_list_init(&ctx.cfg.render_filters, 1);
 	if (ctx.env.script_name)
 		ctx.cfg.script_name = xstrdup(ctx.env.script_name);
 	if (ctx.env.query_string)
diff --git a/cgit.h b/cgit.h
index 325432b..f1e2315 100644
--- a/cgit.h
+++ b/cgit.h
@@ -55,7 +55,7 @@ typedef enum {
 } diff_type;
 
 typedef enum {
-	ABOUT, COMMIT, SOURCE, EMAIL, AUTH, OWNER
+	ABOUT, COMMIT, SOURCE, EMAIL, AUTH, OWNER, RENDER
 } filter_type;
 
 struct cgit_filter {
@@ -261,6 +261,7 @@ struct cgit_config {
 	int branch_sort;
 	int commit_sort;
 	struct string_list mimetypes;
+	struct string_list render_filters;
 	struct cgit_filter *about_filter;
 	struct cgit_filter *commit_filter;
 	struct cgit_filter *source_filter;
@@ -390,5 +391,6 @@ extern int readfile(const char *path, char **buf, size_t *size);
 extern char *expand_macros(const char *txt);
 
 extern char *get_mimetype_for_filename(const char *filename);
+extern struct cgit_filter *get_render_for_filename(const char *filename);
 
 #endif /* CGIT_H */
diff --git a/cgitrc.5.txt b/cgitrc.5.txt
index 9fcf445..f322968 100644
--- a/cgitrc.5.txt
+++ b/cgitrc.5.txt
@@ -354,6 +354,17 @@ renamelimit::
 	 "-1" uses the compiletime value in git (for further info, look at
 	  `man git-diff`). Default value: "-1".
 
+render.<ext>::
+	Specifies a command which will be invoked to render files with the
+	extension `.<ext>`. The command will get the blob content on its STDIN
+	and the name of the blob as its only command line argument. The STDOUT
+	from the command will be included verbatim in the page content. If no
+	render filter is available for a given file extension but the mimetype
+	is specified then the content will be included as an iframe, otherwise
+	the normal source rendering will be used.
++
+Default value: none. See also: "FILTER API".
+
 repo.group::
 	Legacy alias for "section". This option is deprecated and will not be
 	supported in cgit-1.0.
@@ -694,6 +705,13 @@ source filter::
 	file that is to be filtered is available on standard input and the
 	filtered contents is expected on standard output.
 
+render filter::
+	This filter is given a single parameter: the filename of the source
+	file to render. The filter can use the filename to determine (for
+	example) the syntax highlighting mode. The contents of the file that
+	is to be rendered is available on standard input and the rendered
+	content is expected on standard output.
+
 auth filter::
 	The authentication filter receives 12 parameters:
 	  - filter action, explained below, which specifies which action the
diff --git a/filter.c b/filter.c
index 949c931..92f67d7 100644
--- a/filter.c
+++ b/filter.c
@@ -433,6 +433,7 @@ struct cgit_filter *cgit_new_filter(const char *cmd, filter_type filtertype)
 
 		case SOURCE:
 		case ABOUT:
+		case RENDER:
 			argument_count = 1;
 			break;
 
diff --git a/shared.c b/shared.c
index a63633b..79a653a 100644
--- a/shared.c
+++ b/shared.c
@@ -601,3 +601,24 @@ char *get_mimetype_for_filename(const char *filename)
 	fclose(file);
 	return NULL;
 }
+
+struct cgit_filter *get_render_for_filename(const char *filename)
+{
+	char *ext;
+	struct string_list_item *item;
+
+	if (!filename)
+		return NULL;
+
+	ext = strrchr(filename, '.');
+	if (!ext)
+		return NULL;
+	++ext;
+	if (!ext[0])
+		return NULL;
+	item = string_list_lookup(&ctx.cfg.render_filters, ext);
+	if (item)
+		return item->util;
+
+	return NULL;
+}
-- 
2.10.0.rc0.142.g1e9f63b

[RFC/PATCH 4/5] ui-tree: split out buffer printing

[john]

This is a preliminary refactoring for the next commit.

Signed-off-by: John Keeping <john at keeping.me.uk>
---
 ui-tree.c | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/ui-tree.c b/ui-tree.c
index 120066c..5c715a1 100644
--- a/ui-tree.c
+++ b/ui-tree.c
@@ -115,7 +115,22 @@ static void set_title_from_path(const char *path)
 	ctx.page.title = new_title;
 }
 
-static void print_object(const unsigned char *sha1, char *path, const char *basename, const char *rev)
+static void print_buffer(const char *basename, char *buf, unsigned long size)
+{
+	if (ctx.cfg.max_blob_size && size / 1024 > ctx.cfg.max_blob_size) {
+		htmlf("<div class='error'>blob size (%ldKB) exceeds display size limit (%dKB).</div>",
+				size / 1024, ctx.cfg.max_blob_size);
+		return;
+	}
+
+	if (buffer_is_binary(buf, size))
+		print_binary_buffer(buf, size);
+	else
+		print_text_buffer(basename, buf, size);
+}
+
+static void print_object(const unsigned char *sha1, char *path,
+		const char *basename, const char *rev)
 {
 	enum object_type type;
 	char *buf;
@@ -143,16 +158,7 @@ static void print_object(const unsigned char *sha1, char *path, const char *base
 		        rev, path);
 	html(")\n");
 
-	if (ctx.cfg.max_blob_size && size / 1024 > ctx.cfg.max_blob_size) {
-		htmlf("<div class='error'>blob size (%ldKB) exceeds display size limit (%dKB).</div>",
-				size / 1024, ctx.cfg.max_blob_size);
-		return;
-	}
-
-	if (buffer_is_binary(buf, size))
-		print_binary_buffer(buf, size);
-	else
-		print_text_buffer(basename, buf, size);
+	print_buffer(basename, buf, size);
 }
 
 
-- 
2.10.0.rc0.142.g1e9f63b

[RFC/PATCH 5/5] ui-tree: use render filters to display content

[john]

This allows applying filters to files in the repository, for example to
render Markdown or AsciiDoc as HTML.

Signed-off-by: John Keeping <john at keeping.me.uk>
---
 cgit.css  |   5 +++
 cmd.c     |   4 +--
 ui-tree.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 ui-tree.h |   2 +-
 4 files changed, 105 insertions(+), 10 deletions(-)

diff --git a/cgit.css b/cgit.css
index a4331f8..b0970de 100644
--- a/cgit.css
+++ b/cgit.css
@@ -254,6 +254,11 @@ div#cgit div#blob {
 	border: solid 1px black;
 }
 
+div#cgit iframe {
+	width: 100%;
+	height: 40em;
+}
+
 div#cgit div.error {
 	color: red;
 	font-weight: bold;
diff --git a/cmd.c b/cmd.c
index 18cc980..544d705 100644
--- a/cmd.c
+++ b/cmd.c
@@ -131,7 +131,7 @@ static void refs_fn(void)
 
 static void source_fn(void)
 {
-	cgit_print_tree(ctx.qry.sha1, ctx.qry.path);
+	cgit_print_tree(ctx.qry.sha1, ctx.qry.path, false);
 }
 
 static void snapshot_fn(void)
@@ -157,7 +157,7 @@ static void tag_fn(void)
 
 static void tree_fn(void)
 {
-	cgit_print_tree(ctx.qry.sha1, ctx.qry.path);
+	cgit_print_tree(ctx.qry.sha1, ctx.qry.path, true);
 }
 
 #define def_cmd(name, want_repo, want_vpath, is_clone) \
diff --git a/ui-tree.c b/ui-tree.c
index 5c715a1..ed41c3c 100644
--- a/ui-tree.c
+++ b/ui-tree.c
@@ -14,6 +14,7 @@
 struct walk_tree_context {
 	char *curr_rev;
 	char *match_path;
+	bool use_render;
 	int state;
 };
 
@@ -129,11 +130,70 @@ static void print_buffer(const char *basename, char *buf, unsigned long size)
 		print_text_buffer(basename, buf, size);
 }
 
+static void render_buffer(struct cgit_filter *render, const char *name,
+		char *buf, unsigned long size)
+{
+	char *filter_arg = xstrdup(name);
+
+	html("<div class='blob'>");
+	cgit_open_filter(render, filter_arg);
+	html_raw(buf, size);
+	cgit_close_filter(render);
+	html("</div>");
+
+	free(filter_arg);
+}
+
+static void include_file(const unsigned char *sha1, const char *path,
+		const char *mimetype)
+{
+	const char *delim = "?";
+
+	html("<div class='blob'>");
+
+	if (!strncmp(mimetype, "image/", 6)) {
+		html("<img alt='");
+		html_attr(path);
+		html("' src='");
+	} else {
+		html("<iframe sandbox='allow-scripts' src='");
+	}
+
+	if (ctx.cfg.virtual_root) {
+		html_url_path(ctx.cfg.virtual_root);
+		html_url_path(ctx.repo->url);
+		if (ctx.repo->url[strlen(ctx.repo->url) - 1] != '/')
+			html("/");
+		html("plain/");
+		html_url_path(path);
+	} else {
+		html_url_path(ctx.cfg.script_name);
+		html("?url=");
+		html_url_arg(ctx.repo->url);
+		if (ctx.repo->url[strlen(ctx.repo->url) - 1] != '/')
+			html("/");
+		html("plain/");
+		if (path)
+			html_url_arg(path);
+		delim = "&amp;";
+	}
+	if (ctx.qry.head && ctx.repo->defbranch &&
+	    strcmp(ctx.qry.head, ctx.repo->defbranch)) {
+		html(delim);
+		html("h=");
+		html_url_arg(ctx.qry.head);
+		delim = "&amp;";
+	}
+
+	html("'></div>");
+}
+
 static void print_object(const unsigned char *sha1, char *path,
-		const char *basename, const char *rev)
+		const char *basename, const char *rev, bool use_render)
 {
 	enum object_type type;
-	char *buf;
+	struct cgit_filter *render;
+	char *buf, *mimetype;
 	unsigned long size;
 
 	type = sha1_object_info(sha1, &size);
@@ -150,17 +210,44 @@ static void print_object(const unsigned char *sha1, char *path,
 		return;
 	}
 
+	render = get_render_for_filename(path);
+	mimetype = render ? NULL : get_mimetype_for_filename(path);
+
 	set_title_from_path(path);
 
+	/*
+	 * If we don't have a render filter or a mimetype, we won't include the
+	 * file in the page.
+	 */
+	if (!render && !mimetype)
+		use_render = false;
+
 	cgit_print_layout_start();
 	htmlf("blob: %s (", sha1_to_hex(sha1));
 	cgit_plain_link("plain", NULL, NULL, ctx.qry.head,
 		        rev, path);
+	if (use_render) {
+		html(", ");
+		cgit_source_link("source", NULL, NULL, ctx.qry.head,
+				rev, path);
+	} else if (render || mimetype) {
+		html(", ");
+		cgit_tree_link("render", NULL, NULL, ctx.qry.head,
+			       rev, path);
+	}
 	html(")\n");
 
-	print_buffer(basename, buf, size);
-}
+	if (use_render) {
+		if (render)
+			render_buffer(render, basename, buf, size);
+		else
+			include_file(sha1, path, mimetype);
+	} else {
+		print_buffer(basename, buf, size);
+	}
 
+	free(mimetype);
+}
 
 static int ls_item(const unsigned char *sha1, struct strbuf *base,
 		const char *pathname, unsigned mode, int stage, void *cbdata)
@@ -279,7 +366,9 @@ static int walk_tree(const unsigned char *sha1, struct strbuf *base,
 			return READ_TREE_RECURSIVE;
 		} else {
 			walk_tree_ctx->state = 2;
-			print_object(sha1, buffer, pathname, walk_tree_ctx->curr_rev);
+			print_object(sha1, buffer, pathname,
+					walk_tree_ctx->curr_rev,
+					walk_tree_ctx->use_render);
 			return 0;
 		}
 	}
@@ -292,7 +381,7 @@ static int walk_tree(const unsigned char *sha1, struct strbuf *base,
  *   rev:  the commit pointing at the root tree object
  *   path: path to tree or blob
  */
-void cgit_print_tree(const char *rev, char *path)
+void cgit_print_tree(const char *rev, char *path, bool use_render)
 {
 	unsigned char sha1[20];
 	struct commit *commit;
@@ -306,7 +395,8 @@ void cgit_print_tree(const char *rev, char *path)
 	};
 	struct walk_tree_context walk_tree_ctx = {
 		.match_path = path,
-		.state = 0
+		.state = 0,
+		.use_render = use_render,
 	};
 
 	if (!rev)
diff --git a/ui-tree.h b/ui-tree.h
index bbd34e3..a0fc8e2 100644
--- a/ui-tree.h
+++ b/ui-tree.h
@@ -1,6 +1,6 @@
 #ifndef UI_TREE_H
 #define UI_TREE_H
 
-extern void cgit_print_tree(const char *rev, char *path);
+extern void cgit_print_tree(const char *rev, char *path, bool use_render);
 
 #endif /* UI_TREE_H */
-- 
2.10.0.rc0.142.g1e9f63b