[PATCH] netsurf: webfs POST implementation

[PATCH] netsurf: webfs POST implementation

[telephil9]

[-- Attachment #1: Type: text/plain, Size: 259 bytes --]

Hi,

Here is a patch to implement POST requests in netsurf webfs fetcher.
I could not test multipart POST requests yet as I could not find a site that uses this encoding method.
Also, the webfs clients exhaustion issue should now be fixed.

Regards,
Philippe

[-- Attachment #2: Type: text/plain, Size: 5224 bytes --]

--- /mnt/git/object/25c368f547f3c9f3aff3006980e871b2dedcc92e/tree/content/webfs.c	Wed Feb 12 23:12:26 2020
+++ content/webfs.c	Sat Mar 21 19:58:06 2020
@@ -10,6 +10,7 @@
 
 #include "utils/nsurl.h"
 #include "content/llcache.h"
+#include "content/fetch.h"
 #include "netsurf/misc.h"
 #include "desktop/gui_internal.h"
 #include "libwapcaplet/libwapcaplet.h"
@@ -35,7 +36,8 @@ struct webfs_data {
 	data_state state;
 	char *err;
 	char *urls;
-	char *pdat;
+	char *urlenc;
+	struct fetch_multipart_data *multipart;
 
 	int ctlfd; // fd for /mnt/web/N/ctl
 	int bodyfd; // fd for /mnt/web/N/body
@@ -85,8 +87,11 @@ void webfs_data_release(struct webfs_dat
 	if(d->urls != NULL) {
 		free(d->urls);
 	}
-	if(d->pdat != NULL) {
-		free(d->pdat);
+	if(d->urlenc != NULL) {
+		free(d->urlenc);
+	}
+	if(d->multipart != NULL) {
+		fetch_multipart_data_destroy(d->multipart);
 	}
 	if(d->ctlfd != -1) {
 		close(d->ctlfd);
@@ -385,22 +390,20 @@ void start_data(struct webfs_data *d) {
 }
 
 void send_request(struct webfs_data *d) {
-
-	if(d->pdat != NULL) {
-		fprintf(stderr, "[DBG]: Setting POST.\n");
-		int n = write(d->ctlfd, "request POST", 12);
-		if(n <= 0) {
-			char *e = strerror(errno);
-			fprintf(stderr, "[DBG]: [%s] Failed to set POST REQUEST. Error: %s\n", d->urls, e);
-			d->state = DATA_STATE_ERROR;
-			d->err = "Failed to clone webfs."; // TODO: report strerror?
-			return;
-		}
+	char *s;
+	int fd, n;
+	struct fetch_multipart_data *part;
+
+	s = calloc(3+1+strlen(d->urls)+1, sizeof(char));
+	if(s == NULL){
+		fprintf(stderr, "[DBG] send_request: OOM\n");
+		d->state = DATA_STATE_ERROR;
+		d->err = "OOM";
+		return;
 	}
-
-	char urlstr[1024]; // TODO dynamic size?
-	sprintf(urlstr, "url %s", d->urls);
-	int n = write(d->ctlfd, urlstr, strlen(urlstr));
+	sprintf(s, "url %s", d->urls);
+	n = write(d->ctlfd, s, strlen(s));
+	free(s);
 	if(n <= 0) {
 		char *e = strerror(errno);
 		fprintf(stderr, "[DBG]: [%s] Failed to open URL. Error: %s\n", d->urls, e);
@@ -408,7 +411,45 @@ void send_request(struct webfs_data *d) 
 		d->err = "Failed to clone webfs."; // TODO: report strerror?
 		return;
 	}
-
+	/* POST */
+	if(d->urlenc != NULL || d->multipart != NULL) {
+		s = calloc(strlen(d->webdir)+8+1, sizeof(char));
+		sprintf(s, "%spostbody", d->webdir);
+		fd = open(s, O_WRONLY);
+		free(s);
+		if(fd < 0) {
+			char *e = strerror(errno);
+			fprintf(stderr, "[DBG] failed to open postbody file; %s\n", e);
+			d->state = DATA_STATE_ERROR;
+			d->err = "Failed to open POST body";
+			return;
+		}
+		if(d->urlenc != NULL) {
+			n = write(fd, d->urlenc, strlen(d->urlenc));
+			if(n <= 0) {
+				close(fd);
+				char *e = strerror(errno);
+				fprintf(stderr, "[DBG] Could not write post data: Error: %s\n", e);
+				d->state = DATA_STATE_ERROR;
+				d->err = "Could not write post data."; // TODO: report strerror?
+				return;
+			}
+		} else if(d->multipart != NULL) {
+			write(d->ctlfd, "headers Content-Type: multipart/form-data; boundary=HJBOUNDARY", 62);
+			for(part = d->multipart; part; part = part->next) {
+				if(part->file) {
+					fprintf(stderr, "[DBG] webfs::send_request: file part not implemented\n");
+					continue;
+				}
+				n = snprintf(NULL, 0, "--HJBOUNDARY\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", part->name, part->value);
+				s = malloc((n+1)*sizeof(char));
+				sprintf(s, "--HJBOUNDARY\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", part->name, part->value);
+				write(fd, s, n);
+			}
+			write(fd, "--HJBOUNDARY--\r\n", strlen("--HJBOUNDARY--\r\n"));
+		}
+		close(fd);
+	}
 	d->state = DATA_STATE_REQUESTED;
 }
 
@@ -464,6 +505,7 @@ void read_data(struct webfs_data *d) {
 	}
 
 	if(n == 0) {
+		close(d->bodyfd);
 		d->state = DATA_STATE_DONE;
 		return;
 	}
@@ -567,7 +609,6 @@ void update_webfs(void *ignored) {
 	guit->misc->schedule(10, update_webfs, NULL);
 }
 
-
 /**
  * Retrieve a handle for a low-level cache object
  *
@@ -587,16 +628,6 @@ nserror webfs_handle_retrieve(nsurl *url
 
 	// We are going to ignore flags for now.
 
-	if(post != NULL) {
-		// TODO Implement POST data.
-		if(post->type == LLCACHE_POST_URL_ENCODED) {
-			fprintf(stderr, "[DBG]: POST DATA: [%s]\n", post->data.urlenc);
-		} else {
-			fprintf(stderr, "[DBG]: POST DATA NOT IMPLEMENTED!\n");
-		}
-		return NSERROR_BAD_CONTENT;
-	}
-
 	char *scheme = lwc_string_data(nsurl_get_component(url, NSURL_SCHEME));
 	if(strcmp("http", scheme) != 0 && strcmp("https", scheme) != 0) {
 		// TODO proper error handling.
@@ -621,7 +652,21 @@ nserror webfs_handle_retrieve(nsurl *url
 	char *url_cstr = nsurl_access(url);
 	wh->data->urls = malloc(strlen(url_cstr) + 1);
 	strcpy(wh->data->urls, url_cstr);
-	wh->data->pdat = NULL;
+	wh->data->urlenc = NULL;
+	wh->data->multipart = NULL;
+	if(post != NULL) {
+		switch(post->type) {
+		case LLCACHE_POST_URL_ENCODED:
+			wh->data->urlenc = strdup(post->data.urlenc);
+			break;
+		case LLCACHE_POST_MULTIPART:
+			wh->data->multipart = fetch_multipart_data_clone(post->data.multipart);
+			break;
+		default:
+			fprintf(stderr, "[DBG] encode_post_data: unknown post request type %d\n", post->type);
+			break;
+		}
+	}
 	wh->data->ctlfd = -1;
 	wh->data->bodyfd = -1;
 	wh->data->webdir = NULL;

Re: [9front] [PATCH] netsurf: webfs POST implementation

[Kyle Nusbaum]

Thanks a bunch!

The only hesitation I have is with the closing of the body when data is finished. This frees the webfs handle, but if netsurf tries to read any headers after, it will encounter errors. Maybe it is not doing that, but I don't see any reason why it couldn't do that.


On March 21, 2020 2:17:04 PM CDT, telephil9@gmail.com wrote:
>Hi,
>
>Here is a patch to implement POST requests in netsurf webfs fetcher.
>I could not test multipart POST requests yet as I could not find a site
>that uses this encoding method.
>Also, the webfs clients exhaustion issue should now be fixed.
>
>Regards,
>Philippe

-- Kyle

Re: [9front] [PATCH] netsurf: webfs POST implementation

[Kyle Nusbaum]

That said, we *know* we're getting these handle exhaustion errors. I think we can merge this now, and if we start seeing header errors, we can fix that then.

On March 22, 2020 2:25:57 PM CDT, Kyle Nusbaum <knusbaum@sdf.org> wrote:
>Thanks a bunch!
>
>The only hesitation I have is with the closing of the body when data is
>finished. This frees the webfs handle, but if netsurf tries to read any
>headers after, it will encounter errors. Maybe it is not doing that,
>but I don't see any reason why it couldn't do that.
>
>
>On March 21, 2020 2:17:04 PM CDT, telephil9@gmail.com wrote:
>>Hi,
>>
>>Here is a patch to implement POST requests in netsurf webfs fetcher.
>>I could not test multipart POST requests yet as I could not find a
>site
>>that uses this encoding method.
>>Also, the webfs clients exhaustion issue should now be fixed.
>>
>>Regards,
>>Philippe
>
>-- Kyle

-- Kyle

Re: [9front] [PATCH] netsurf: webfs POST implementation

[telephil9]

Tests did not show any issue so far but broader testing could help here, so I'm all for pushing when there is no obvious contraindication (at this stage of the project obviously).

Re: [9front] [PATCH] netsurf: webfs POST implementation

[kokamoto]

Thank you very much Philippe!

> Also, the webfs clients exhaustion issue should now be fixed.

Yes, now I have no error of '/mnt/web/clone no more clients'.

Kenji