From 10d71ba47a3851c3a5031bd2926859898ed5da9d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 14:58:31 +0200 Subject: [PATCH 01/11] added initial webclient API that mirrors the Webserver API and uses the same Response and Request encodings --- src/org/rascalmpl/library/util/Webclient.java | 162 ++++++++++++++++++ src/org/rascalmpl/library/util/Webclient.rsc | 44 +++++ src/org/rascalmpl/library/util/Webserver.java | 6 +- 3 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 src/org/rascalmpl/library/util/Webclient.java create mode 100644 src/org/rascalmpl/library/util/Webclient.rsc diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java new file mode 100644 index 00000000000..f0237f9f11a --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -0,0 +1,162 @@ +package org.rascalmpl.library.util; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.List; + +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.values.IRascalValueFactory; + +import fi.iki.elonen.NanoHTTPD.Response.Status; +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IMap; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeFactory; +import io.usethesource.vallang.type.TypeStore; + +public class Webclient { + private final IRascalValueFactory vf; + private final IRascalMonitor monitor; + private final TypeStore store; + private final TypeFactory tf; + + public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { + this.vf = vf; + this.monitor = monitor; + this.store = store; + this.tf = tf; + } + + public IConstructor fetch(IConstructor input) { + try { + var params = input.asWithKeywordParameters(); + + switch (input.getName()) { + case "get": + HttpRequest request = HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .GET() + .build(); + + HttpResponse response = + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + + return translateTextResponse(response); + default: + // not yet implemented + throw RuntimeExceptionFactory.illegalArgument(input); + } + } + catch (IOException | InterruptedException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } + } + + private IConstructor translateTextResponse(HttpResponse response) { + IMap headers = response + .headers() + .map() + .entrySet() + .stream() + .map(e -> vf.tuple( + vf.string(e.getKey()), + vf.string(e.getValue().stream().collect(Collectors.joining(",")) + ))) + .collect(vf.mapWriter()); + + IString body = vf.string(response.body()); + IConstructor status = toStatusConstructor(response.statusCode()); + + Type respCons = store.lookupConstructors("response").iterator().next(); + + var contentType = response.headers().firstValue("Content-Type"); + + IString mimeType = vf.string(contentType.get().split(";")[0]); + + return vf.constructor(respCons, status, mimeType, headers, body); + } + + IConstructor toStatusConstructor(int stCode) { + Type statusType = store.lookupAbstractDataType("Status"); + + Status status = Status.lookup(stCode); + switch (status) { + case OK: + return vf.constructor(store.lookupConstructor(statusType, "ok", tf.tupleEmpty())); + case NOT_FOUND: + return vf.constructor(store.lookupConstructor(statusType, "notFound", tf.tupleEmpty())); + case ACCEPTED: + return vf.constructor(store.lookupConstructor(statusType, "accepted", tf.tupleEmpty())); + case BAD_REQUEST: + return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())) + case CONFLICT: + return vf.constructor(store.lookupConstructor(statusType, "conflict", tf.tupleEmpty())); + case CREATED: + return vf.constructor(store.lookupConstructor(statusType, "create", tf.tupleEmpty())); + case EXPECTATION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "expectationFailed", tf.tupleEmpty())); + case FORBIDDEN: + return vf.constructor(store.lookupConstructor(statusType, "forbidden", tf.tupleEmpty())); + case FOUND: + return vf.constructor(store.lookupConstructor(statusType, "found", tf.tupleEmpty())); + case GONE: + return vf.constructor(store.lookupConstructor(statusType, "gone", tf.tupleEmpty())); + case INTERNAL_ERROR: + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); + case LENGTH_REQUIRED: + return vf.constructor(store.lookupConstructor(statusType, "lengthRequired", tf.tupleEmpty())); + case METHOD_NOT_ALLOWED: + return vf.constructor(store.lookupConstructor(statusType, "methodNotAllowed", tf.tupleEmpty())); + case MULTI_STATUS: + return vf.constructor(store.lookupConstructor(statusType, "multiStatus", tf.tupleEmpty())); + case NOT_ACCEPTABLE: + return vf.constructor(store.lookupConstructor(statusType, "notAcceptible", tf.tupleEmpty())); + case NOT_IMPLEMENTED: + return vf.constructor(store.lookupConstructor(statusType, "notImplemented", tf.tupleEmpty())); + case NOT_MODIFIED: + return vf.constructor(store.lookupConstructor(statusType, "notModified", tf.tupleEmpty())); + case NO_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "noContent", tf.tupleEmpty())); + case PARTIAL_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "partialContent", tf.tupleEmpty())); + case PAYLOAD_TOO_LARGE: + return vf.constructor(store.lookupConstructor(statusType, "payloadTooLarge", tf.tupleEmpty())); + case PRECONDITION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "preconditionFailed", tf.tupleEmpty())); + case RANGE_NOT_SATISFIABLE: + return vf.constructor(store.lookupConstructor(statusType, "rangeNotSatisfieable", tf.tupleEmpty())); + case REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "redirect", tf.tupleEmpty())); + case REDIRECT_SEE_OTHER: + return vf.constructor(store.lookupConstructor(statusType, "redirectSeeOther", tf.tupleEmpty())); + case REQUEST_TIMEOUT: + return vf.constructor(store.lookupConstructor(statusType, "requestTimeout", tf.tupleEmpty())); + case SERVICE_UNAVAILABLE: + return vf.constructor(store.lookupConstructor(statusType, "serviceUnavailable", tf.tupleEmpty())); + case SWITCH_PROTOCOL: + return vf.constructor(store.lookupConstructor(statusType, "switchProtocol", tf.tupleEmpty())); + case TEMPORARY_REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "temporaryRedirect", tf.tupleEmpty())); + case TOO_MANY_REQUESTS: + return vf.constructor(store.lookupConstructor(statusType, "tooManyRequests", tf.tupleEmpty())); + case UNAUTHORIZED: + return vf.constructor(store.lookupConstructor(statusType, "unauthorized", tf.tupleEmpty())); + case UNSUPPORTED_HTTP_VERSION: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedHTTPVersion", tf.tupleEmpty())); + case UNSUPPORTED_MEDIA_TYPE: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedMediaType", tf.tupleEmpty())); + default: + break; + } + } + } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc new file mode 100644 index 00000000000..edd3d66751a --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -0,0 +1,44 @@ +module util::Webclient + +extend Content; + +@synopsis{Extends ((Content-Status)) with everything HTTP has out there in the wild.} +data Status + = ok() + | notFound() + | accepted() + | badRequest() + | conflict() + | create() + | expectationFailed() + | forbidden() + | found() + | gone() + | internalError() + | lengthRequired() + | methodNotAllowed() + | multiStatus() + | notAcceptible() + | notImplemented() + | notModified() + | noContent() + | partialContent() + | payloadTooLarge() + | preconditionFailed() + | rangeNotSatisfieable() + | redirect() + | redirectSeeOther() + | requestTimeout() + | serviceUnavailable() + | switchProtocol() + | temporaryRedirect() + | tooManyRequests() + | unauthorized() + | unsupportedHTTPVersion() + | unsupportedMediaType() + ; + +data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain"); + +@javaClass{org.rascalmpl.library.util.Webclient} +java Response fetch(Request request); diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 5dd103a0b5e..d8a885cf45f 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -379,11 +379,7 @@ private IMap makeMap(Map headers) { throw RuntimeExceptionFactory.io(e); } } - - - - - + public void shutdown(ISourceLocation server) { NanoHTTPD nano = servers.get(server); if (nano != null) { From 645d334cc36fa4f8c92c05a8b52eeca4a18d5057 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 17:09:12 +0200 Subject: [PATCH 02/11] cleanup and added POST method --- src/org/rascalmpl/library/util/Webclient.java | 62 ++++++++++++------- src/org/rascalmpl/library/util/Webclient.rsc | 4 ++ 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index f0237f9f11a..4eee8d464dd 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -1,25 +1,21 @@ package org.rascalmpl.library.util; import java.io.IOException; -import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.charset.Charset; -import java.util.Map; import java.util.stream.Collectors; -import java.util.List; - import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.types.TypeReifier; import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.functions.IFunction; import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; -import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -37,25 +33,42 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store this.tf = tf; } + private HttpRequest makeGetRequest(IConstructor input) { + return HttpRequest getRequest = HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .GET() + .build(); + } + + private HttpRequest makePostRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + IFunction postBody = (IFunction) input.get("body"); + IConstructor rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .POST(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) + .build(); + } + + private HttpRequest makeRequest(IConstructor input) { + switch (input.getName()) { + case "get": + return makeGetRequest(input); + case "post": + return makePostRequest(input); + default: + throw RuntimeExceptionFactory.illegalArgument(input); + } + } public IConstructor fetch(IConstructor input) { try { - var params = input.asWithKeywordParameters(); + var request = makeRequest(input); + var response = HttpClient + .newHttpClient() + .send(request, HttpResponse.BodyHandlers.ofString()); - switch (input.getName()) { - case "get": - HttpRequest request = HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) - .GET() - .build(); - - HttpResponse response = - HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - - return translateTextResponse(response); - default: - // not yet implemented - throw RuntimeExceptionFactory.illegalArgument(input); - } + return translateTextResponse(response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); @@ -98,7 +111,7 @@ IConstructor toStatusConstructor(int stCode) { case ACCEPTED: return vf.constructor(store.lookupConstructor(statusType, "accepted", tf.tupleEmpty())); case BAD_REQUEST: - return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())) + return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())); case CONFLICT: return vf.constructor(store.lookupConstructor(statusType, "conflict", tf.tupleEmpty())); case CREATED: @@ -156,7 +169,8 @@ IConstructor toStatusConstructor(int stCode) { case UNSUPPORTED_MEDIA_TYPE: return vf.constructor(store.lookupConstructor(statusType, "unsupportedMediaType", tf.tupleEmpty())); default: - break; + // if we don't understand the error code; let's call it an internal error + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); } } } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index edd3d66751a..ad01b4a9db6 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -42,3 +42,7 @@ data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); + +@synopsis{Short-hand for a string post} +Request post(str path, str body, loc uri = |http://www.example.com|) + = post(uri = uri, path, str (type[str] _) { return body; }); From 61cadea36cf11bb268c8253934dcffa51dc22c8f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 17:19:42 +0200 Subject: [PATCH 03/11] added the other methods --- src/org/rascalmpl/library/util/Webclient.java | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 4eee8d464dd..0207cc56f2c 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.http.HttpClient; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; @@ -34,16 +35,45 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store } private HttpRequest makeGetRequest(IConstructor input) { - return HttpRequest getRequest = HttpRequest.newBuilder() + var params = input.asWithKeywordParameters(); + return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) .GET() .build(); } + private HttpRequest makePutRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + var postBody = (IFunction) input.get("body"); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) + .build(); + } + + private HttpRequest makeDeleteRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .DELETE() + .build(); + } + + private HttpRequest makeHeadRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .method("HEAD", BodyPublishers.noBody()) + .build(); + } + private HttpRequest makePostRequest(IConstructor input) { var params = input.asWithKeywordParameters(); - IFunction postBody = (IFunction) input.get("body"); - IConstructor rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + var postBody = (IFunction) input.get("body"); + var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) @@ -56,11 +86,22 @@ private HttpRequest makeRequest(IConstructor input) { case "get": return makeGetRequest(input); case "post": - return makePostRequest(input); + return makePostRequest(input); + case "put": + return makePutRequest(input); + case "delete": + return makeDeleteRequest(input); + case "head": + return makeHeadRequest(input); + default: throw RuntimeExceptionFactory.illegalArgument(input); } } + + /** + * This is the main API method for the Rascal side + */ public IConstructor fetch(IConstructor input) { try { var request = makeRequest(input); @@ -76,7 +117,7 @@ public IConstructor fetch(IConstructor input) { } private IConstructor translateTextResponse(HttpResponse response) { - IMap headers = response + var headers = response .headers() .map() .entrySet() @@ -87,22 +128,22 @@ private IConstructor translateTextResponse(HttpResponse response) { ))) .collect(vf.mapWriter()); - IString body = vf.string(response.body()); - IConstructor status = toStatusConstructor(response.statusCode()); + var body = vf.string(response.body()); + var status = toStatusConstructor(response.statusCode()); - Type respCons = store.lookupConstructors("response").iterator().next(); + var respCons = store.lookupConstructors("response").iterator().next(); var contentType = response.headers().firstValue("Content-Type"); - IString mimeType = vf.string(contentType.get().split(";")[0]); + var mimeType = vf.string(contentType.get().split(";")[0]); return vf.constructor(respCons, status, mimeType, headers, body); } - IConstructor toStatusConstructor(int stCode) { - Type statusType = store.lookupAbstractDataType("Status"); + private IConstructor toStatusConstructor(int stCode) { + var statusType = store.lookupAbstractDataType("Status"); - Status status = Status.lookup(stCode); + var status = Status.lookup(stCode); switch (status) { case OK: return vf.constructor(store.lookupConstructor(statusType, "ok", tf.tupleEmpty())); From 6cce6de693e5ddd0d1259b40cf0222ddf38ea245 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 20:33:55 +0200 Subject: [PATCH 04/11] added progress bar --- src/org/rascalmpl/library/util/Webclient.java | 98 ++++++++++++++++--- src/org/rascalmpl/library/util/Webclient.rsc | 6 +- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 0207cc56f2c..a3ddc50808f 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -1,6 +1,8 @@ package org.rascalmpl.library.util; +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; @@ -8,16 +10,16 @@ import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.library.Prelude; import org.rascalmpl.types.TypeReifier; +import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; -import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -37,7 +39,9 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store private HttpRequest makeGetRequest(IConstructor input) { var params = input.asWithKeywordParameters(); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(URIUtil.getChildLocation( + (ISourceLocation) params.getParameter("uri"), + ((IString) input.get("path")).getValue()).getURI()) .GET() .build(); } @@ -45,6 +49,7 @@ private HttpRequest makeGetRequest(IConstructor input) { private HttpRequest makePutRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var postBody = (IFunction) input.get("body"); + var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) @@ -106,17 +111,19 @@ public IConstructor fetch(IConstructor input) { try { var request = makeRequest(input); var response = HttpClient - .newHttpClient() - .send(request, HttpResponse.BodyHandlers.ofString()); + .newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateTextResponse(response); + return translateTextResponse(request.uri().toString(), response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - private IConstructor translateTextResponse(HttpResponse response) { + private IConstructor translateTextResponse(String url, HttpResponse response) throws IOException { var headers = response .headers() .map() @@ -128,15 +135,25 @@ private IConstructor translateTextResponse(HttpResponse response) { ))) .collect(vf.mapWriter()); - var body = vf.string(response.body()); - var status = toStatusConstructor(response.statusCode()); + long totalBytes = response.headers() + .firstValueAsLong("Content-Length") + .orElse(-1); - var respCons = store.lookupConstructors("response").iterator().next(); + var input = totalBytes > 0 + ? new MonitoredInputStream(response.body(), monitor, "Fetching " + url, totalBytes) + : response.body(); var contentType = response.headers().firstValue("Content-Type"); var mimeType = vf.string(contentType.get().split(";")[0]); - + + // TODO: extract from contentType if present + var charset = "utf-8"; + + var body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + var status = toStatusConstructor(response.statusCode()); + var respCons = store.lookupConstructors("response").iterator().next(); + return vf.constructor(respCons, status, mimeType, headers, body); } @@ -214,4 +231,63 @@ private IConstructor toStatusConstructor(int stCode) { return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); } } + + private class MonitoredInputStream extends FilterInputStream { + private final IRascalMonitor monitor; + private final String jobName; + + private final long totalBytes; + private long bytesRead = 0; + private boolean started = false; + private boolean done = false; + + public MonitoredInputStream(InputStream in, IRascalMonitor monitor, String jobName, long totalBytes) { + super(in); + this.totalBytes = totalBytes; + this.monitor = monitor; + this.jobName = jobName; + } + + private void ensureStarted() { + if (!started) { + started = true; + monitor.jobStart(jobName, Integer.MAX_VALUE); + } + } + + private void updateProgress(int bytesRead) { + ensureStarted(); + long numberOfTheseSteps = (int) (totalBytes / bytesRead); + int stepSize = (int) (Integer.MAX_VALUE / numberOfTheseSteps); + monitor.jobStep(jobName, "", java.lang.Math.max(stepSize, 1)); + checkDone(); + } + + private void checkDone() { + if (!done && bytesRead >= totalBytes) { + done = true; + monitor.jobEnd(jobName, true); + } + } + + @Override + public int read() throws IOException { + int b = super.read(); + if (b != -1) { + bytesRead += 1; + updateProgress(1); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = super.read(b, off, len); + if (n > 0) { + bytesRead += n; + updateProgress(n); + } + return n; + } + } } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index ad01b4a9db6..dbc31fb37f0 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -43,6 +43,6 @@ data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); -@synopsis{Short-hand for a string post} -Request post(str path, str body, loc uri = |http://www.example.com|) - = post(uri = uri, path, str (type[str] _) { return body; }); +// @synopsis{Short-hand for a string post} +// Request post(str path, str body, loc uri = |http://www.example.com|) +// = post(uri = uri, path, str (type[str] _) { return body; }); From 17723ffaf6c7d7821763570530f580c6b6624595 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 12:25:56 +0200 Subject: [PATCH 05/11] fixed post --- src/org/rascalmpl/library/Content.rsc | 1 - src/org/rascalmpl/library/util/Webclient.java | 110 ++++++++++++++---- src/org/rascalmpl/library/util/Webclient.rsc | 15 ++- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index f7615a63368..765dce04345 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -71,7 +71,6 @@ data Request (map[str, str] headers = (), map[str, str] parameters = (), map[str | head(str path) ; - @synopsis{A response encodes what is send back from the server to the browser client.} @description{ The three kinds of responses, encode either content that is already a `str`, diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index a3ddc50808f..71df5371c5c 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -3,23 +3,38 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.io.StringWriter; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.util.stream.Collectors; +import java.io.Writer; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.library.Prelude; +import org.rascalmpl.library.lang.json.internal.JsonValueReader; +import org.rascalmpl.library.lang.json.internal.JsonValueWriter; import org.rascalmpl.types.TypeReifier; +import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; +import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -28,20 +43,24 @@ public class Webclient { private final IRascalMonitor monitor; private final TypeStore store; private final TypeFactory tf; + private final TypeReifier tr; public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { this.vf = vf; this.monitor = monitor; this.store = store; this.tf = tf; + this.tr = new TypeReifier(vf); } private HttpRequest makeGetRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); + return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation( - (ISourceLocation) params.getParameter("uri"), - ((IString) input.get("path")).getValue()).getURI()) + .uri(URIUtil.getChildLocation(host, path).getURI()) .GET() .build(); } @@ -50,40 +69,67 @@ private HttpRequest makePutRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var postBody = (IFunction) input.get("body"); var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) .build(); } private HttpRequest makeDeleteRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .DELETE() .build(); } private HttpRequest makeHeadRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .method("HEAD", BodyPublishers.noBody()) .build(); } private HttpRequest makePostRequest(IConstructor input) { var params = input.asWithKeywordParameters(); - var postBody = (IFunction) input.get("body"); - var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); - - return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) - .POST(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) - .build(); + var postBody = (IFunction) input.get("content"); + var rt = new TypeReifier(vf).typeToValue(tf.valueType(), store, vf.map()); + var host = ((ISourceLocation) params.getParameter("host")); + var val = postBody.call(rt); + var path = ((IString) input.get("path")).getValue(); + + try { + PipedOutputStream out = new PipedOutputStream(); + PipedInputStream in = new PipedInputStream(out, 64 * 1024); + + Thread writer = new Thread(() -> { + try (OutputStream os = out; Writer w = new OutputStreamWriter(out)) { + JsonWriter jsonWriter = new JsonWriter(w); + JsonValueWriter jsonOut = new JsonValueWriter(); + jsonOut.write(jsonWriter, val); + } + catch (Exception e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } + }); + + writer.start(); + + return HttpRequest.newBuilder() + .uri(URIUtil.getChildLocation(host, path).getURI()) + .POST(HttpRequest.BodyPublishers.ofInputStream(() -> in)) + .build(); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } } private HttpRequest makeRequest(IConstructor input) { @@ -116,14 +162,14 @@ public IConstructor fetch(IConstructor input) { .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateTextResponse(request.uri().toString(), response); + return translateResponse(request.uri().toString(), (IConstructor) input.asWithKeywordParameters().getParameter("body"), response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - private IConstructor translateTextResponse(String url, HttpResponse response) throws IOException { + private IConstructor translateResponse(String url, IConstructor expect, HttpResponse response) throws IOException { var headers = response .headers() .map() @@ -149,12 +195,32 @@ private IConstructor translateTextResponse(String url, HttpResponse // TODO: extract from contentType if present var charset = "utf-8"; - - var body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + var status = toStatusConstructor(response.statusCode()); - var respCons = store.lookupConstructors("response").iterator().next(); - return vf.constructor(respCons, status, mimeType, headers, body); + Type respCons; + IString body; + + switch (expect != null ? expect.getName() : "textBody") { + case "jsonBody": + JsonReader jsonReader = new JsonReader(new InputStreamReader(input)); + JsonValueReader parser = new JsonValueReader(vf, store, monitor, URIUtil.assumeCorrectLocation(url)); + respCons = store.lookupConstructors("jsonResponse").iterator().next(); + var value = parser.read(jsonReader, tr.valueToType((IConstructor) expect.get("expected"))); + return vf.constructor(respCons, status, headers, value); + case "fileBody": + respCons = store.lookupConstructors("fileResponse").iterator().next(); + var loc = (ISourceLocation) expect.get("storage"); + try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(loc, false)) { + input.transferTo(out); + } + return vf.constructor(respCons, loc, mimeType, headers); + case "textBody": + default: + respCons = store.lookupConstructors("response").iterator().next(); + body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + return vf.constructor(respCons, status, mimeType, headers, body); + } } private IConstructor toStatusConstructor(int stCode) { @@ -255,7 +321,11 @@ private void ensureStarted() { } } - private void updateProgress(int bytesRead) { + private void updateProgress(int bytesRead) throws InterruptedIOException { + if (monitor.jobIsCanceled(jobName)) { + throw new InterruptedIOException(jobName); + } + ensureStarted(); long numberOfTheseSteps = (int) (totalBytes / bytesRead); int stepSize = (int) (Integer.MAX_VALUE / numberOfTheseSteps); diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index dbc31fb37f0..2f0bbc2bc8b 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -2,6 +2,12 @@ module util::Webclient extend Content; +data BodyExpectation + = textBody() + | jsonBody(type[value] expected) + | fileBody(loc storage) + ; + @synopsis{Extends ((Content-Status)) with everything HTTP has out there in the wild.} data Status = ok() @@ -38,7 +44,14 @@ data Status | unsupportedMediaType() ; -data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain"); +data Request( + loc host = |http://www.example.com|, + str \content-type = "text/plain", + BodyExpectation body = text() + ); + +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPost(str path, &T content, loc host=|http://www.example.com|) = post(path, &T (type[&T] _expected) { return content; }, host=host); @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); From b9c1e6ea7b5ac93ed1ef7be158630501c7b2d053 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 12:33:40 +0200 Subject: [PATCH 06/11] constructor typo --- src/org/rascalmpl/library/util/Webclient.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index 2f0bbc2bc8b..8b66830cc30 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -47,7 +47,7 @@ data Status data Request( loc host = |http://www.example.com|, str \content-type = "text/plain", - BodyExpectation body = text() + BodyExpectation body = textBody() ); @synopsis{Short-hand for construction of JSON post bodies} From 8c9c0dafa5b1103bea49457066f022f18e2ec112 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 16:25:36 +0200 Subject: [PATCH 07/11] fix post bug --- src/org/rascalmpl/library/util/Webclient.rsc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index 8b66830cc30..42066546f3b 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -51,7 +51,8 @@ data Request( ); @synopsis{Short-hand for construction of JSON post bodies} -Request jsonPost(str path, &T content, loc host=|http://www.example.com|) = post(path, &T (type[&T] _expected) { return content; }, host=host); +Request jsonPost(str path, &T content, loc host=|http://www.example.com|, BodyExpectation body = textBody()) + = post(path, &T (type[&T] _expected) { return content; }, host=host, body=body); @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); From ddf4bfa91c806aa6f1aa6f97b88680ce0aa97b79 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 16:28:01 +0200 Subject: [PATCH 08/11] added path to other requests kinds but GET and POST --- src/org/rascalmpl/library/util/Webclient.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 71df5371c5c..9a9a4c4ba94 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -70,9 +70,11 @@ private HttpRequest makePutRequest(IConstructor input) { var postBody = (IFunction) input.get("body"); var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) .build(); } @@ -80,9 +82,11 @@ private HttpRequest makePutRequest(IConstructor input) { private HttpRequest makeDeleteRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .DELETE() .build(); } @@ -90,9 +94,11 @@ private HttpRequest makeDeleteRequest(IConstructor input) { private HttpRequest makeHeadRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .method("HEAD", BodyPublishers.noBody()) .build(); } @@ -161,7 +167,6 @@ public IConstructor fetch(IConstructor input) { .followRedirects(HttpClient.Redirect.NORMAL) .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateResponse(request.uri().toString(), (IConstructor) input.asWithKeywordParameters().getParameter("body"), response); } catch (IOException | InterruptedException e) { From 53ae8a0ca99723437a2f844da15cc3b5d938fcdd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 19 May 2026 15:38:32 +0200 Subject: [PATCH 09/11] started rewrite of Server and Client interface to canonically treat a body in a POST and PUT ina Request, the same way as on the Response side. So support for text, json and file content. HTML is for later. --- src/org/rascalmpl/library/Content.rsc | 61 ++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index 765dce04345..c5c51dcf404 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -3,6 +3,7 @@ module Content import lang::json::IO; +import IO; @synopsis{Content wraps the HTTP Request/Response API to support interactive visualization types on the terminal.} @@ -51,9 +52,6 @@ Content file(loc src) = content(response(src)); @synopsis{Directly serve the contents of a string as plain text} Content plainText(str text) = content(plain(text)); -alias Body = value (type[value] expected); - - @synopsis{Request values represent what a browser is asking for, most importantly the URL path.} @description{ A request value also contains the full HTTP headers, the URL parameters as a `map[str,str]` @@ -65,27 +63,68 @@ and possibly uploaded content, also coded as a map[str,str]. From the constructo } data Request (map[str, str] headers = (), map[str, str] parameters = (), map[str,str] uploads = ()) = get (str path) - | put (str path, Body content) - | post(str path, Body content) + | put (str path, BodyProvider content) + | post(str path, BodyProvider content) | delete(str path) | head(str path) ; +@synopsis{Special (non-HTTP standard) options to create POST and Response body internals.} +@description{ +This interface bridges Rascal data to the HTTP protocol. Typically large input +such as (composite) strings and JSON code is _streamed_ onto the HTTP socket. +} +data BodyProvider + = fromText(str content) + | fromJSON(value object, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, + bool explicitConstructorNames=false, bool explicitDataTypes=false, bool dateTimeAsInt=true, bool rationalsAsString=false) + | fromFile(loc source) + ; + +@synsopsis{Special (non-HTTP standard) to consume POST and Response bodies.} +@description{ +This interface bridges data on HTTP sockets back into Rascal. +} +data BodyConsumer[&T] + = toText(str () reader) + | toJSON(type[&T] expect, &T () consumer) + | toFile(loc target) + ; + @synopsis{A response encodes what is send back from the server to the browser client.} @description{ The three kinds of responses, encode either content that is already a `str`, some file which is streamed directly from its source location or a jsonResponse which involves a handy, automatic, encoding of Rascal values into json values. } -data Response - = response(Status status, str mimeType, map[str,str] header, str content) - | fileResponse(loc file, str mimeType, map[str,str] header) - | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, +data Response = response(Status, str mimeType, map[str, str] header, BodyProvider body); + +@synopsis{Convenience function for construction a JSON response value} +Response jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, bool explicitConstructorNames=false, bool explicitDataTypes=false, bool dateTimeAsInt=true, bool rationalsAsString=false) + = response(status, "application/json", header, fromJSON(val, dateTimeFormat=dateTimeFormat, formatter=formatter, explicitConstructorNames=explicitConstructorNames, + explicitDataTypes=explicitDataTypes, dateTimeAsInt=dateTimeAsInt, rationalsAsString=rationalsAsString)); + +@synopsis{Convenience function for construction a text response value} +Response response(Status status, str mimeType, map[str,str] header, str content) + = response(status, mimeType, header, fromText(content)); + +@synopsis{Convenience function for file response value} +Response fileResponse(loc file, str mimeType, map[str,str] header) + = exists(file) + ? response(ok(), mimeType, header, fromFile(file)) + : response(notFound(), "text/plain", (), fromText(" not found.")) ; - + +@synopsis{Convenience function for construction a file response value with automatic mimetype} +Response fileResponse(loc file, map[str,str] header) + = exists(file) + ? response(ok(), mimeTypes[file.extension]?"text/plain", header, fromFile(file)) + : response(notFound(), "text/plain", (), fromText(" not found.")) + ; + @synopsis{Utility to quickly render a string as HTML content} -Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, content); +Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, fromText(content)); @synopsis{Utility to quickly report an HTTP error with a user-defined message} Response response(Status status, str explanation, map[str,str] header = ()) = response(status, "text/plain", header, explanation); From cd8bdda8c812dcc1b81856095827f0c358515926 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 May 2026 15:45:23 +0200 Subject: [PATCH 10/11] big cleanup of Webclient, but Webserver is broken now and I still have a deadlock --- src/org/rascalmpl/library/Content.rsc | 201 ++++---- src/org/rascalmpl/library/util/Webclient.java | 463 ++++++++++++++---- src/org/rascalmpl/library/util/Webclient.rsc | 78 +-- .../repl/rascal/RascalReplServices.java | 1 - .../semantics/dynamic/Statement.java | 1 - 5 files changed, 499 insertions(+), 245 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index c5c51dcf404..8ca87cef951 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -40,6 +40,36 @@ data Content | content(Response response, str title="*static content*", ViewColumn viewColumn = normalViewColumn(1)) ; +@synopsis{A static map with default MIME interpretations for particular file extensions.} +public map[str extension, str mimeType] mimeTypes = ( + "json" :"application/json", + "css" : "text/css", + "htm" : "text/html", + "html" : "text/html", + "xml" : "text/xml", + "java" : "text/x-java-source, text/java", + "txt" : "text/plain", + "asc" : "text/plain", + "ico" : "image/x-icon", + "gif" : "image/gif", + "jpg" : "image/jpeg", + "jpeg" : "image/jpeg", + "png" : "image/png", + "mp3" : "audio/mpeg", + "m3u" : "audio/mpeg-url", + "mp4" : "video/mp4", + "ogv" : "video/ogg", + "flv" : "video/x-flv", + "mov" : "video/quicktime", + "swf" : "application/x-shockwave-flash", + "js" : "application/javascript", + "pdf" : "application/pdf", + "doc" : "application/msword", + "ogg" : "application/x-ogg", + "zip" : "application/octet-stream", + "exe" : "application/octet-stream", + "class" : "application/octet-stream" + ); @synopsis{Directly serve a static html page} Content html(str html) = content(response(html)); @@ -63,68 +93,83 @@ and possibly uploaded content, also coded as a map[str,str]. From the constructo } data Request (map[str, str] headers = (), map[str, str] parameters = (), map[str,str] uploads = ()) = get (str path) - | put (str path, BodyProvider content) - | post(str path, BodyProvider content) + | put (str path, Body content) + | post(str path, Body content) | delete(str path) | head(str path) ; -@synopsis{Special (non-HTTP standard) options to create POST and Response body internals.} +@synopsis{A response encodes what is send back from the server to the browser client.} @description{ -This interface bridges Rascal data to the HTTP protocol. Typically large input -such as (composite) strings and JSON code is _streamed_ onto the HTTP socket. +The three kinds of responses, encode either content that is already a `str`, +some file which is streamed directly from its source location or a jsonResponse +which involves a handy, automatic, encoding of Rascal values into json values. +} +data Response = response(Status, str mimeType, map[str, str] header, Body body); + +@synopsis{Bodies can be sent or received, depending on the context (client or)} +@description{ +* put and post requests, when received by a server, receive bodies. +* put and post requests, when fetched by a client, sent bodies. +* a response y a server sends a body. +* a response that was fetched by a client receives a body. + +The ((BodyKind)) encodes what we expect from the sender when it +puts the value onto the socket, and what we expect from the receiver +when it reads the contents off the socket. This is where builtin +conversions (formatters, parsers and validators) are activated on +the bridge between Rascal and the HTTP protocol. } -data BodyProvider - = fromText(str content) - | fromJSON(value object, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, - bool explicitConstructorNames=false, bool explicitDataTypes=false, bool dateTimeAsInt=true, bool rationalsAsString=false) - | fromFile(loc source) +data Body + = send(BodyKind kind, value source) + | receive(&T (BodyKind kind, type[&T] expect) receiver) ; -@synsopsis{Special (non-HTTP standard) to consume POST and Response bodies.} +@synopsis{The type's of ((Body)) that we are sending or expecting to receive} @description{ -This interface bridges data on HTTP sockets back into Rascal. +This interface bridges Rascal data to the HTTP protocol. Typically large input +such as (composite) strings and JSON code is _streamed_ onto the HTTP socket. } -data BodyConsumer[&T] - = toText(str () reader) - | toJSON(type[&T] expect, &T () consumer) - | toFile(loc target) +data BodyKind + = text() + | json(JSONOptions options=jsonOptions()) + | file(loc storage=|unknown:///|) ; -@synopsis{A response encodes what is send back from the server to the browser client.} -@description{ -The three kinds of responses, encode either content that is already a `str`, -some file which is streamed directly from its source location or a jsonResponse -which involves a handy, automatic, encoding of Rascal values into json values. -} -data Response = response(Status, str mimeType, map[str, str] header, BodyProvider body); +data JSONOptions + = jsonOptions( + str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", + JSONFormatter[value] formatter = str (value _) { fail; }, + bool explicitConstructorNames=false, + bool explicitDataTypes=false, + bool dateTimeAsInt=true, + bool rationalsAsString=false + ); @synopsis{Convenience function for construction a JSON response value} -Response jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, - bool explicitConstructorNames=false, bool explicitDataTypes=false, bool dateTimeAsInt=true, bool rationalsAsString=false) - = response(status, "application/json", header, fromJSON(val, dateTimeFormat=dateTimeFormat, formatter=formatter, explicitConstructorNames=explicitConstructorNames, - explicitDataTypes=explicitDataTypes, dateTimeAsInt=dateTimeAsInt, rationalsAsString=rationalsAsString)); +Response jsonResponse(Status status, map[str,str] header, value val, JSONOptions options = jsonOptions()) + = response(status, "application/json", header, send(json(options=options), val)); @synopsis{Convenience function for construction a text response value} Response response(Status status, str mimeType, map[str,str] header, str content) - = response(status, mimeType, header, fromText(content)); + = response(status, mimeType, header, send(text(), content)); @synopsis{Convenience function for file response value} -Response fileResponse(loc file, str mimeType, map[str,str] header) - = exists(file) - ? response(ok(), mimeType, header, fromFile(file)) - : response(notFound(), "text/plain", (), fromText(" not found.")) +Response fileResponse(loc source, str mimeType, map[str,str] header) + = exists(source) + ? response(ok(), mimeType, header, send(file(), source)) + : response(notFound(), "text/plain", (), send(text(), " not found.")) ; @synopsis{Convenience function for construction a file response value with automatic mimetype} -Response fileResponse(loc file, map[str,str] header) - = exists(file) - ? response(ok(), mimeTypes[file.extension]?"text/plain", header, fromFile(file)) - : response(notFound(), "text/plain", (), fromText(" not found.")) +Response fileResponse(loc source, map[str,str] header) + = exists(source) + ? response(ok(), mimeTypes[source.extension]?"text/plain", header, send(file(), source)) + : response(notFound(), "text/plain", (), send(text(), " not found.")) ; @synopsis{Utility to quickly render a string as HTML content} -Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, fromText(content)); +Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, send(text(), content)); @synopsis{Utility to quickly report an HTTP error with a user-defined message} Response response(Status status, str explanation, map[str,str] header = ()) = response(status, "text/plain", header, explanation); @@ -146,57 +191,45 @@ default Response response(value val, map[str,str] header = ()) = jsonResponse(o @benefits{ Fast way of producing JSON strings for embedded DSLs on the Rascal side. } -Response response(value val, JSONFormatter[value] formatter, map[str,str] header = ()) = jsonResponse(ok(), header, val, formatter=formatter); +Response response(value val, JSONFormatter[value] formatter, map[str,str] header = ()) = jsonResponse(ok(), header, val, options=jsonOptions(formatter=formatter)); @synopsis{Encoding of HTTP status} -data Status - = ok() - | created() - | accepted() - | noContent() - | partialContent() - | redirect() - | notModified() - | badRequest() - | unauthorized() - | forbidden() - | notFound() - | rangeNotSatisfiable() - | internalError() - ; +data Status + = ok() + | notFound() + | accepted() + | badRequest() + | conflict() + | created() + | expectationFailed() + | forbidden() + | found() + | gone() + | internalError() + | lengthRequired() + | methodNotAllowed() + | multiStatus() + | notAcceptible() + | notImplemented() + | notModified() + | noContent() + | partialContent() + | payloadTooLarge() + | preconditionFailed() + | rangeNotSatisfiable() + | redirect() + | redirectSeeOther() + | requestTimeout() + | serviceUnavailable() + | switchProtocol() + | temporaryRedirect() + | tooManyRequests() + | unauthorized() + | unsupportedHTTPVersion() + | unsupportedMediaType() + ; -@synopsis{A static map with default MIME interpretations for particular file extensions.} -public map[str extension, str mimeType] mimeTypes = ( - "json" :"application/json", - "css" : "text/css", - "htm" : "text/html", - "html" : "text/html", - "xml" : "text/xml", - "java" : "text/x-java-source, text/java", - "txt" : "text/plain", - "asc" : "text/plain", - "ico" : "image/x-icon", - "gif" : "image/gif", - "jpg" : "image/jpeg", - "jpeg" : "image/jpeg", - "png" : "image/png", - "mp3" : "audio/mpeg", - "m3u" : "audio/mpeg-url", - "mp4" : "video/mp4", - "ogv" : "video/ogg", - "flv" : "video/x-flv", - "mov" : "video/quicktime", - "swf" : "application/x-shockwave-flash", - "js" : "application/javascript", - "pdf" : "application/pdf", - "doc" : "application/msword", - "ogg" : "application/x-ogg", - "zip" : "application/octet-stream", - "exe" : "application/octet-stream", - "class" : "application/octet-stream" - ); - @synopsis{Hint the IDE where to open the next web view or editor} @description{ The `viewColumn` decides where in the IDE a web client or editor is opened, diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 9a9a4c4ba94..f0be0c97232 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -1,5 +1,6 @@ package org.rascalmpl.library.util; +import java.io.BufferedOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -7,21 +8,29 @@ import java.io.InterruptedIOException; import java.io.OutputStream; import java.io.OutputStreamWriter; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.io.StringWriter; +import java.net.URI; import java.net.http.HttpClient; +import java.net.http.HttpHeaders; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublisher; import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Flow.Subscriber; import java.util.stream.Collectors; -import java.io.Writer; +import java.util.stream.Stream; +import org.apache.commons.io.input.ReaderInputStream; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.library.Prelude; import org.rascalmpl.library.lang.json.internal.JsonValueReader; import org.rascalmpl.library.lang.json.internal.JsonValueWriter; -import org.rascalmpl.types.TypeReifier; +import org.rascalmpl.types.RascalTypeFactory; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; @@ -31,9 +40,14 @@ import com.google.gson.stream.JsonWriter; import fi.iki.elonen.NanoHTTPD.Response.Status; +import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; +import io.usethesource.vallang.ITuple; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.IWithKeywordParameters; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -43,113 +57,275 @@ public class Webclient { private final IRascalMonitor monitor; private final TypeStore store; private final TypeFactory tf; - private final TypeReifier tr; + private final ExecutorService executor; + private final HttpClient.Builder client; public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { this.vf = vf; this.monitor = monitor; this.store = store; this.tf = tf; - this.tr = new TypeReifier(vf); + this.executor = Executors.newCachedThreadPool(); + this.client = HttpClient.newBuilder(); } - private HttpRequest makeGetRequest(IConstructor input) { - var params = input.asWithKeywordParameters(); - var host = ((ISourceLocation) params.getParameter("host")); - host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; - var path = ((IString) input.get("path")).getValue(); + private String[] makeHeaders(IMap headers) { + return headers + .stream() + .map(ITuple.class::cast) + .flatMap((ITuple t) -> Stream.of( + ((IString) t.get(0)).getValue(), + ((IString) t.get(1)).getValue() + )) + .toArray(String[]::new) + ; + } - return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation(host, path).getURI()) + private HttpRequest makeGetRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) .GET() .build(); } - private HttpRequest makePutRequest(IConstructor input) { - var params = input.asWithKeywordParameters(); - var postBody = (IFunction) input.get("body"); - var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); - var host = ((ISourceLocation) params.getParameter("host")); - host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; - var path = ((IString) input.get("path")).getValue(); - - return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation(host, path).getURI()) - .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) + private HttpRequest makePutRequest(IConstructor input, URI uri, String[] headers, String charset) { + return HttpRequest.newBuilder(uri) + .headers(headers) + .PUT(publishBody(input, charset)) .build(); } - private HttpRequest makeDeleteRequest(IConstructor input) { - var params = input.asWithKeywordParameters(); - var host = ((ISourceLocation) params.getParameter("host")); - host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; - var path = ((IString) input.get("path")).getValue(); - - return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation(host, path).getURI()) + private HttpRequest makeDeleteRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) .DELETE() .build(); } - private HttpRequest makeHeadRequest(IConstructor input) { - var params = input.asWithKeywordParameters(); - var host = ((ISourceLocation) params.getParameter("host")); - host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; - var path = ((IString) input.get("path")).getValue(); + private URI requireHost(IConstructor input) { + ISourceLocation host = (ISourceLocation) input.asWithKeywordParameters().getParameter("host"); + IString path = (IString) input.get("path"); - return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation(host, path).getURI()) + if (host == null) { + throw RuntimeExceptionFactory.illegalArgument(input, "missing `host` field"); + } + + if (!host.getPath().equals("/") && !host.getPath().equals("")) { + throw RuntimeExceptionFactory.illegalArgument(host, "path after hostname should be given with the \'path\' parameter of a request"); + } + + // TODO: query, fragment + return URIUtil.getChildLocation(host, path.getValue()).getURI(); + } + + private HttpRequest makeHeadRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) .method("HEAD", BodyPublishers.noBody()) .build(); } - private HttpRequest makePostRequest(IConstructor input) { - var params = input.asWithKeywordParameters(); - var postBody = (IFunction) input.get("content"); - var rt = new TypeReifier(vf).typeToValue(tf.valueType(), store, vf.map()); - var host = ((ISourceLocation) params.getParameter("host")); - var val = postBody.call(rt); - var path = ((IString) input.get("path")).getValue(); - - try { - PipedOutputStream out = new PipedOutputStream(); - PipedInputStream in = new PipedInputStream(out, 64 * 1024); - - Thread writer = new Thread(() -> { - try (OutputStream os = out; Writer w = new OutputStreamWriter(out)) { - JsonWriter jsonWriter = new JsonWriter(w); - JsonValueWriter jsonOut = new JsonValueWriter(); - jsonOut.write(jsonWriter, val); - } - catch (Exception e) { - throw RuntimeExceptionFactory.io(e.getMessage()); - } - }); - - writer.start(); + private OutputStreamBodySupplier publishJsonBody(IConstructor input, String charset) { + // make a stream to write to that can also be pulled from by the HTTP client API. + OutputStreamBodySupplier supplier = new OutputStreamBodySupplier(); + + // start the asynchronous task of writing the JSON string to the outputstream + // TODO: how do make sure this job starts and really runs? + executor.submit(() -> { + IWithKeywordParameters kws = input.asWithKeywordParameters().getParameter("options").asWithKeywordParameters(); + IString dtf = kws.getParameter("dateTimeFormat"); + IBool dai = kws.getParameter("dateTimeAsInt"); + IBool ras = kws.getParameter("rationalsAsString"); + IFunction formatters = kws.getParameter("formatter"); + IBool ecn = kws.getParameter("explicitConstructorNames"); + IBool edt = kws.getParameter("explicitDataTypes"); + + JsonValueWriter writer = new JsonValueWriter() + .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters(formatters) + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setRationalsAsString(ras != null ? ((IBool) ras).getValue() : false) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; - return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation(host, path).getURI()) - .POST(HttpRequest.BodyPublishers.ofInputStream(() -> in)) - .build(); + try (JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(supplier, charset))) { + writer.write(jsonWriter, input); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + }, true); + + return supplier; + } + + private BodyPublisher publishFileBody(IConstructor kind, IConstructor input) { + final var loc = (ISourceLocation) input.get("source"); + + // have to hope that the encoding matches the actual contents of the stream + + return BodyPublishers.ofInputStream(() -> { + try { + return URIResolverRegistry.getInstance().getInputStream(loc); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + }); + } + + private BodyPublisher publishTextBody(IConstructor input, String charset) { + final var text = (IString) input.get("source"); + return BodyPublishers.ofInputStream(() -> new ReaderInputStream(text.asReader(), charset)); + } + + /** + * This is for PUT and POST requests that need to send data along with the request + */ + private BodyPublisher publishBody(IConstructor input, String charset) { + var postBody = (IConstructor) input.get("content"); + + if (!postBody.getName().equals("send")) { + throw RuntimeExceptionFactory.illegalArgument(postBody, "Client-side POST should send a Body, not receive one"); } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e.getMessage()); + + final var kind = (IConstructor) postBody.get("kind"); + + switch (kind.getName()) { + case "json": + return publishJsonBody(postBody, charset); + case "file": + return publishFileBody(kind, postBody); + case "text": + return publishTextBody(postBody, charset); + default: + return null; + } + } + + /** + * On demand streamer for the HttpClient API (for sending bodies for PUT and POST) + */ + private static class OutputStreamBodySupplier extends BufferedOutputStream implements BodyPublisher { + private final List> subscribers; + + public OutputStreamBodySupplier() { + super(new PublishingStream()); + this.subscribers = ((PublishingStream)super.out).subscribers; + } + + /** + * The buffed outputstream will take care to collect the bytes untill there's a decent chunk to forward to the consumers + */ + private static class PublishingStream extends OutputStream { + private final List> subscribers = new CopyOnWriteArrayList<>(); + + @Override + public void write(int b) throws IOException { + for (var sub: subscribers) { + sub.onNext(ByteBuffer.wrap(new byte[] { (byte)(b & 0xFF) })); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + for (var sub: subscribers) { + sub.onNext(ByteBuffer.wrap(b, off, len).asReadOnlyBuffer()); + } + } + + @Override + public void close() throws IOException { + for (var sub: subscribers) { + sub.onComplete(); + } + } + } + + @Override + public void subscribe(Subscriber subscriber) { + this.subscribers.add(subscriber); + } + + @Override + public long contentLength() { + return -1; + } + } + + private HttpRequest makePostRequest(IConstructor input, URI uri, String[] headers, String charset) { + return HttpRequest.newBuilder() + .uri(uri) + .headers(headers) + .POST(publishBody(input, charset)) + .build(); + } + + private String defaultContentType(IConstructor input) { + if (input.getName().equals("post") || input.getName().equals("put")) { + var body = (IConstructor) input.get("content"); + var kind = (IConstructor) body.get("kind"); + + switch (kind.getName()) { + case "json": + return "application/json"; + case "file": + return "application/octet-stream"; + case "text": + default: + return "text/plain"; + } } + + // only post and put have content to send + return ""; } private HttpRequest makeRequest(IConstructor input) { + var host = requireHost(input); + var headers = (IMap) input.asWithKeywordParameters().getParameter("header"); + + if (headers == null) { + headers = vf.map(); + } + + var charset = ((IString) input.asWithKeywordParameters().getParameter("charset")); + + if (charset == null) { + charset = vf.string(StandardCharsets.UTF_8.name()); + } + + var contentType = ((IString) input.asWithKeywordParameters().getParameter("content-type")); + + if (contentType == null) { + contentType = vf.string(defaultContentType(input)); + } + + if (headers.get(vf.string("Content-Type")) != null) { + throw RuntimeExceptionFactory.illegalArgument(input, "For POST and PUT, use the keyword fields 'content-type' and 'charset' instead of the 'Content-Type' header field."); + } + + if (contentType.length() != 0) { + headers = headers.put(vf.string("Content-Type"), vf.string(contentType + ";charset=" + charset)); + } + + // need at least one header to avoid IllegalArgumentExceptions by the HTTP builder + headers = headers.put(vf.string("User-Agent"), vf.string("rascal-stdlib")); + + var httpHeaders = makeHeaders(headers); + switch (input.getName()) { case "get": - return makeGetRequest(input); + return makeGetRequest(input, host, httpHeaders); case "post": - return makePostRequest(input); + return makePostRequest(input, host, httpHeaders, charset.getValue()); case "put": - return makePutRequest(input); + return makePutRequest(input, host, httpHeaders, charset.getValue()); case "delete": - return makeDeleteRequest(input); + return makeDeleteRequest(input, host, httpHeaders); case "head": - return makeHeadRequest(input); + return makeHeadRequest(input, host, httpHeaders); default: throw RuntimeExceptionFactory.illegalArgument(input); @@ -162,8 +338,7 @@ private HttpRequest makeRequest(IConstructor input) { public IConstructor fetch(IConstructor input) { try { var request = makeRequest(input); - var response = HttpClient - .newBuilder() + var response = client .followRedirects(HttpClient.Redirect.NORMAL) .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); @@ -174,6 +349,82 @@ public IConstructor fetch(IConstructor input) { } } + /* + * This creates a Body::receive() constructor that wraps a lambda to call and receive the data + * from the HTTP response as a Rascal value. The intermediate step is required such that the + * caller of `fetch` can express their expectations on the data and call appropriate parsing, + * validation and binding mechanisms. The options are encoded as `BodyKind`. + * + * Also this generated function asks for a reified type parameter for (dynamic) type safety + * and validation of abstract grammars during reception of the body. + */ + private IFunction createBodyReceiver(InputStream input, String url, String contentType, String charset) { + var BodyKind = store.lookupAbstractDataType("BodyKind"); + var rt = RascalTypeFactory.getInstance().reifiedType(tf.parameterType("T")); + var ft = tf.functionType(tf.parameterType("T"), tf.tupleType(BodyKind, rt), tf.tupleEmpty()); + + return vf.function(ft, (args, kwargs) -> { + IConstructor kind = (IConstructor) args[0]; + IConstructor reified = (IConstructor) args[1]; + Type expect = reified.getType().getTypeParameters().getFieldType(0); + + switch (kind.getName()) { + case "json": + return receiveJsonBody(input, url, kind, expect, contentType, charset); + case "file": + return receiveFileBody(input, kind, expect); + case "text": + default: + return receiveTextBody(input, kind, expect, contentType, charset); + } + }); + } + + private IValue receiveTextBody(InputStream input, IConstructor kind, Type expect, String contentType, String charset) { + if (!expect.isSubtypeOf(tf.stringType())) { + throw RuntimeExceptionFactory.illegalArgument(kind, "a text response expects a `str` type"); + } + + if (!contentType.contains("text/")) { + throw RuntimeExceptionFactory.illegalArgument(kind, "a text response was expected but the mimetype is " + contentType); + } + + try { + return vf.string(Prelude.consumeInputStream(new InputStreamReader(input, charset))); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + + private IValue receiveFileBody(InputStream input, IConstructor kind, Type expect) { + ISourceLocation loc = (ISourceLocation) kind.get("storage"); + + // note that mimetype and charset are kept as-is during the download directly to disk + try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(loc, false)) { + input.transferTo(out); + return loc; + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + + private IValue receiveJsonBody(InputStream input, String url, IConstructor kind, Type expect, String contentType, String charset) { + if (!contentType.equals("application/json")) { + throw RuntimeExceptionFactory.illegalArgument(kind, "expected content-type 'application/json', got: " + contentType); + } + + try { + JsonReader jsonReader = new JsonReader(new InputStreamReader(input, charset)); + JsonValueReader parser = new JsonValueReader(vf, store, monitor, URIUtil.assumeCorrectLocation(url)); + return parser.read(jsonReader, expect); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + private IConstructor translateResponse(String url, IConstructor expect, HttpResponse response) throws IOException { var headers = response .headers() @@ -194,38 +445,38 @@ private IConstructor translateResponse(String url, IConstructor expect, HttpResp ? new MonitoredInputStream(response.body(), monitor, "Fetching " + url, totalBytes) : response.body(); - var contentType = response.headers().firstValue("Content-Type"); - - var mimeType = vf.string(contentType.get().split(";")[0]); - - // TODO: extract from contentType if present - var charset = "utf-8"; - + String mimeType = getMimeType(response.headers()); + String charset = getCharset(response.headers()); var status = toStatusConstructor(response.statusCode()); - Type respCons; - IString body; - - switch (expect != null ? expect.getName() : "textBody") { - case "jsonBody": - JsonReader jsonReader = new JsonReader(new InputStreamReader(input)); - JsonValueReader parser = new JsonValueReader(vf, store, monitor, URIUtil.assumeCorrectLocation(url)); - respCons = store.lookupConstructors("jsonResponse").iterator().next(); - var value = parser.read(jsonReader, tr.valueToType((IConstructor) expect.get("expected"))); - return vf.constructor(respCons, status, headers, value); - case "fileBody": - respCons = store.lookupConstructors("fileResponse").iterator().next(); - var loc = (ISourceLocation) expect.get("storage"); - try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(loc, false)) { - input.transferTo(out); - } - return vf.constructor(respCons, loc, mimeType, headers); - case "textBody": - default: - respCons = store.lookupConstructors("response").iterator().next(); - body = vf.string(new String(Prelude.consumeInputStream(input), charset)); - return vf.constructor(respCons, status, mimeType, headers, body); + Type respCons = store.lookupConstructors("response").iterator().next(); + IFunction bodyReceiver = createBodyReceiver(input, url, mimeType, charset); + Type bodyConstructor = store.lookupConstructors("receive").iterator().next(); + IConstructor body = vf.constructor(bodyConstructor, bodyReceiver); + + return vf.constructor(respCons, status, vf.string(mimeType), headers, body); + } + + private String getCharset(HttpHeaders headers) { + String contentType = headers.firstValue("Content-Type").orElse("text/plain"); + String result = StandardCharsets.UTF_8.name(); + + String[] parts = contentType.split(";"); + if (parts.length > 1) { + String[] assign = parts[1].split("="); + + if (assign[0].equals("charset")) { + result = assign[1].trim(); + } } + + return result; + } + + private String getMimeType(HttpHeaders headers) { + String contenType = headers.firstValue("Content-Type").orElse("text/plain"); + String[] parts = contenType.split(";"); + return parts[0].trim(); } private IConstructor toStatusConstructor(int stCode) { diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index 42066546f3b..192455c8694 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -1,62 +1,34 @@ module util::Webclient extend Content; +import Exception; -data BodyExpectation - = textBody() - | jsonBody(type[value] expected) - | fileBody(loc storage) - ; - -@synopsis{Extends ((Content-Status)) with everything HTTP has out there in the wild.} -data Status - = ok() - | notFound() - | accepted() - | badRequest() - | conflict() - | create() - | expectationFailed() - | forbidden() - | found() - | gone() - | internalError() - | lengthRequired() - | methodNotAllowed() - | multiStatus() - | notAcceptible() - | notImplemented() - | notModified() - | noContent() - | partialContent() - | payloadTooLarge() - | preconditionFailed() - | rangeNotSatisfieable() - | redirect() - | redirectSeeOther() - | requestTimeout() - | serviceUnavailable() - | switchProtocol() - | temporaryRedirect() - | tooManyRequests() - | unauthorized() - | unsupportedHTTPVersion() - | unsupportedMediaType() - ; - -data Request( - loc host = |http://www.example.com|, - str \content-type = "text/plain", - BodyExpectation body = textBody() - ); +private str DEFAULT_CHARSET = "UTF-8"; -@synopsis{Short-hand for construction of JSON post bodies} -Request jsonPost(str path, &T content, loc host=|http://www.example.com|, BodyExpectation body = textBody()) - = post(path, &T (type[&T] _expected) { return content; }, host=host, body=body); +@synopsis{From the client side, a host name is required and also an expected content-type} +@description{ +* `host` is the essential information, the URL to fetch input from. The path will be retrieved from the `path` parameter of `get`, `put`, `post` and `head` +* `content-type` is relevant for POST and PUT options where content is uploaded to the server. The mimetype will be set to this value. +* `charset` is also relevant for POST and PUT options, the outgoing stream will be encoded accordingly and the headers will contained the right meta-data. +} +data Request(loc host = hostIsRequired(), str \content-type = "text/plain", str charset=DEFAULT_CHARSET); @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); -// @synopsis{Short-hand for a string post} -// Request post(str path, str body, loc uri = |http://www.example.com|) -// = post(uri = uri, path, str (type[str] _) { return body; }); +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPost(str path, value content, loc host=hostIsRequired(), JSONOptions options=jsonOptions()) + = post(path, send(json(options=options), content), host=host, \content-type="application/json"); + +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPut(str path, value content, loc host=hostIsRequired(), JSONOptions options=jsonOptions()) + = put(path, send(json(options=options), content), host=host, \content-type="application/json"); + +@synopsis{Short-hand for construction of JSON post bodies} +Request filePut(str path, loc content, loc host=hostIsRequired()) + = put(path, send(file(), content), host=host, \content-type="application/json"); + + +private loc hostIsRequired() throws IllegalArgument { + throw IllegalArgument("missing host parameter"); +} diff --git a/src/org/rascalmpl/repl/rascal/RascalReplServices.java b/src/org/rascalmpl/repl/rascal/RascalReplServices.java index d732c71cfba..b7421a612c6 100644 --- a/src/org/rascalmpl/repl/rascal/RascalReplServices.java +++ b/src/org/rascalmpl/repl/rascal/RascalReplServices.java @@ -43,7 +43,6 @@ import org.rascalmpl.parser.gtd.exception.ParseError; import org.rascalmpl.repl.IREPLService; import org.rascalmpl.repl.StopREPLException; -import org.rascalmpl.repl.TerminalProgressBarMonitor; import org.rascalmpl.repl.completers.CompletionMatcherWithEscapes; import org.rascalmpl.repl.completers.RascalCommandCompletion; import org.rascalmpl.repl.completers.RascalIdentifierCompletion; diff --git a/src/org/rascalmpl/semantics/dynamic/Statement.java b/src/org/rascalmpl/semantics/dynamic/Statement.java index 315acc84e6f..b078fae719f 100644 --- a/src/org/rascalmpl/semantics/dynamic/Statement.java +++ b/src/org/rascalmpl/semantics/dynamic/Statement.java @@ -48,7 +48,6 @@ import org.rascalmpl.interpreter.staticErrors.UninitializedVariable; import org.rascalmpl.interpreter.utils.Cases; import org.rascalmpl.interpreter.utils.Cases.CaseBlock; -import org.rascalmpl.uri.URIUtil; import org.rascalmpl.interpreter.utils.Names; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; From 8c51344e45dbb692ec85bbdbfb7152a0a5b3068f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 May 2026 17:39:31 +0200 Subject: [PATCH 11/11] debugging with @davylandman --- .../exceptions/RuntimeExceptionFactory.java | 3 +- src/org/rascalmpl/library/util/Webclient.java | 130 +++++++++--------- 2 files changed, 64 insertions(+), 69 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index fbb88d3653f..0ae68f807d2 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -450,6 +450,7 @@ public static Throw io(String msg) { } private static String mapIOException(IOException ex) { + var msg = ex.getMessage(); if (ex instanceof FileSystemException) { // nio exceptions lack proper messages, they are encoded in the class name @@ -471,7 +472,7 @@ private static String mapIOException(IOException ex) { return "No such file: " + msg; } // otherwise fallback to the message - return msg; + return msg + (ex.getCause() != null ? (", due to: " + ex.getCause().getMessage()) : ""); } public static Throw io(IOException ex) { diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index f0be0c97232..8dbc107b335 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -17,10 +17,8 @@ import java.net.http.HttpResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.Collections; +import java.util.Map; import java.util.concurrent.Flow.Subscriber; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -47,7 +45,6 @@ import io.usethesource.vallang.IString; import io.usethesource.vallang.ITuple; import io.usethesource.vallang.IValue; -import io.usethesource.vallang.IWithKeywordParameters; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -57,7 +54,7 @@ public class Webclient { private final IRascalMonitor monitor; private final TypeStore store; private final TypeFactory tf; - private final ExecutorService executor; + // private final ExecutorService executor; private final HttpClient.Builder client; public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { @@ -65,7 +62,7 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store this.monitor = monitor; this.store = store; this.tf = tf; - this.executor = Executors.newCachedThreadPool(); + // this.executor = Executors.newCachedThreadPool(); this.client = HttpClient.newBuilder(); } @@ -125,39 +122,45 @@ private HttpRequest makeHeadRequest(IConstructor input, URI uri, String[] header .build(); } - private OutputStreamBodySupplier publishJsonBody(IConstructor input, String charset) { - // make a stream to write to that can also be pulled from by the HTTP client API. - OutputStreamBodySupplier supplier = new OutputStreamBodySupplier(); - - // start the asynchronous task of writing the JSON string to the outputstream - // TODO: how do make sure this job starts and really runs? - executor.submit(() -> { - IWithKeywordParameters kws = input.asWithKeywordParameters().getParameter("options").asWithKeywordParameters(); - IString dtf = kws.getParameter("dateTimeFormat"); - IBool dai = kws.getParameter("dateTimeAsInt"); - IBool ras = kws.getParameter("rationalsAsString"); - IFunction formatters = kws.getParameter("formatter"); - IBool ecn = kws.getParameter("explicitConstructorNames"); - IBool edt = kws.getParameter("explicitDataTypes"); - - JsonValueWriter writer = new JsonValueWriter() - .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setFormatters(formatters) - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) - .setRationalsAsString(ras != null ? ((IBool) ras).getValue() : false) - .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) - .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) - ; - - try (JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(supplier, charset))) { - writer.write(jsonWriter, input); + private BodyPublisher publishJsonBody(IConstructor input, String charset) { + return new BodyPublisher() { + @Override + public void subscribe(Subscriber subscriber) { + // executor.submit(() -> { + IConstructor options = input.asWithKeywordParameters().getParameter("options"); + Map kws = options != null + ? options.asWithKeywordParameters().getParameters() + : Collections.emptyMap(); + IString dtf = (IString) kws.get("dateTimeFormat"); + IBool dai = (IBool) kws.get("dateTimeAsInt"); + IBool ras = (IBool) kws.get("rationalsAsString"); + IFunction formatters = (IFunction) kws.get("formatter"); + IBool ecn = (IBool) kws.get("explicitConstructorNames"); + IBool edt = (IBool) kws.get("explicitDataTypes"); + + JsonValueWriter writer = new JsonValueWriter() + .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters(formatters) + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setRationalsAsString(ras != null ? ((IBool) ras).getValue() : false) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; + + try (JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(new OutputStreamPublisher(subscriber), charset))) { + writer.write(jsonWriter, input); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + // }, true); } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); - } - }, true); - return supplier; + @Override + public long contentLength() { + return -1; + } + }; } private BodyPublisher publishFileBody(IConstructor kind, IConstructor input) { @@ -207,51 +210,36 @@ private BodyPublisher publishBody(IConstructor input, String charset) { /** * On demand streamer for the HttpClient API (for sending bodies for PUT and POST) */ - private static class OutputStreamBodySupplier extends BufferedOutputStream implements BodyPublisher { - private final List> subscribers; + private static class OutputStreamPublisher extends BufferedOutputStream { - public OutputStreamBodySupplier() { - super(new PublishingStream()); - this.subscribers = ((PublishingStream)super.out).subscribers; + public OutputStreamPublisher(Subscriber subscriber) { + super(new PublishingStream(subscriber)); } - /** * The buffed outputstream will take care to collect the bytes untill there's a decent chunk to forward to the consumers */ private static class PublishingStream extends OutputStream { - private final List> subscribers = new CopyOnWriteArrayList<>(); + private final Subscriber subscriber; + + public PublishingStream(Subscriber subscriber) { + this.subscriber = subscriber; + } @Override - public void write(int b) throws IOException { - for (var sub: subscribers) { - sub.onNext(ByteBuffer.wrap(new byte[] { (byte)(b & 0xFF) })); - } + public void write(int b) throws IOException { + subscriber.onNext(ByteBuffer.wrap(new byte[] { (byte)(b & 0xFF) })); } @Override public void write(byte[] b, int off, int len) throws IOException { - for (var sub: subscribers) { - sub.onNext(ByteBuffer.wrap(b, off, len).asReadOnlyBuffer()); - } + subscriber.onNext(ByteBuffer.wrap(b, off, len).asReadOnlyBuffer()); } @Override public void close() throws IOException { - for (var sub: subscribers) { - sub.onComplete(); - } + subscriber.onComplete(); } } - - @Override - public void subscribe(Subscriber subscriber) { - this.subscribers.add(subscriber); - } - - @Override - public long contentLength() { - return -1; - } } private HttpRequest makePostRequest(IConstructor input, URI uri, String[] headers, String charset) { @@ -342,10 +330,16 @@ public IConstructor fetch(IConstructor input) { .followRedirects(HttpClient.Redirect.NORMAL) .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateResponse(request.uri().toString(), (IConstructor) input.asWithKeywordParameters().getParameter("body"), response); + var body = (IConstructor) input.asWithKeywordParameters().getParameter("body"); + + return translateResponse(request.uri().toString(), body, response); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); } - catch (IOException | InterruptedException e) { - throw RuntimeExceptionFactory.io(e.getMessage()); + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; } }