From 10d71ba47a3851c3a5031bd2926859898ed5da9d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 14:58:31 +0200 Subject: [PATCH 1/8] added initial webclient API that mirrors the Webserver API and uses the same Response and Request encodings --- src/org/rascalmpl/library/util/Webclient.java | 162 ++++++++++++++++++ src/org/rascalmpl/library/util/Webclient.rsc | 44 +++++ src/org/rascalmpl/library/util/Webserver.java | 6 +- 3 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 src/org/rascalmpl/library/util/Webclient.java create mode 100644 src/org/rascalmpl/library/util/Webclient.rsc diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java new file mode 100644 index 00000000000..f0237f9f11a --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -0,0 +1,162 @@ +package org.rascalmpl.library.util; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.List; + +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.values.IRascalValueFactory; + +import fi.iki.elonen.NanoHTTPD.Response.Status; +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IMap; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeFactory; +import io.usethesource.vallang.type.TypeStore; + +public class Webclient { + private final IRascalValueFactory vf; + private final IRascalMonitor monitor; + private final TypeStore store; + private final TypeFactory tf; + + public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { + this.vf = vf; + this.monitor = monitor; + this.store = store; + this.tf = tf; + } + + public IConstructor fetch(IConstructor input) { + try { + var params = input.asWithKeywordParameters(); + + switch (input.getName()) { + case "get": + HttpRequest request = HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .GET() + .build(); + + HttpResponse response = + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + + return translateTextResponse(response); + default: + // not yet implemented + throw RuntimeExceptionFactory.illegalArgument(input); + } + } + catch (IOException | InterruptedException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } + } + + private IConstructor translateTextResponse(HttpResponse response) { + IMap headers = response + .headers() + .map() + .entrySet() + .stream() + .map(e -> vf.tuple( + vf.string(e.getKey()), + vf.string(e.getValue().stream().collect(Collectors.joining(",")) + ))) + .collect(vf.mapWriter()); + + IString body = vf.string(response.body()); + IConstructor status = toStatusConstructor(response.statusCode()); + + Type respCons = store.lookupConstructors("response").iterator().next(); + + var contentType = response.headers().firstValue("Content-Type"); + + IString mimeType = vf.string(contentType.get().split(";")[0]); + + return vf.constructor(respCons, status, mimeType, headers, body); + } + + IConstructor toStatusConstructor(int stCode) { + Type statusType = store.lookupAbstractDataType("Status"); + + Status status = Status.lookup(stCode); + switch (status) { + case OK: + return vf.constructor(store.lookupConstructor(statusType, "ok", tf.tupleEmpty())); + case NOT_FOUND: + return vf.constructor(store.lookupConstructor(statusType, "notFound", tf.tupleEmpty())); + case ACCEPTED: + return vf.constructor(store.lookupConstructor(statusType, "accepted", tf.tupleEmpty())); + case BAD_REQUEST: + return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())) + case CONFLICT: + return vf.constructor(store.lookupConstructor(statusType, "conflict", tf.tupleEmpty())); + case CREATED: + return vf.constructor(store.lookupConstructor(statusType, "create", tf.tupleEmpty())); + case EXPECTATION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "expectationFailed", tf.tupleEmpty())); + case FORBIDDEN: + return vf.constructor(store.lookupConstructor(statusType, "forbidden", tf.tupleEmpty())); + case FOUND: + return vf.constructor(store.lookupConstructor(statusType, "found", tf.tupleEmpty())); + case GONE: + return vf.constructor(store.lookupConstructor(statusType, "gone", tf.tupleEmpty())); + case INTERNAL_ERROR: + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); + case LENGTH_REQUIRED: + return vf.constructor(store.lookupConstructor(statusType, "lengthRequired", tf.tupleEmpty())); + case METHOD_NOT_ALLOWED: + return vf.constructor(store.lookupConstructor(statusType, "methodNotAllowed", tf.tupleEmpty())); + case MULTI_STATUS: + return vf.constructor(store.lookupConstructor(statusType, "multiStatus", tf.tupleEmpty())); + case NOT_ACCEPTABLE: + return vf.constructor(store.lookupConstructor(statusType, "notAcceptible", tf.tupleEmpty())); + case NOT_IMPLEMENTED: + return vf.constructor(store.lookupConstructor(statusType, "notImplemented", tf.tupleEmpty())); + case NOT_MODIFIED: + return vf.constructor(store.lookupConstructor(statusType, "notModified", tf.tupleEmpty())); + case NO_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "noContent", tf.tupleEmpty())); + case PARTIAL_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "partialContent", tf.tupleEmpty())); + case PAYLOAD_TOO_LARGE: + return vf.constructor(store.lookupConstructor(statusType, "payloadTooLarge", tf.tupleEmpty())); + case PRECONDITION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "preconditionFailed", tf.tupleEmpty())); + case RANGE_NOT_SATISFIABLE: + return vf.constructor(store.lookupConstructor(statusType, "rangeNotSatisfieable", tf.tupleEmpty())); + case REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "redirect", tf.tupleEmpty())); + case REDIRECT_SEE_OTHER: + return vf.constructor(store.lookupConstructor(statusType, "redirectSeeOther", tf.tupleEmpty())); + case REQUEST_TIMEOUT: + return vf.constructor(store.lookupConstructor(statusType, "requestTimeout", tf.tupleEmpty())); + case SERVICE_UNAVAILABLE: + return vf.constructor(store.lookupConstructor(statusType, "serviceUnavailable", tf.tupleEmpty())); + case SWITCH_PROTOCOL: + return vf.constructor(store.lookupConstructor(statusType, "switchProtocol", tf.tupleEmpty())); + case TEMPORARY_REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "temporaryRedirect", tf.tupleEmpty())); + case TOO_MANY_REQUESTS: + return vf.constructor(store.lookupConstructor(statusType, "tooManyRequests", tf.tupleEmpty())); + case UNAUTHORIZED: + return vf.constructor(store.lookupConstructor(statusType, "unauthorized", tf.tupleEmpty())); + case UNSUPPORTED_HTTP_VERSION: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedHTTPVersion", tf.tupleEmpty())); + case UNSUPPORTED_MEDIA_TYPE: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedMediaType", tf.tupleEmpty())); + default: + break; + } + } + } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc new file mode 100644 index 00000000000..edd3d66751a --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -0,0 +1,44 @@ +module util::Webclient + +extend Content; + +@synopsis{Extends ((Content-Status)) with everything HTTP has out there in the wild.} +data Status + = ok() + | notFound() + | accepted() + | badRequest() + | conflict() + | create() + | expectationFailed() + | forbidden() + | found() + | gone() + | internalError() + | lengthRequired() + | methodNotAllowed() + | multiStatus() + | notAcceptible() + | notImplemented() + | notModified() + | noContent() + | partialContent() + | payloadTooLarge() + | preconditionFailed() + | rangeNotSatisfieable() + | redirect() + | redirectSeeOther() + | requestTimeout() + | serviceUnavailable() + | switchProtocol() + | temporaryRedirect() + | tooManyRequests() + | unauthorized() + | unsupportedHTTPVersion() + | unsupportedMediaType() + ; + +data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain"); + +@javaClass{org.rascalmpl.library.util.Webclient} +java Response fetch(Request request); diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 5dd103a0b5e..d8a885cf45f 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -379,11 +379,7 @@ private IMap makeMap(Map headers) { throw RuntimeExceptionFactory.io(e); } } - - - - - + public void shutdown(ISourceLocation server) { NanoHTTPD nano = servers.get(server); if (nano != null) { From 645d334cc36fa4f8c92c05a8b52eeca4a18d5057 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 17:09:12 +0200 Subject: [PATCH 2/8] cleanup and added POST method --- src/org/rascalmpl/library/util/Webclient.java | 62 ++++++++++++------- src/org/rascalmpl/library/util/Webclient.rsc | 4 ++ 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index f0237f9f11a..4eee8d464dd 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -1,25 +1,21 @@ package org.rascalmpl.library.util; import java.io.IOException; -import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.charset.Charset; -import java.util.Map; import java.util.stream.Collectors; -import java.util.List; - import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.types.TypeReifier; import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.functions.IFunction; import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; -import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -37,25 +33,42 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store this.tf = tf; } + private HttpRequest makeGetRequest(IConstructor input) { + return HttpRequest getRequest = HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .GET() + .build(); + } + + private HttpRequest makePostRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + IFunction postBody = (IFunction) input.get("body"); + IConstructor rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .POST(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) + .build(); + } + + private HttpRequest makeRequest(IConstructor input) { + switch (input.getName()) { + case "get": + return makeGetRequest(input); + case "post": + return makePostRequest(input); + default: + throw RuntimeExceptionFactory.illegalArgument(input); + } + } public IConstructor fetch(IConstructor input) { try { - var params = input.asWithKeywordParameters(); + var request = makeRequest(input); + var response = HttpClient + .newHttpClient() + .send(request, HttpResponse.BodyHandlers.ofString()); - switch (input.getName()) { - case "get": - HttpRequest request = HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) - .GET() - .build(); - - HttpResponse response = - HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - - return translateTextResponse(response); - default: - // not yet implemented - throw RuntimeExceptionFactory.illegalArgument(input); - } + return translateTextResponse(response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); @@ -98,7 +111,7 @@ IConstructor toStatusConstructor(int stCode) { case ACCEPTED: return vf.constructor(store.lookupConstructor(statusType, "accepted", tf.tupleEmpty())); case BAD_REQUEST: - return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())) + return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())); case CONFLICT: return vf.constructor(store.lookupConstructor(statusType, "conflict", tf.tupleEmpty())); case CREATED: @@ -156,7 +169,8 @@ IConstructor toStatusConstructor(int stCode) { case UNSUPPORTED_MEDIA_TYPE: return vf.constructor(store.lookupConstructor(statusType, "unsupportedMediaType", tf.tupleEmpty())); default: - break; + // if we don't understand the error code; let's call it an internal error + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); } } } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index edd3d66751a..ad01b4a9db6 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -42,3 +42,7 @@ data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); + +@synopsis{Short-hand for a string post} +Request post(str path, str body, loc uri = |http://www.example.com|) + = post(uri = uri, path, str (type[str] _) { return body; }); From 61cadea36cf11bb268c8253934dcffa51dc22c8f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 17:19:42 +0200 Subject: [PATCH 3/8] added the other methods --- src/org/rascalmpl/library/util/Webclient.java | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 4eee8d464dd..0207cc56f2c 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.http.HttpClient; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; @@ -34,16 +35,45 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store } private HttpRequest makeGetRequest(IConstructor input) { - return HttpRequest getRequest = HttpRequest.newBuilder() + var params = input.asWithKeywordParameters(); + return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) .GET() .build(); } + private HttpRequest makePutRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + var postBody = (IFunction) input.get("body"); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) + .build(); + } + + private HttpRequest makeDeleteRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .DELETE() + .build(); + } + + private HttpRequest makeHeadRequest(IConstructor input) { + var params = input.asWithKeywordParameters(); + + return HttpRequest.newBuilder() + .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .method("HEAD", BodyPublishers.noBody()) + .build(); + } + private HttpRequest makePostRequest(IConstructor input) { var params = input.asWithKeywordParameters(); - IFunction postBody = (IFunction) input.get("body"); - IConstructor rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + var postBody = (IFunction) input.get("body"); + var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) @@ -56,11 +86,22 @@ private HttpRequest makeRequest(IConstructor input) { case "get": return makeGetRequest(input); case "post": - return makePostRequest(input); + return makePostRequest(input); + case "put": + return makePutRequest(input); + case "delete": + return makeDeleteRequest(input); + case "head": + return makeHeadRequest(input); + default: throw RuntimeExceptionFactory.illegalArgument(input); } } + + /** + * This is the main API method for the Rascal side + */ public IConstructor fetch(IConstructor input) { try { var request = makeRequest(input); @@ -76,7 +117,7 @@ public IConstructor fetch(IConstructor input) { } private IConstructor translateTextResponse(HttpResponse response) { - IMap headers = response + var headers = response .headers() .map() .entrySet() @@ -87,22 +128,22 @@ private IConstructor translateTextResponse(HttpResponse response) { ))) .collect(vf.mapWriter()); - IString body = vf.string(response.body()); - IConstructor status = toStatusConstructor(response.statusCode()); + var body = vf.string(response.body()); + var status = toStatusConstructor(response.statusCode()); - Type respCons = store.lookupConstructors("response").iterator().next(); + var respCons = store.lookupConstructors("response").iterator().next(); var contentType = response.headers().firstValue("Content-Type"); - IString mimeType = vf.string(contentType.get().split(";")[0]); + var mimeType = vf.string(contentType.get().split(";")[0]); return vf.constructor(respCons, status, mimeType, headers, body); } - IConstructor toStatusConstructor(int stCode) { - Type statusType = store.lookupAbstractDataType("Status"); + private IConstructor toStatusConstructor(int stCode) { + var statusType = store.lookupAbstractDataType("Status"); - Status status = Status.lookup(stCode); + var status = Status.lookup(stCode); switch (status) { case OK: return vf.constructor(store.lookupConstructor(statusType, "ok", tf.tupleEmpty())); From 6cce6de693e5ddd0d1259b40cf0222ddf38ea245 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 May 2026 20:33:55 +0200 Subject: [PATCH 4/8] added progress bar --- src/org/rascalmpl/library/util/Webclient.java | 98 ++++++++++++++++--- src/org/rascalmpl/library/util/Webclient.rsc | 6 +- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 0207cc56f2c..a3ddc50808f 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -1,6 +1,8 @@ package org.rascalmpl.library.util; +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; @@ -8,16 +10,16 @@ import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.library.Prelude; import org.rascalmpl.types.TypeReifier; +import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; -import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -37,7 +39,9 @@ public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store private HttpRequest makeGetRequest(IConstructor input) { var params = input.asWithKeywordParameters(); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(URIUtil.getChildLocation( + (ISourceLocation) params.getParameter("uri"), + ((IString) input.get("path")).getValue()).getURI()) .GET() .build(); } @@ -45,6 +49,7 @@ private HttpRequest makeGetRequest(IConstructor input) { private HttpRequest makePutRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var postBody = (IFunction) input.get("body"); + var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); return HttpRequest.newBuilder() .uri(((ISourceLocation) params.getParameter("uri")).getURI()) @@ -106,17 +111,19 @@ public IConstructor fetch(IConstructor input) { try { var request = makeRequest(input); var response = HttpClient - .newHttpClient() - .send(request, HttpResponse.BodyHandlers.ofString()); + .newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateTextResponse(response); + return translateTextResponse(request.uri().toString(), response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - private IConstructor translateTextResponse(HttpResponse response) { + private IConstructor translateTextResponse(String url, HttpResponse response) throws IOException { var headers = response .headers() .map() @@ -128,15 +135,25 @@ private IConstructor translateTextResponse(HttpResponse response) { ))) .collect(vf.mapWriter()); - var body = vf.string(response.body()); - var status = toStatusConstructor(response.statusCode()); + long totalBytes = response.headers() + .firstValueAsLong("Content-Length") + .orElse(-1); - var respCons = store.lookupConstructors("response").iterator().next(); + var input = totalBytes > 0 + ? new MonitoredInputStream(response.body(), monitor, "Fetching " + url, totalBytes) + : response.body(); var contentType = response.headers().firstValue("Content-Type"); var mimeType = vf.string(contentType.get().split(";")[0]); - + + // TODO: extract from contentType if present + var charset = "utf-8"; + + var body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + var status = toStatusConstructor(response.statusCode()); + var respCons = store.lookupConstructors("response").iterator().next(); + return vf.constructor(respCons, status, mimeType, headers, body); } @@ -214,4 +231,63 @@ private IConstructor toStatusConstructor(int stCode) { return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); } } + + private class MonitoredInputStream extends FilterInputStream { + private final IRascalMonitor monitor; + private final String jobName; + + private final long totalBytes; + private long bytesRead = 0; + private boolean started = false; + private boolean done = false; + + public MonitoredInputStream(InputStream in, IRascalMonitor monitor, String jobName, long totalBytes) { + super(in); + this.totalBytes = totalBytes; + this.monitor = monitor; + this.jobName = jobName; + } + + private void ensureStarted() { + if (!started) { + started = true; + monitor.jobStart(jobName, Integer.MAX_VALUE); + } + } + + private void updateProgress(int bytesRead) { + ensureStarted(); + long numberOfTheseSteps = (int) (totalBytes / bytesRead); + int stepSize = (int) (Integer.MAX_VALUE / numberOfTheseSteps); + monitor.jobStep(jobName, "", java.lang.Math.max(stepSize, 1)); + checkDone(); + } + + private void checkDone() { + if (!done && bytesRead >= totalBytes) { + done = true; + monitor.jobEnd(jobName, true); + } + } + + @Override + public int read() throws IOException { + int b = super.read(); + if (b != -1) { + bytesRead += 1; + updateProgress(1); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = super.read(b, off, len); + if (n > 0) { + bytesRead += n; + updateProgress(n); + } + return n; + } + } } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index ad01b4a9db6..dbc31fb37f0 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -43,6 +43,6 @@ data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); -@synopsis{Short-hand for a string post} -Request post(str path, str body, loc uri = |http://www.example.com|) - = post(uri = uri, path, str (type[str] _) { return body; }); +// @synopsis{Short-hand for a string post} +// Request post(str path, str body, loc uri = |http://www.example.com|) +// = post(uri = uri, path, str (type[str] _) { return body; }); From 17723ffaf6c7d7821763570530f580c6b6624595 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 12:25:56 +0200 Subject: [PATCH 5/8] fixed post --- src/org/rascalmpl/library/Content.rsc | 1 - src/org/rascalmpl/library/util/Webclient.java | 110 ++++++++++++++---- src/org/rascalmpl/library/util/Webclient.rsc | 15 ++- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index f7615a63368..765dce04345 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -71,7 +71,6 @@ data Request (map[str, str] headers = (), map[str, str] parameters = (), map[str | head(str path) ; - @synopsis{A response encodes what is send back from the server to the browser client.} @description{ The three kinds of responses, encode either content that is already a `str`, diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index a3ddc50808f..71df5371c5c 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -3,23 +3,38 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.io.StringWriter; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.util.stream.Collectors; +import java.io.Writer; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.library.Prelude; +import org.rascalmpl.library.lang.json.internal.JsonValueReader; +import org.rascalmpl.library.lang.json.internal.JsonValueWriter; import org.rascalmpl.types.TypeReifier; +import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + import fi.iki.elonen.NanoHTTPD.Response.Status; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IString; +import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeStore; @@ -28,20 +43,24 @@ public class Webclient { private final IRascalMonitor monitor; private final TypeStore store; private final TypeFactory tf; + private final TypeReifier tr; public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { this.vf = vf; this.monitor = monitor; this.store = store; this.tf = tf; + this.tr = new TypeReifier(vf); } private HttpRequest makeGetRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); + return HttpRequest.newBuilder() - .uri(URIUtil.getChildLocation( - (ISourceLocation) params.getParameter("uri"), - ((IString) input.get("path")).getValue()).getURI()) + .uri(URIUtil.getChildLocation(host, path).getURI()) .GET() .build(); } @@ -50,40 +69,67 @@ private HttpRequest makePutRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var postBody = (IFunction) input.get("body"); var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) .build(); } private HttpRequest makeDeleteRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .DELETE() .build(); } private HttpRequest makeHeadRequest(IConstructor input) { var params = input.asWithKeywordParameters(); + var host = ((ISourceLocation) params.getParameter("host")); return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) + .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) .method("HEAD", BodyPublishers.noBody()) .build(); } private HttpRequest makePostRequest(IConstructor input) { var params = input.asWithKeywordParameters(); - var postBody = (IFunction) input.get("body"); - var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); - - return HttpRequest.newBuilder() - .uri(((ISourceLocation) params.getParameter("uri")).getURI()) - .POST(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) - .build(); + var postBody = (IFunction) input.get("content"); + var rt = new TypeReifier(vf).typeToValue(tf.valueType(), store, vf.map()); + var host = ((ISourceLocation) params.getParameter("host")); + var val = postBody.call(rt); + var path = ((IString) input.get("path")).getValue(); + + try { + PipedOutputStream out = new PipedOutputStream(); + PipedInputStream in = new PipedInputStream(out, 64 * 1024); + + Thread writer = new Thread(() -> { + try (OutputStream os = out; Writer w = new OutputStreamWriter(out)) { + JsonWriter jsonWriter = new JsonWriter(w); + JsonValueWriter jsonOut = new JsonValueWriter(); + jsonOut.write(jsonWriter, val); + } + catch (Exception e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } + }); + + writer.start(); + + return HttpRequest.newBuilder() + .uri(URIUtil.getChildLocation(host, path).getURI()) + .POST(HttpRequest.BodyPublishers.ofInputStream(() -> in)) + .build(); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } } private HttpRequest makeRequest(IConstructor input) { @@ -116,14 +162,14 @@ public IConstructor fetch(IConstructor input) { .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateTextResponse(request.uri().toString(), response); + return translateResponse(request.uri().toString(), (IConstructor) input.asWithKeywordParameters().getParameter("body"), response); } catch (IOException | InterruptedException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - private IConstructor translateTextResponse(String url, HttpResponse response) throws IOException { + private IConstructor translateResponse(String url, IConstructor expect, HttpResponse response) throws IOException { var headers = response .headers() .map() @@ -149,12 +195,32 @@ private IConstructor translateTextResponse(String url, HttpResponse // TODO: extract from contentType if present var charset = "utf-8"; - - var body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + var status = toStatusConstructor(response.statusCode()); - var respCons = store.lookupConstructors("response").iterator().next(); - return vf.constructor(respCons, status, mimeType, headers, body); + Type respCons; + IString body; + + switch (expect != null ? expect.getName() : "textBody") { + case "jsonBody": + JsonReader jsonReader = new JsonReader(new InputStreamReader(input)); + JsonValueReader parser = new JsonValueReader(vf, store, monitor, URIUtil.assumeCorrectLocation(url)); + respCons = store.lookupConstructors("jsonResponse").iterator().next(); + var value = parser.read(jsonReader, tr.valueToType((IConstructor) expect.get("expected"))); + return vf.constructor(respCons, status, headers, value); + case "fileBody": + respCons = store.lookupConstructors("fileResponse").iterator().next(); + var loc = (ISourceLocation) expect.get("storage"); + try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(loc, false)) { + input.transferTo(out); + } + return vf.constructor(respCons, loc, mimeType, headers); + case "textBody": + default: + respCons = store.lookupConstructors("response").iterator().next(); + body = vf.string(new String(Prelude.consumeInputStream(input), charset)); + return vf.constructor(respCons, status, mimeType, headers, body); + } } private IConstructor toStatusConstructor(int stCode) { @@ -255,7 +321,11 @@ private void ensureStarted() { } } - private void updateProgress(int bytesRead) { + private void updateProgress(int bytesRead) throws InterruptedIOException { + if (monitor.jobIsCanceled(jobName)) { + throw new InterruptedIOException(jobName); + } + ensureStarted(); long numberOfTheseSteps = (int) (totalBytes / bytesRead); int stepSize = (int) (Integer.MAX_VALUE / numberOfTheseSteps); diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index dbc31fb37f0..2f0bbc2bc8b 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -2,6 +2,12 @@ module util::Webclient extend Content; +data BodyExpectation + = textBody() + | jsonBody(type[value] expected) + | fileBody(loc storage) + ; + @synopsis{Extends ((Content-Status)) with everything HTTP has out there in the wild.} data Status = ok() @@ -38,7 +44,14 @@ data Status | unsupportedMediaType() ; -data Request(loc uri = |http://www.example.com|, str \content-type = "text/plain"); +data Request( + loc host = |http://www.example.com|, + str \content-type = "text/plain", + BodyExpectation body = text() + ); + +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPost(str path, &T content, loc host=|http://www.example.com|) = post(path, &T (type[&T] _expected) { return content; }, host=host); @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); From b9c1e6ea7b5ac93ed1ef7be158630501c7b2d053 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 12:33:40 +0200 Subject: [PATCH 6/8] constructor typo --- src/org/rascalmpl/library/util/Webclient.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index 2f0bbc2bc8b..8b66830cc30 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -47,7 +47,7 @@ data Status data Request( loc host = |http://www.example.com|, str \content-type = "text/plain", - BodyExpectation body = text() + BodyExpectation body = textBody() ); @synopsis{Short-hand for construction of JSON post bodies} From 8c9c0dafa5b1103bea49457066f022f18e2ec112 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 16:25:36 +0200 Subject: [PATCH 7/8] fix post bug --- src/org/rascalmpl/library/util/Webclient.rsc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc index 8b66830cc30..42066546f3b 100644 --- a/src/org/rascalmpl/library/util/Webclient.rsc +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -51,7 +51,8 @@ data Request( ); @synopsis{Short-hand for construction of JSON post bodies} -Request jsonPost(str path, &T content, loc host=|http://www.example.com|) = post(path, &T (type[&T] _expected) { return content; }, host=host); +Request jsonPost(str path, &T content, loc host=|http://www.example.com|, BodyExpectation body = textBody()) + = post(path, &T (type[&T] _expected) { return content; }, host=host, body=body); @javaClass{org.rascalmpl.library.util.Webclient} java Response fetch(Request request); From ddf4bfa91c806aa6f1aa6f97b88680ce0aa97b79 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 May 2026 16:28:01 +0200 Subject: [PATCH 8/8] added path to other requests kinds but GET and POST --- src/org/rascalmpl/library/util/Webclient.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java index 71df5371c5c..9a9a4c4ba94 100644 --- a/src/org/rascalmpl/library/util/Webclient.java +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -70,9 +70,11 @@ private HttpRequest makePutRequest(IConstructor input) { var postBody = (IFunction) input.get("body"); var rt = new TypeReifier(vf).typeToValue(tf.stringType(), store, vf.map()); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .PUT(HttpRequest.BodyPublishers.ofString(((IString) postBody.call(rt)).getValue())) .build(); } @@ -80,9 +82,11 @@ private HttpRequest makePutRequest(IConstructor input) { private HttpRequest makeDeleteRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .DELETE() .build(); } @@ -90,9 +94,11 @@ private HttpRequest makeDeleteRequest(IConstructor input) { private HttpRequest makeHeadRequest(IConstructor input) { var params = input.asWithKeywordParameters(); var host = ((ISourceLocation) params.getParameter("host")); + host = host == null ? URIUtil.assumeCorrectLocation("http://www.example.com") : host; + var path = ((IString) input.get("path")).getValue(); return HttpRequest.newBuilder() - .uri(host != null ? host.getURI() : URIUtil.assumeCorrect("http://www.example.com")) + .uri(URIUtil.getChildLocation(host, path).getURI()) .method("HEAD", BodyPublishers.noBody()) .build(); } @@ -161,7 +167,6 @@ public IConstructor fetch(IConstructor input) { .followRedirects(HttpClient.Redirect.NORMAL) .build() .send(request, HttpResponse.BodyHandlers.ofInputStream()); - return translateResponse(request.uri().toString(), (IConstructor) input.asWithKeywordParameters().getParameter("body"), response); } catch (IOException | InterruptedException e) {