Synsemadocsv0.6.xENES

Standard library / primitives

HTTP client

Every request (http* and fetch) is gated by net(host) — deny-by-default, even under run. HTTPS works out of the box (rustls + OS root CAs).

http-client.syn
-- Doc example: the HTTP client is gated by net(host). A request to an undeclared
-- host is refused at the capability check — before any network call happens. Every
-- test below stops before the network: nothing here needs a server.
intent: "doc example: HTTP client capability gating"
require net("api.allowed.com")
require net("http://localhost:8545")    -- a URL as the grant: this host, ONLY this port

task reach_undeclared()
    give http_get("https://evil.example.com/data")    -- host not declared → denied

task reach_other_port()
    give http_get("http://localhost:9999/")           -- same host, another port → denied

-- The message of an error, or nothing when the call did not fail.
task error_of(f)
    try
        f()
    recover e
        give e
    give nothing

test "a request to an undeclared host is blocked (deny-by-default)"
    assert_error(reach_undeclared)

test "an explicit port is part of the grant"
    assert(contains(error_of(reach_other_port), "net(\"localhost:9999\")"))

test "credentials in a URL never reach an error message"
    let e be error_of(() => http_get("https://ana:hunter2@evil.example.com/"))
    assert(contains(e, "net(\"evil.example.com\")"))
    assert(not contains(e, "hunter2"))
    let e2 be error_of(() => http_get("http://ana:hunter2@/rpc"))
    assert(contains(e2, "the URL has no host"))
    assert(not contains(e2, "hunter2"))

test "an extra argument is an error, not silently dropped"
    let e be error_of(() => http_get("https://api.allowed.com/", nothing, nothing, 5, "extra"))
    assert(contains(e, "http_get() takes at most 4 arguments, got 5"))

test "multipart_encode builds the body and its content type (pure)"
    let m be multipart_encode([
        {"name": "title", "value": "Q3 report"},
        {"name": "file", "filename": "q3.bin", "content_type": "application/octet-stream", "bytes": bytes([1, 2, 3])}])
    assert_eq(keys(m), ["body", "content_type"])
    assert_eq(type_of(body of m), "bytes")
    assert(starts_with(content_type of m, "multipart/form-data; boundary="))
    assert(contains(error_of(() => multipart_encode([{"name": "x"}])), "needs \"value\" or \"bytes\""))

test "fetch takes its options in a map: unknown keys and a trailing argument are errors"
    let e be error_of(() => fetch("https://api.allowed.com/", {"pin": "00"}))
    assert(contains(e, "unknown option \"pin\" (valid options: method, headers, body, form, timeout, tls_pin, attested)"))
    let e2 be error_of(() => fetch("https://api.allowed.com/", {"timeout": 5}, 30))
    assert(contains(e2, "the options map is the last argument"))

test "tls_pin must be a public key, checked before any connection"
    let e be error_of(() => fetch("https://api.allowed.com/", {"tls_pin": "abcd"}))
    assert(contains(e, "tls_pin is not a DER SubjectPublicKeyInfo"))

test "named arguments: leave out what you do not need (engine v0.6.45+)"
    -- The host is not declared, so each call stops at the capability check — after its
    -- arguments were read. A bad name or a duplicate fails before that.
    let e be error_of(() => http_get("https://evil.example.com/", headers = {"accept": "application/json"}, timeout = 5))
    assert(contains(e, "net(\"evil.example.com\")"))
    let e2 be error_of(() => http(method = "PUT", url = "https://evil.example.com/x", body = {"a": 1}))
    assert(contains(e2, "net(\"evil.example.com\")"))
    assert(starts_with(error_of(() => http_get("https://api.allowed.com/", timout = 5)), "unknown parameter 'timout'"))
    assert_eq(error_of(() => http_get("https://api.allowed.com/", {"a": "b"}, headers = {"a": "b"})), "duplicate argument 'headers'")

test "header() finds a header regardless of case"
    let r be {"status": 429, "ok": false, "body": "", "json": nothing, "headers": {"Retry-After": "5"}}
    assert_eq(header(r, "retry-after"), "5")
    assert_eq(header({"content-type": "text/html"}, "Content-Type"), "text/html")
    assert_eq(header(r, "x-missing"), nothing)

test "url_encode / url_decode follow RFC 3986 and decode strictly"
    assert_eq(url_encode("café & tea?"), "caf%C3%A9%20%26%20tea%3F")
    assert_eq(url_encode("a-b.c_d~e"), "a-b.c_d~e")
    assert_eq(url_decode("caf%C3%A9%20%26%20tea%3F"), "café & tea?")
    assert_eq(url_decode("a+b"), "a+b")
    assert(contains(error_of(() => url_decode("%zz")), "malformed escape"))
    assert(contains(error_of(() => url_decode("%FF")), "not valid UTF-8"))

test "html_select / html_text / html_unescape read real-world HTML"
    let page be "<ul id=menu><li class=item data-id=7>Tea &amp; cake<li class=item data-id=9>Coffee</ul><script>track()</script><p>Open<br>daily"
    let items be html_select(page, "li.item")
    assert_eq(length(items), 2)
    assert_eq(tag of items[0], "li")
    assert_eq((attrs of items[0])["data-id"], "7")
    assert_eq(text of items[0], "Tea & cake")
    assert_eq(html of items[0], "Tea &amp; cake")
    assert_eq(html_select(page, "table"), [])
    assert_eq(html_text(page), "Tea & cake\nCoffee\nOpen\ndaily")
    assert_eq(html_unescape("&lt;b&gt; &amp; &eacute;"), "<b> & é")
    assert(contains(error_of(() => html_select(page, "td[[")), "invalid CSS selector"))

Making requests§

require net("api.store.com")

let r be http_get("https://api.store.com/products")
let url be "https://api.store.com/products"
let r2 be http_get(url, headers = {"x-api-key": secret("API_KEY")}, query = {"page": "1"})
let r3 be http_post(url, {"name": "Alice"}, headers = {"Authorization": bearer(secret("API_KEY"))})   -- a map → JSON body + Content-Type
let r4 be http(method = "PATCH", url = url, body = {"stock": 3}, timeout = 10)   -- full control

Named arguments (engine v0.6.45+)§

The HTTP builtins take their arguments by name: http_get(url, headers = {…}, query = {…}, timeout = 10), http_post(url, body, headers = {…}, timeout = 30), http(method = "PUT", url = u, body = b), and the same for http_bytes, http_put and http_delete. Leave out what you do not need — no nothing placeholders. The names are the ones of the signatures below (method, url, headers, query, body, timeout, and form since v0.6.46).

The body is sent by type (engine v0.6.20+). A map or list goes out as JSON with Content-Type: application/json (unless you set your own Content-Type); text goes out as-is, with no content type added; bytes go out raw. Before v0.6.20 a map was sent as its display text with no header, and you wrote json_encode(map) plus the header yourself — that still works unchanged.

Credentials go in headers (a secret materializes only at the socket); a secret bound to a host (require secret("K") for net("api.example.com"), engine v0.6.46+) may also go in the URL, the query, the body or a form — see Secrets.

Hosts, ports and credentials in the URL (engine v0.6.29+)§

The example's tests check the grant, the messages and the argument count without touching the network.

Timeout§

Every HTTP builtin takes an optional timeout in seconds — timeout = … by name, or the timeout key in fetch's options map — (default 30; absent or invalid falls back to 30 — never an error):

require net("api.store.com")
let url be "https://api.store.com/report"
let r be fetch(url, {"timeout": 60})                  -- fetch takes its options in a map
let r2 be http(method = "GET", url = url, timeout = 120)   -- slow API: wait up to 2 min
let r3 be http_get(url, timeout = 5)                   -- fail fast: 5s

Signatures (every argument of http* can also be passed as name = value): fetch(url, {method, headers, body, form, timeout, tls_pin, attested}) · http(method, url, headers?, query?, body?, timeout?, form?) · http_get(url, headers?, query?, timeout?) · http_post(url, body?, headers?, timeout?, form?) · http_put(url, body?, headers?, timeout?, form?) · http_delete(url, headers?, timeout?) · http_bytes(method, url, headers?, query?, body?, timeout?, form?). On timeout the response comes back with ok: false and the OS error in error of r.

fetch with a map (engine v0.6.43+) is the main form: every key is optional — method (default "GET"), headers, body (text, bytes, or a map/list sent as JSON), form (v0.6.46+, below), timeout (seconds, default 30), tls_pin and attested, with the same types and defaults as the positional form; a key set to nothing counts as absent, except tls_pin and attested: there nothing is an error, so a missing value never turns the check off in silence (leave the key out for the usual certificate check). Nothing may follow the map (fetch(url, {…}, 30) is an error) and an unknown key is an error that lists the valid ones.

The positional fetch(url, method, headers, body, timeout) is deprecated (engine v0.6.45) and goes away in v1. Until then it still works, and the first call of a process prints one notice to stderr: [synsema] deprecated: fetch(url, method, headers, body, timeout) goes away in v1 — write fetch(url, {"method": …, "headers": …, "body": …, "timeout": …}). synsema check points at each call whose second argument is a text literal or nothing, so you can fix them before running.

tls_pin fixes the server's key: its SubjectPublicKeyInfo as hex, bytes or a PUBLIC KEY PEM. Instead of checking the certificate against the system roots and the host name, the connection requires the key the server presents to be exactly that one (the handshake signature is still verified with it); another key is the error tls_pin: the server public key (SPKI) does not match the pinned key. It needs an https:// URL. This is how a client talks to a serve --attested, whose certificate is self-signed with the attested key — see Attestation. ws_connect takes the same tls_pin in its options (wss://).

attested ({program_sha, formats, measurements?, now?}) connects to a serve --attested without knowing its key: on one connection it reads /.well-known/attestation, requires its key to be the handshake's, verifies every document and the program_sha, and only then sends your request — which never leaves if anything fails. The answer carries attested. The steps are in Attestation; ws_connect takes it too.

In the WebAssembly embedding (synsema-wasm-web, where the host does the HTTP) tls_pin and attested fail closed with an error: the host's transport cannot pin a key or read the identity over the handshake's connection.

An argument past the last one is an error since engine v0.6.29 (http_get() takes at most 4 arguments, got 5); before, it was dropped without a word.

The response§

A request returns a map:

status of r     -- 200
ok of r         -- true (200–299)
body of r       -- raw text — always text (a binary body is lossy: see http_bytes)
json of r       -- the parsed body when the response is JSON (its content type says so and it parses); otherwise nothing (v0.6.20+)
headers of r    -- response headers, names exactly as the server sent them
error of r      -- ONLY present when the transport failed (DNS, refused, timeout)

Exactly those keys — keys(r) on a normal response is [status, ok, body, json, headers] (json arrived in v0.6.20 — on older engines the key does not exist). When the transport fails, the map is {status: 0, ok: false, body: "", json: nothing, error} — with error and without headers:

let r be {"status": 429, "ok": false, "body": "", "json": nothing, "headers": {"Retry-After": "5"}}
print(header(r, "retry-after"))                       -- 5
print(header({"content-type": "text/html"}, "Content-Type"))   -- text/html (a map of headers works too)
print(header(r, "x-missing"))                         -- nothing

Before v0.6.45 the manual showed a hand-written task header(r, name); a task of yours with that name hides the builtin, so drop it when you upgrade.

Binary bodies — http_bytes (engine v0.6.20+)§

body of r is always text, lossy for binary. For a PDF, an image or a protobuf, http_bytes(method, url, headers?, query?, body?, timeout?) has the signature of http and returns {status, ok, bytes, headers} — the exact bytes the server sent, no body key. Same net(host) gate.

File uploads — multipart_encode (engine v0.6.20+)§

let m be multipart_encode([
    {"name": "title", "value": "Q3 report"},
    {"name": "file", "filename": "q3.pdf", "content_type": "application/pdf", "bytes": pdf}])
let r be http_post(url, body of m, {"Content-Type": content_type of m})

multipart_encode(parts) → {body: bytes, content_type: "multipart/form-data; boundary=…"}; a part is {name, value} (a text field) or {name, filename, content_type?, bytes} (a file). Pure and deterministic (the boundary derives from the parts). A Synsema serve on the other side reads it as form of request. Streaming very large files is not covered — the body is built in memory.

Form bodies — form = {…} (engine v0.6.46+)§

http, http_bytes, http_post, http_put and fetch take a form: a map sent as an application/x-www-form-urlencoded body (the encoder of oauth_basic), with that Content-Type unless you set your own. It is what OAuth token endpoints and classic HTML forms expect.

require net("auth.example.com")
let r be http_post("https://auth.example.com/token", form = {"grant_type": "client_credentials", "scope": "read write", "n": 1, "renew": true})
let r2 be fetch("https://auth.example.com/token", {"method": "POST", "form": {"grant_type": "refresh_token"}})
-- sent as: grant_type=client_credentials&scope=read+write&n=1&renew=true

Redirects are not followed§

The client returns a redirect (3xx) as the response — status and the Location header (header(r, "location")) — and your program decides whether to go there (with its own net grant). A redirect never carries a header or a secret to a host the program did not name. A host that embeds the engine and implements http receives redirect: "error" when the request carries a secret (engine v0.6.46+), and should honor it — see WebAssembly.

A wildcard reaches only public addresses (engine v0.6.46+)§

require net without a scope, net("") and net(".example.com") reach only public IP addresses. Loopback (127.0.0.0/8, ::1, 0.0.0.0), the private networks (10/8, 172.16/12, 192.168/16, CGNAT 100.64/10, ULA fc00::/7), link-local (169.254/16 with the cloud metadata endpoint 169.254.169.254, fe80::/10), multicast and the documentation ranges (192.0.2/24, 198.51.100/24, 203.0.113/24) are out. Before v0.6.46 net compared only the name, so 127.0.0.1.nip.io — or any name that resolved inside — reached your internal services (SSRF). The client resolves once, checks the address it is going to use and connects to that one, so a DNS answer cannot change in between; an IPv4 mapped or translated in IPv6 (::ffff:127.0.0.1, 64:ff9b::7f00:1) counts as that IPv4.

To reach a private address, name it:

require env("CONTROL_URL")
require net("10.0.0.0/8")              -- a private range, named: covers 10.x.x.x and names that resolve inside
require net("db.internal:5432")        -- an exact host reaches it wherever it resolves
require net(env("CONTROL_URL"))        -- the host of a configured URL, named from there
require net("*")                       -- any host, but only public addresses

A wildcard that meets a private address does not connect. The HTTP builtins return it like any transport failure — {status: 0, ok: false, …, error} — with the line to write in error of r: `http_get: 127.0.0.1.nip.io resolved to 127.0.0.1, a private address; a net grant with a wildcard reaches only public addresses. Name it: require net("127.0.0.1.nip.io:47100"), its address or its CIDR (the scope can come from the configuration: require net(env("…_URL"))). tcp_connect, ws_connect and the blockchain RPC builtins raise the same message as an error. When the wildcard is a layer above the program, the message names that layer: the host ceiling (--sandbox/--cap-set) reaches only public addresses for it. The program cannot fix this; the host must name it: --cap-set net=127.0.0.1:47100, or the enclosing sandbox under block reaches only public addresses for it: name it there (net("127.0.0.1:47100"))`. Every layer that has a wildcard has to name the private host.

net-private.syn
-- Doc example: a `net` grant with a wildcard reaches only public addresses (engine v0.6.46+).
-- Nothing here leaves the machine: a private address is refused before connecting, and the
-- named range points at a closed local port.
intent: "doc example: wildcard net grants and private addresses"
require net("*")                 -- any host, public addresses only
require net("127.0.0.0/8")       -- loopback, named: its literal addresses (and, with the wildcard, names inside)

test "a wildcard does not reach loopback, private networks or the cloud metadata endpoint"
    each u in ["http://10.0.0.5/", "http://192.168.1.1/", "http://169.254.169.254/latest/", "http://[::1]:8080/", "http://[fd00::1]/"]
        let r be http_get(u, timeout = 2)
        assert_eq(r.status, 0)                                 -- a transport failure, never a connection
        assert(contains(r.error, "a private address; a `net` grant with a wildcard reaches only public addresses"))
    -- the message gives the line to write
    assert(contains(http_get("http://10.0.0.5:5432/").error, "Name it: `require net(\"10.0.0.5:5432\")`"))
    -- an IPv4 mapped in IPv6 counts as that IPv4
    assert(contains(http_get("http://[::ffff:10.0.0.5]/").error, "resolved to 10.0.0.5"))

test "a named range reaches the address: the request is attempted"
    let r be http_get("http://127.0.0.1:9/", timeout = 2)   -- port 9 (discard): closed here
    assert_eq(r.status, 0)
    assert(not contains(r.error, "private address"))      -- refused by the OS, not by the grant

It applies to http/fetch and the http_* builtins, ws_connect (and every reconnection), tcp_connect, the blockchain RPC builtins, push_send and proxy to (see serve). A secret bound to a host goes to a private address only if its binding names it too. The database connections (postgres, mysql, mongo, redis) are gated by db, not net, and are not affected.

On Windows localhost may resolve to ::1 first, and a serve listens on 0.0.0.0 (IPv4): the client connects to the one address it checked, so http://localhost:8080/ can be refused while http://127.0.0.1:8080/ answers. Use 127.0.0.1 for a local server.

Encoding a URL — url_encode / url_decode (engine v0.6.45+)§

url_encode(text) percent-encodes everything but A-Z a-z 0-9 - . _ ~ (RFC 3986), so the result is safe in a path segment or a query value. url_decode(text) is strict: a % not followed by two hex digits is the error url_decode: malformed escape at position N …, bytes that are not UTF-8 are url_decode: the decoded bytes are not valid UTF-8, and + stays a + (it is a space only in HTML form bodies, which form of request already decodes). For a query string you do not need either: query = {…} encodes the values.

require net("api.store.com")
let q be url_encode("café & tea?")
print(q)                                  -- caf%C3%A9%20%26%20tea%3F
print(url_decode(q))                      -- café & tea?
print(url_decode("a+b"))                  -- a+b  (a + is not a space)
let r be http_get("https://api.store.com/search", query = {"q": "café & tea"})   -- query = … encodes for you

Reading HTML — html_select, html_text, html_unescape (engine v0.6.45+)§

A page you fetched is HTML, not JSON. These three parse it as a browser does (WHATWG parsing: unclosed tags, unquoted attributes and entities are fine):

All three are pure (no capability) and what they return keeps the labels of the input: reading a private page does not make its contents public (see Labels).

let page be "<ul id=menu><li class=item data-id=7>Tea &amp; cake<li class=item data-id=9>Coffee</ul><script>track()</script><p>Open<br>daily"
each m in html_select(page, "li.item")
    print((attrs of m)["data-id"] + ": " + text of m)    -- 7: Tea & cake / 9: Coffee
print(html_text(page))                                     -- Tea & cake, Coffee, Open, daily: one line each
print(html_unescape("&lt;b&gt; &amp; &eacute;"))           -- <b> & é

require net / net("") allow any host, and net(".x.com") matches subdomains — on public addresses only (engine v0.6.46+, above).