Preserve binary request bodies in http-fetch-bytes; release 0.18.6

Changed

 .woodpecker.yml                       |  6 +++---
 CHANGELOG.md                          | 12 ++++++++++++
 README.md                             | 17 +++++++++++++++++
 package.sgl                           |  2 +-
 src/sigil/http/client.sgl             | 21 ++++++++++++++++++---
 test/integration/run-upload-tests.mjs | 26 ++++++++++++++++++++++++++
 test/test-upload-bytes.sgl            | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 127 insertions(+), 7 deletions(-)

Diff

.woodpecker.ymlmodified

@@ -21,9 +21,9 @@ steps:

test:

image: *alpine_image

commands:

−24

- apk add --no-cache curl ca-certificates git

−25

- curl -fLo ./sigil https://codeberg.org/sigil/sigil/releases/download/v0.14.3/sigil-linux-amd64-static

+24

- apk add --no-cache curl ca-certificates git nodejs

+25

- curl -fLo ./sigil https://codeberg.org/sigil/sigil/releases/download/v0.21.0/sigil-linux-amd64-static

- chmod +x ./sigil

- ./sigil --version

- ./sigil deps install

−29

- ./sigil test

+29

- PATH="$PWD:$PATH" node test/integration/run-upload-tests.mjs

CHANGELOG.mdmodified

@@ -5,6 +5,18 @@ All notable changes to **sigil-http** are documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.18.6] - 2026-09-06

+10

### Fixed

+11

+12

- Accept bytevector request bodies in `http-fetch-bytes`, preserving binary

+13

uploads without UTF-8 conversion. Compute Content-Length from the bytevector

+14

and replace conflicting length/transfer-encoding headers case-insensitively.

+15

Existing string request bodies keep their UTF-8 behavior.

+16

- Exercise binary assembly and actual empty/2 MB upload round trips alongside

+17

the HTTP regression suite. CI now uses Sigil 0.21.0, matching the test runner's

+18

existing minimum dependency, and runs the loopback upload checks with Node.

+19

## [0.18.5] - 2026-08-27

### Added

README.mdmodified

@@ -32,6 +32,23 @@ under `packages/sigil-http/` is preserved here as `master`.

See `docs/http.md` for the full API reference, including streaming bodies,

Server-Sent Events, and low-level request/response manipulation.

+35

## Binary uploads

+36

+37

`(sigil http client)` exports `http-fetch-bytes`. Its `body:` argument

+38

accepts a string or bytevector; use bytevectors for images and other binary data.

+39

The response has `status:`, an ordered `headers:` alist, and bytevector `body:`.

+40

Redirects are returned to the caller and transport failures return `#f`.

+41

+42

```scheme

+43

(import (sigil http client))

+44

(http-fetch-bytes 'PUT "https://example.com/object"

+45

headers: #{ content-type: "application/octet-stream" }

+46

body: image-bytes timeout: 30)

+47

```

+48

+49

Run `node test/integration/run-upload-tests.mjs` (Node 18+) for the full Sigil

+50

suite plus an ephemeral loopback server exercising empty and 2 MB binary bodies.

+51

## Dependencies

- `sigil-stdlib`, `sigil-socket`, `sigil-tls` — pinned via `from-git`

package.sglmodified

@@ -5,7 +5,7 @@

(package

name: "sigil-http"

−8

version: "0.18.5"

version: "0.18.6"

sigil: "0.20"

description: "HTTP client and server library for Sigil"

url: "https://codeberg.org/sigil/sigil-http"

src/sigil/http/client.sglmodified

@@ -77,6 +77,7 @@

chunked-body-complete?

framing-complete?

;; http-fetch-bytes internals — exported for testing

+80

build-request-bytes

fetch-parse-headers

fetch-content-length

fetch-assemble

@@ -610,6 +611,20 @@

611

;; Body

612

(or body ""))))

613

+614

;; Byte-faithful request assembly for http-fetch-bytes. Existing string

+615

;; request behavior stays unchanged; binary bodies never round-trip UTF-8.

+616

(define (build-request-bytes method parsed-url headers body)

+617

(if (bytevector? body)

+618

(let* ((length-value (number->string (bytevector-length body)))

+619

(entries (if (dict? headers)

+620

(map (lambda (h) (cons (keyword->string (car h)) (cdr h))) (dict-entries headers))

+621

headers))

+622

(with-length (cons (cons "content-length" length-value)

+623

(filter (lambda (h)

+624

(not (member (string-downcase (car h)) '("content-length" "transfer-encoding")))) entries))))

+625

(bytevector-append (string->utf8 (build-request-string method parsed-url with-length #f)) body))

+626

(string->utf8 (build-request-string method parsed-url headers body))))

+627

628

;;; Build header lines from headers (dict or alist)

629

(define (build-header-lines headers)

630

(cond

@@ -781,7 +796,7 @@

796

(define (write-all-data conn data deadline)

797

(if (not deadline)

798

(and (conn-write conn data) #t)

−784

(let* ((bv (string->utf8 data))

+799

(let* ((bv (if (bytevector? data) data (string->utf8 data)))

800

(len (bytevector-length bv)))

801

(let loop ((sent 0))

802

(cond

@@ -1408,7 +1423,7 @@

1423

(define fetch-max-idle-polls 6000)

1424

1425

;;; Fetch `url` (a `method` symbol, optional request `headers` dict and

−1411

;;; string `body`) and return the response as raw bytes:

+1426

;;; string or bytevector `body`) and return the response as raw bytes:

1427

;;;

1428

;;; #{ status: <integer>

1429

;;; headers: <ordered alist of (lowercased-name . value)>

@@ -1471,7 +1486,7 @@

1486

(conn-set-non-blocking! conn)

1487

(let ((deadline (timeout->deadline secs)))

1488

(write-all-data conn

−1474

(build-request-string method parsed headers body)

+1489

(build-request-bytes method parsed headers body)

1490

deadline))

1491

(let ((result (fetch-read-response conn method secs)))

1492

(fetch-safe-close conn)

test/integration/run-upload-tests.mjsadded

@@ -0,0 +1,26 @@

// Node 18+; ephemeral loopback listener, no third-party services or credentials.

import http from 'node:http';

import assert from 'node:assert/strict';

import { spawn } from 'node:child_process';

const lengths = [];

const server = http.createServer(async (req, res) => {

const chunks = [];

for await (const chunk of req) chunks.push(chunk);

const body = Buffer.concat(chunks);

+10

lengths.push(body.length);

+11

res.writeHead(200, { 'content-length': String(body.length) });

+12

res.end(body);

+13

});

+14

await new Promise((resolve, reject) => {

+15

server.once('error', reject); server.listen(0, '127.0.0.1', resolve);

+16

});

+17

try {

+18

const child = spawn('sigil', ['test', '--sgl', '--strict', ...process.argv.slice(2)], {

+19

stdio: 'inherit', env: { ...process.env, HTTP_UPLOAD_TEST_PORT: String(server.address().port) },

+20

});

+21

const timeout = setTimeout(() => child.kill('SIGTERM'), 180_000);

+22

const code = await new Promise((resolve, reject) => { child.once('error', reject); child.once('exit', resolve); });

+23

clearTimeout(timeout);

+24

assert.equal(code, 0, 'HTTP suite including binary round-trip passed');

+25

assert.deepEqual(lengths, [2 * 1024 * 1024, 0], 'both binary requests reached the server');

+26

} finally { server.closeAllConnections(); server.close(); }

test/test-upload-bytes.sgladded

@@ -0,0 +1,50 @@

(import (sigil core) (sigil io) (sigil test) (sigil string) (sigil process) (sigil http client))

(define (same-bytes? a b)

(and (= (bytevector-length a) (bytevector-length b))

(let loop ((i 0))

(or (= i (bytevector-length a))

(and (= (bytevector-u8-ref a i) (bytevector-u8-ref b i)) (loop (+ i 1)))))))

(define (sample-bytes size)

(let ((bytes (make-bytevector size)))

+10

(let loop ((i 0))

+11

(when (< i size) (bytevector-u8-set! bytes i (modulo i 256)) (loop (+ i 1)))) bytes))

+12

+13

(test-group "binary request assembly"

+14

(test "arbitrary bytes bypass UTF-8 encoding"

+15

(let* ((body (sample-bytes 256))

+16

(wire (build-request-bytes 'PUT (parse-url "http://example.com/object") #{} body))

+17

(split (- (bytevector-length wire) 256))

+18

(head (utf8->string (bytevector-copy wire 0 split))))

+19

(assert-true (string-contains? head "content-length: 256\r\n"))

+20

(assert-true (same-bytes? (bytevector-copy wire split) body))))

+21

(test "dict framing headers replaced case-insensitively"

+22

(let* ((wire (build-request-bytes 'PUT (parse-url "http://example.com/")

+23

#{ Content-Length: "999" content-length: "88" Transfer-Encoding: "chunked" } (make-bytevector 0)))

+24

(head (string-downcase (utf8->string wire))))

+25

(assert-equal (length (string-split head "content-length:")) 2)

+26

(assert-true (string-contains? head "content-length: 0\r\n"))

+27

(assert-false (string-contains? head "transfer-encoding:"))))

+28

(test "alist framing headers replaced case-insensitively"

+29

(let ((head (string-downcase (utf8->string

+30

(build-request-bytes 'PUT (parse-url "http://example.com/")

+31

'(("Content-Length" . "999") ("TRANSFER-ENCODING" . "chunked")) (make-bytevector 0))))))

+32

(assert-equal (length (string-split head "content-length:")) 2)

+33

(assert-false (string-contains? head "transfer-encoding:"))))

+34

(test "string body preserves existing UTF-8 wire format"

+35

(let ((url (parse-url "http://example.com/")))

+36

(let ((wire (utf8->string (build-request-bytes 'POST url #{} "héllo"))))

+37

(assert-true (string-contains? wire "Content-Length: 6\r\n"))

+38

(assert-true (string-contains? wire "\r\n\r\nhéllo"))))))

+39

+40

(when (getenv "HTTP_UPLOAD_TEST_PORT")

+41

(test-group "binary upload network"

+42

(test "large binary body and empty body round-trip through HTTP"

+43

(let ((url (string-append "http://127.0.0.1:" (getenv "HTTP_UPLOAD_TEST_PORT") "/bytes")))

+44

(for-each (lambda (body)

+45

(let ((result (http-fetch-bytes 'PUT url headers: #{ content-type: "application/octet-stream" }

+46

body: body timeout: 5)))

+47

(assert-true result)

+48

(assert-equal (dict-ref result status:) 200)

+49

(assert-true (same-bytes? body (dict-ref result body:)))))

+50

(list (sample-bytes (* 1024 1024 2)) (make-bytevector 0)))))))