CSV, stats & charts

-- Doc example: CSV parsing, descriptive statistics and native SVG charts (all pure).
intent: "doc example: csv, stats and charts"

let raw be "month,total\njan,10\nfeb,25\nmar,17\napr,25\n"
let rows be csv_parse(raw, {"types": {"total": "int"}})
let svg be chart_svg("bar", rows, {"x": "month", "y": "total", "title": "Sales"})
print("rows = " + text(length(rows)) + ",  median = " + text(median([10, 25, 17, 25])) + ",  svg = " + text(starts_with(svg, "<svg")))

test "csv_parse: rows are maps (same shape sql() returns), lossless by default"
    let plain be csv_parse("a,b\n00123,x\n")
    assert_eq(plain[0]["a"], "00123")
    assert_eq(type_of(plain[0]["a"]), "text")
    assert_eq(rows[1]["total"], 25)

test "csv round-trip survives embedded commas and quotes"
    let tricky be [{"name": "a,b", "note": "say \"hi\""}]
    assert_eq(csv_parse(csv_encode(tricky)), tricky)

test "median / percentile / histogram (NumPy semantics)"
    assert_eq(median([1, 2, 3, 4]), 2.5)
    assert_eq(percentile([1, 2, 3, 4], 25), 1.75)
    let h be histogram(range(10), 5)
    assert_eq(h["counts"], [2, 2, 2, 2, 2])
    assert_eq(length(h["edges"]), 6)

test "chart_svg is deterministic and escapes data text"
    assert(contains(svg, "<svg"))
    assert_eq(svg, chart_svg("bar", rows, {"x": "month", "y": "total", "title": "Sales"}))
    let hostile be chart_svg("pie", {"<b>x</b>": 4, "y": 2})
    assert(contains(hostile, "&lt;b&gt;"))

test "chart errors are clear and catchable"
    let msg be ""
    try
        chart_svg("bar", [])
    recover e
        set msg to e
    assert(contains(msg, "data is empty"))

test "every chart option, exercised: title/x_label/y_label/legend/colors/width/height/background"
    let multi be [{"m": "a", "sales": 10, "costs": 6}, {"m": "b", "sales": 25, "costs": 12}]
    let full be chart_svg("bar", multi, {
        "x": "m", "y": ["sales", "costs"],
        "title": "Sales vs costs", "x_label": "Month", "y_label": "USD",
        "legend": true, "colors": ["#112233", "#445566"],
        "width": 400, "height": 300, "background": "#fcfcfb"
    })
    assert(contains(full, "Sales vs costs"))
    assert(contains(full, "Month") and contains(full, "USD"))
    assert(contains(full, "sales") and contains(full, "costs"))
    assert(contains(full, "#112233") and contains(full, "#445566"))
    assert(contains(full, "viewBox=\"0 0 400 300\""))
    -- legend: false hides the series names; then the other data shapes:
    assert(contains(chart_svg("line", multi, {"x": "m", "y": "sales", "legend": false}), ">sales<") == false)
    assert(contains(chart_svg("pie", {"a": 3, "b": 1}), "<path"))
    assert(contains(chart_svg("line", [1, 2, 3]), "<polyline"))
    assert(contains(chart_svg("scatter", [[1, 2], [3, 4]]), "<circle"))

test "csv options, exercised: headers/delimiter/numbers/eol"
    let semi be csv_parse("a;b\n1;2\n", {"delimiter": ";", "numbers": true})
    assert_eq(semi[0]["b"], 2)
    let positional be csv_parse("1,2\n3,4\n", {"headers": false})
    assert_eq(positional[1][0], "3")
    assert_eq(csv_encode([{"a": "1", "b": "2"}], {"headers": ["b"], "eol": "\n"}), "b\n2\n")

test "histogram bins: integer or explicit edges (last bin closed, out-of-range dropped)"
    assert_eq(histogram([1, 2, 3, 4], [0, 2, 4])["counts"], [1, 3])
    assert_eq(histogram([-5, 1, 99], [0, 2])["counts"], [1])

test "business kinds: area/heatmap/histogram/boxplot/donut/waterfall + stack"
    let multi be [{"m": "a", "sales": 10, "costs": 6}, {"m": "b", "sales": 25, "costs": 12}]
    -- stack is an OPT on bar/area (there is no "stacked_bar" kind)
    assert(contains(chart_svg("bar", multi, {"x": "m", "y": ["sales", "costs"], "stack": true}), "<rect"))
    assert(contains(chart_svg("area", multi, {"x": "m", "y": ["sales", "costs"], "stack": true}), "<polygon"))
    -- heatmap: tidy rows (x/y/value) or a matrix (+ x_labels/y_labels); scale auto/sequential/diverging
    let cells be [{"d": "mon", "h": "9", "v": 1}, {"d": "mon", "h": "10", "v": 5}, {"d": "tue", "h": "9", "v": 9}]
    assert(contains(chart_svg("heatmap", cells, {"x": "h", "y": "d", "value": "v"}), "<rect"))
    let diverging be chart_svg("heatmap", [[-8, 0], [4, 9]], {"x_labels": ["q1", "q2"], "y_labels": ["a", "b"], "scale": "diverging", "center": 0})
    assert(contains(diverging, "<rect"))
    -- histogram plots raw numbers OR the exact map histogram() returns (same binning)
    let data be [1, 2, 2, 3, 3, 3, 9]
    assert_eq(chart_svg("histogram", data, {"bins": 4}), chart_svg("histogram", histogram(data, 4)))
    -- boxplot: >= 2 values per group; quartiles match percentile()
    assert(contains(chart_svg("boxplot", {"web": [1, 2, 3, 4, 100], "tel": [2, 3, 8, 9]}), "<circle"))
    -- donut = pie with a hole (same rules); waterfall takes DELTAS, total is opt-in
    assert(contains(chart_svg("donut", {"a": 4, "b": 2}), "<path"))
    assert(contains(chart_svg("waterfall", {"sales": 100, "costs": -40}, {"total": true}), ">Total<"))

test "theme dark + kind/opt validation errors are clear and catchable"
    let multi be [{"m": "a", "sales": 10, "costs": 6}, {"m": "b", "sales": 25, "costs": 12}]
    let dark be chart_svg("bar", multi, {"x": "m", "y": "sales", "theme": "dark"})
    assert(contains(dark, "<svg"))
    assert(dark != chart_svg("bar", multi, {"x": "m", "y": "sales"}))
    let msg be ""
    try
        chart_svg("treemap", [1])
    recover e
        set msg to e
    assert(contains(msg, "valid kinds are: area, bar, boxplot, donut, heatmap, histogram, line, pie, scatter, waterfall"))
    set msg to ""
    try
        chart_svg("pie", {"a": 1}, {"stack": true})
    recover e
        set msg to e
    assert(contains(msg, "bar, area"))
    set msg to ""
    try
        chart_svg("heatmap", [[1]], {"center": 5})
    recover e
        set msg to e
    assert(contains(msg, "diverging"))

test "svg_to_png / svg_to_pdf: every option, real deterministic bytes"
    let png be svg_to_png(svg, {"scale": 2})
    assert(contains(decode(png, "hex"), "89504e470d0a1a0a"))
    assert_eq(png, svg_to_png(svg, {"scale": 2}))
    let sized be svg_to_png(svg, {"width": 320, "background": "#ffffff", "max_pixels": 1000000})
    assert(contains(decode(sized, "hex"), "89504e470d0a1a0a"))
    let pdf be svg_to_pdf(svg, {"width": 400})
    assert(contains(decode(pdf, "hex"), "255044462d"))
    -- the anti-DoS ceiling names its option:
    let msg be ""
    try
        svg_to_png(svg, {"width": 10000000})
    recover e
        set msg to e
    assert(contains(msg, "max_pixels"))

test "stats: percentile 0..100, quantile 0..1, std sample, nothing skipped, NaN propagates"
    assert_eq(quantile([1, 2, 3, 4], 0.25), 1.75)
    assert_eq(percentile([1, 2, 3, 4], 50), quantile([1, 2, 3, 4], 0.5))
    assert_error(() => percentile([1, 2], 101))
    assert_error(() => quantile([1, 2], 50))
    assert_eq(std([1, 2, 3, 4]), 1.2909944487358056)
    assert_eq(std([1, 2, 3, 4], ddof = 0), 1.118033988749895)
    assert_eq(median([1, nothing, 3]), 2.0)
    assert(is_nan(median([1, nan, 3])))
    assert_error(() => histogram([1, nothing, 3]))                       -- histogram takes numbers only
    assert_error(() => histogram([1, nan, 3]))
    assert_error(() => histogram([]))

test "charts take summarize output; a nothing in a numeric field is an error"
    let sales be [{"r": "n", "v": 10}, {"r": "s", "v": 5}, {"r": "n", "v": 1}]
    let by_r be summarize(sales, "r", {"total": sum_of("v")})
    assert(contains(chart_svg("bar", by_r, {"x": "r", "y": "total"}), "<rect"))
    assert_error(() => chart_svg("bar", [{"x": "a", "y": nothing}], {"x": "x", "y": "y"}))
    assert(contains(chart_svg("bar", fill_missing([{"x": "a", "y": nothing}], 0), {"x": "x", "y": "y"}), "<svg"))

-- The message of an error, or nothing when the call did not fail.
task error_of(f)
    try
        f()
    recover e
        give e
    give nothing

test "x_axis rotates, thins and formats the category labels (engine v0.6.45+)"
    let days be apply((i) => {"day": add_days(date(2026, 1, 1), i), "n": i}, range(0, 30))
    let c be chart_svg("bar", days, {"x": "day", "y": "n", "x_axis": {"angle": 30, "every": 7, "format": "%d/%m"}})
    assert(contains(c, ">08/01<"))              -- every 7th label, formatted with strftime
    assert(not contains(c, ">02/01<"))          -- the ones in between are skipped
    assert(contains(c, "rotate(-30 "))
    -- without x_axis the SVG is byte for byte the one it always was
    assert_eq(svg, chart_svg("bar", rows, {"x": "month", "y": "total", "title": "Sales"}))
    assert(contains(error_of(() => chart_svg("pie", {"a": 1}, {"x_axis": {"angle": 45}})), "only applies to: bar, line, area, heatmap, boxplot, waterfall"))
    assert(contains(error_of(() => chart_svg("bar", rows, {"x": "month", "y": "total", "x_axis": {"angel": 45}})), "unknown x_axis key \"angel\" (valid: angle, every, format)"))

test "qr_matrix / qr_svg encode a QR code (ISO/IEC 18004)"
    let m be qr_matrix("https://synsema.dev")
    assert_eq(length(m), length(m[0]))           -- square: rows of 0/1
    assert_eq(m[0][0], 1)                        -- the finder pattern's corner
    assert_eq(length(qr_matrix("hi", {"version": 5})), 37)   -- version v is 17 + 4v modules a side
    let q be qr_svg("hi", {"ecc": "H", "module": 4, "margin": 4, "dark": "#112233", "light": "#ffffff"})
    assert(starts_with(q, "<svg"))
    assert(contains(q, "#112233"))
    assert_eq(q, qr_svg("hi", {"ecc": "H", "module": 4, "margin": 4, "dark": "#112233", "light": "#ffffff"}))
    assert_eq(error_of(() => qr_svg("hi", {"ecc": "X"})), "qr_svg: ecc must be \"L\", \"M\", \"Q\" or \"H\", got \"X\"")
    assert(contains(error_of(() => qr_svg("hi", {"size": 4})), "valid: ecc, version, module, margin, dark, light"))
    -- the SVG converts like any other
    assert(contains(decode(svg_to_png(q), "hex"), "89504e470d0a1a0a"))

test "svg_to_png refuses a <text> over 16 384 characters unless you raise max_text_chars"
    let long be join(apply((i) => "a", range(0, 20000)), "")
    let doc be `<svg xmlns="http://www.w3.org/2000/svg" width="100" height="20"><text x="0" y="15">{long}</text></svg>`
    let e be error_of(() => svg_to_png(doc))
    assert(contains(e, "a <text> has 20000 characters, above the 16384 character limit"))
    assert(contains(e, "max_text_chars"))
    assert_eq(type_of(svg_to_png(doc, {"max_text_chars": 20000})), "bytes")