First Commit

detections/tests/suricata/.gitignore

@@ -0,0 +1,5 @@
+# scratch captures created by run.sh (binary pcaps); never committed.
+# Match both files and directories (.scratch.* , not .scratch.*/) so that any
+# leftover .scratch.<name> is ignored even when an interrupted run (SIGKILL,
+# power loss) skips the EXIT cleanup, and so `git check-ignore` reports it.
+.scratch.*

detections/tests/suricata/gen_pcap.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Deterministic pcap generator for the ARTEX Suricata rule tests.
+
+Synthesizes N independent plaintext HTTP request/response flows from a single
+source, each carrying a chosen User-Agent, so `suricata -r` can be run offline
+to prove the rules in ../../suricata/artex.rules fire (or stay silent) exactly
+as documented. Output is regenerated on every run and is never committed -- the
+test ships as source, not as a binary capture.
+
+Usage:
+    gen_pcap.py <out.pcap> <user-agent> [num_flows] [interval_seconds]
+
+The capture is fully deterministic: fixed addresses, ports derived from the
+flow index, a fixed base timestamp, and flows spaced `interval_seconds` apart.
+Nothing here sends a packet or touches a network -- it only writes a file.
+"""
+import sys
+
+from scapy.all import Ether, IP, TCP, Raw, wrpcap
+
+# Fixed, private, non-routable endpoints. One source so Suricata's
+# `detection_filter ... track by_src` on sid 1000002 counts per source.
+SRC_MAC = "02:00:00:00:00:01"
+DST_MAC = "02:00:00:00:00:02"
+SRC_IP = "10.10.10.9"
+DST_IP = "10.10.10.80"
+DST_PORT = 80
+BASE_EPOCH = 1_760_000_000.0  # fixed so timestamps never depend on wall clock
+CLIENT_ISN = 1000
+SERVER_ISN = 2000
+
+
+def http_request(user_agent: str) -> bytes:
+    return (
+        "GET /products?category=all HTTP/1.1\r\n"
+        "Host: shop.example.test\r\n"
+        f"User-Agent: {user_agent}\r\n"
+        "Accept: */*\r\n"
+        "Connection: close\r\n"
+        "\r\n"
+    ).encode()
+
+
+HTTP_RESPONSE = (
+    "HTTP/1.1 200 OK\r\n"
+    "Content-Type: text/html\r\n"
+    "Content-Length: 13\r\n"
+    "Connection: close\r\n"
+    "\r\n"
+    "<html></html>"
+).encode()
+
+
+def flow(index: int, user_agent: str, t0: float):
+    """One complete TCP+HTTP conversation; returns a list of timestamped packets."""
+    sport = 40000 + index
+    eth_c = Ether(src=SRC_MAC, dst=DST_MAC)
+    eth_s = Ether(src=DST_MAC, dst=SRC_MAC)
+    ip_c = IP(src=SRC_IP, dst=DST_IP)
+    ip_s = IP(src=DST_IP, dst=SRC_IP)
+
+    req = http_request(user_agent)
+    rlen = len(req)
+    slen = len(HTTP_RESPONSE)
+
+    pkts = []
+
+    def add(pkt, offset):
+        pkt.time = t0 + offset
+        pkts.append(pkt)
+
+    # Handshake
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="S", seq=CLIENT_ISN), 0.000)
+    add(eth_s / ip_s / TCP(sport=DST_PORT, dport=sport, flags="SA", seq=SERVER_ISN, ack=CLIENT_ISN + 1), 0.001)
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="A", seq=CLIENT_ISN + 1, ack=SERVER_ISN + 1), 0.002)
+    # Request
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="PA", seq=CLIENT_ISN + 1, ack=SERVER_ISN + 1) / Raw(req), 0.003)
+    add(eth_s / ip_s / TCP(sport=DST_PORT, dport=sport, flags="A", seq=SERVER_ISN + 1, ack=CLIENT_ISN + 1 + rlen), 0.004)
+    # Response
+    add(eth_s / ip_s / TCP(sport=DST_PORT, dport=sport, flags="PA", seq=SERVER_ISN + 1, ack=CLIENT_ISN + 1 + rlen) / Raw(HTTP_RESPONSE), 0.005)
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="A", seq=CLIENT_ISN + 1 + rlen, ack=SERVER_ISN + 1 + slen), 0.006)
+    # Teardown
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="FA", seq=CLIENT_ISN + 1 + rlen, ack=SERVER_ISN + 1 + slen), 0.007)
+    add(eth_s / ip_s / TCP(sport=DST_PORT, dport=sport, flags="A", seq=SERVER_ISN + 1 + slen, ack=CLIENT_ISN + 2 + rlen), 0.008)
+    add(eth_s / ip_s / TCP(sport=DST_PORT, dport=sport, flags="FA", seq=SERVER_ISN + 1 + slen, ack=CLIENT_ISN + 2 + rlen), 0.009)
+    add(eth_c / ip_c / TCP(sport=sport, dport=DST_PORT, flags="A", seq=CLIENT_ISN + 2 + rlen, ack=SERVER_ISN + 2 + slen), 0.010)
+    return pkts
+
+
+def main() -> int:
+    if len(sys.argv) < 3:
+        print(__doc__)
+        return 2
+    out = sys.argv[1]
+    user_agent = sys.argv[2]
+    num_flows = int(sys.argv[3]) if len(sys.argv) > 3 else 35
+    interval = float(sys.argv[4]) if len(sys.argv) > 4 else 1.0
+
+    packets = []
+    for i in range(num_flows):
+        packets.extend(flow(i, user_agent, BASE_EPOCH + i * interval))
+
+    wrpcap(out, packets)
+    print(f"wrote {len(packets)} packets across {num_flows} flows to {out} (UA={user_agent!r})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

detections/tests/suricata/run.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+#
+# Reproducible regression test for the ARTEX Suricata rules
+# (../../suricata/artex.rules). It proves four properties with no committed
+# binary capture and no host dependencies beyond Docker:
+#
+#   1. the whole rules file loads with zero errors       (validity)
+#      `suricata -T --init-errors-fatal`; a rule that fails to parse or
+#      initialise is fatal even when no capture below exercises it
+#   2. sid 1000001 fires exactly once per enrich probe   (presence)
+#   3. sid 1000002 fires once the 30-in-300s rate is hit (velocity)
+#   4. sid 1000003 fires once per norma WebFetch request (presence)
+#      and the enrich sids stay silent on that capture   (specificity)
+#   5. an identical capture with a benign browser UA     (specificity)
+#      produces zero alerts
+#
+# Everything runs in containers: `suricata -T` validates the ruleset, scapy
+# synthesizes a deterministic pcap, then `suricata -r` reads it offline. The
+# pcap is generated into a scratch dir that is removed on exit and is never
+# committed.
+#
+# Usage:   detections/tests/suricata/run.sh
+# Env:     SURICATA_IMAGE (default jasonish/suricata:latest)
+#          PYTHON_IMAGE   (default python:3.12-slim)
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$HERE/../../.." && pwd)"
+RULES_DIR="$REPO/detections/suricata"
+SURICATA_IMAGE="${SURICATA_IMAGE:-jasonish/suricata:latest}"
+PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.12-slim}"
+
+NUM_FLOWS=35
+ENRICH_UA="artex-enrich/1.0"
+# norma SDK WebFetch tool, hardcoded in github.com/Autumn-27/norma/tool/webfetch.go
+# (literal "norma/0.4", verified in the go.sum-pinned v0.4.3 module source). Fewer flows
+# than NUM_FLOWS because sid 1000003 is a single-hit presence rule with no rate component.
+NORMA_UA="norma/0.4"
+NORMA_FLOWS=8
+BENIGN_UA="Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
+
+# Scratch must live under the repo tree so Docker Desktop (macOS) can bind-mount
+# it; /tmp and $TMPDIR are not shared by default. It is git-ignored and removed
+# on exit.
+SCRATCH="$(mktemp -d "$HERE/.scratch.XXXXXX")"
+cleanup() { rm -rf "$SCRATCH"; }
+trap cleanup EXIT
+
+fail=0
+note() { printf '  %s\n' "$1"; }
+
+echo "== 1/5  validate the full ruleset loads (suricata -T) =="
+# `suricata -T` loads the whole rules file in test mode and exits; --init-errors-fatal
+# makes any rule that fails to parse or initialise a hard error. This catches a broken
+# rule even when no capture below exercises it: plain `suricata -r` skips such a rule and
+# still exits 0, so the firing checks would stay green while a signature silently fails to
+# load. This is the Suricata analogue of the Sigma suite's `sigma check` validity assertion.
+if docker run --rm -v "$RULES_DIR:/r:ro" "$SURICATA_IMAGE" \
+     suricata -T -S /r/artex.rules -l /tmp --init-errors-fatal >/dev/null 2>&1; then
+  note "PASS  ruleset loads with zero parse/init errors (suricata -T)"
+else
+  note "FAIL  ruleset loads with zero parse/init errors (suricata -T)"
+  fail=1
+fi
+
+echo "== 2/5  synthesize deterministic captures (scapy) =="
+docker run --rm -v "$SCRATCH:/out" -v "$HERE:/src:ro" "$PYTHON_IMAGE" sh -c "
+  pip install --quiet --disable-pip-version-check scapy >/dev/null 2>&1 &&
+  python /src/gen_pcap.py /out/enrich.pcap '$ENRICH_UA' $NUM_FLOWS &&
+  python /src/gen_pcap.py /out/norma.pcap  '$NORMA_UA'  $NORMA_FLOWS &&
+  python /src/gen_pcap.py /out/benign.pcap '$BENIGN_UA' $NUM_FLOWS
+"
+
+run_suricata() { # $1 = capture basename
+  local name="$1"
+  mkdir -p "$SCRATCH/$name-out"
+  # -k none: crafted packets carry no valid checksums; do not drop on them.
+  docker run --rm -v "$SCRATCH:/data" -v "$RULES_DIR:/r:ro" "$SURICATA_IMAGE" \
+    suricata -r "/data/$name.pcap" -S /r/artex.rules -k none -l "/data/$name-out" \
+    >/dev/null 2>&1
+}
+
+alerts() { # $1 = capture basename, $2 = sid (or "any")
+  python3 - "$SCRATCH/$1-out/eve.json" "$2" <<'PY'
+import json, sys
+path, sid = sys.argv[1], sys.argv[2]
+n = 0
+with open(path) as f:
+    for line in f:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            e = json.loads(line)
+        except ValueError:
+            continue
+        if e.get("event_type") != "alert":
+            continue
+        if sid == "any" or e.get("alert", {}).get("signature_id") == int(sid):
+            n += 1
+print(n)
+PY
+}
+
+expect() { # $1 label, $2 actual, $3 op (eq|ge), $4 expected
+  local label="$1" actual="$2" op="$3" expected="$4" ok
+  case "$op" in
+    eq) [ "$actual" -eq "$expected" ] && ok=1 || ok=0 ;;
+    ge) [ "$actual" -ge "$expected" ] && ok=1 || ok=0 ;;
+  esac
+  if [ "$ok" -eq 1 ]; then
+    note "PASS  $label  (got $actual, want $op $expected)"
+  else
+    note "FAIL  $label  (got $actual, want $op $expected)"
+    fail=1
+  fi
+}
+
+echo "== 3/5  run Suricata offline over the enrich capture =="
+run_suricata enrich
+e1="$(alerts enrich 1000001)"
+e2="$(alerts enrich 1000002)"
+expect "sid 1000001 presence: one alert per probe" "$e1" eq "$NUM_FLOWS"
+expect "sid 1000002 velocity: fires past 30-in-300s" "$e2" ge 1
+note "reference (Suricata 8.0.7): sid 1000002 = 5 (flows 31-35)"
+
+echo "== 4/5  run Suricata offline over the norma WebFetch capture =="
+run_suricata norma
+n3="$(alerts norma 1000003)"
+nenrich="$(( $(alerts norma 1000001) + $(alerts norma 1000002) ))"
+expect "sid 1000003 presence: one alert per WebFetch request" "$n3" eq "$NORMA_FLOWS"
+expect "enrich sids stay silent on norma traffic (specificity)" "$nenrich" eq 0
+
+echo "== 5/5  run Suricata offline over the benign capture =="
+run_suricata benign
+b="$(alerts benign any)"
+expect "benign browser UA produces no ARTEX alerts" "$b" eq 0
+
+echo
+if [ "$fail" -eq 0 ]; then
+  echo "RESULT: PASS"
+else
+  echo "RESULT: FAIL"
+fi
+exit "$fail"