sell.py: optimal-ask model via empirical first-passage, not random walk

A driftless-random-walk closed form was tried first and backtested wrong
against 3 years of real exchange history (overstated touch probability
2-4x, confounded by discrete daily-VWAP sampling and real mean-reversion).
Replaced with puga.history.empirical_touch(): a non-parametric estimator
measuring first-passage directly from a ticker's own history, conditioned
on current state (trailing-60d z-score) since mean reversion's effect
depends on where price is now, not just move size. Queue-depth (existing
resting quantity) folds in as a deterministic shift of the hitting time,
not a max, ranking by the resulting hit_factor rather than a point
estimate of median days (which created a false cliff at the 50%-touched
threshold). Optional fixed-horizon right-censoring avoids recency bias
in "never reached" samples.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

puga/history.py

@@ -1,6 +1,6 @@
-"""Daily exchange price/volume series from FIO's cxpc candles, shared by tools/history.py and
+"""Daily exchange price/volume series from FIO's cxpc candles, shared by tools/history.py,
-tools/undervalued.py."""
+tools/undervalued.py, tools/sell.py and puga/parkcash.py."""
-import datetime
+import datetime, math, statistics
 from . import fio
 
 
@@ -11,3 +11,126 @@ def daily_prices(tk: str, cx: str) -> dict:
         if e.get("Interval") == "DAY_ONE" and e.get("Traded"):
             out[datetime.datetime.fromtimestamp(e["DateEpochMs"] / 1000, datetime.timezone.utc).date()] = (e["Volume"] / e["Traded"], e["Traded"])
     return out
+
+
+def trading_days(tk: str, cx: str, days: int = 30) -> list[tuple[datetime.date, float]]:
+    """(date, units traded) for actual trading days in the last `days` CALENDAR days - fio.cxpc
+    only emits a row when a trade happened, so a sparse ticker needs a real calendar cutoff here,
+    not "last N rows" (that would silently reach back far more than `days` actual days)."""
+    cutoff = datetime.datetime.now(datetime.timezone.utc).date() - datetime.timedelta(days=days)
+    return sorted((d, v) for d, (_, v) in daily_prices(tk, cx).items() if d >= cutoff)
+
+
+def trade_gaps_days(dates: list[datetime.date]) -> list[int]:
+    return [(b - a).days for a, b in zip(dates, dates[1:])]
+
+
+def full_series(tk: str, cx: str) -> list[tuple[datetime.date, float]]:
+    """Full traded-price history (all available days, not windowed) ascending by date - for the
+    empirical first-passage estimator below, which needs as much real history as exists rather
+    than a 30d window, especially for distant target prices."""
+    return sorted((d, p) for d, (p, _) in daily_prices(tk, cx).items())
+
+
+def _trailing_zscores(logp: list[float], trail: int = 60, min_window: int = 10) -> list[float | None]:
+    """z-score of each day's log-price against its own trailing `trail`-day mean/stdev (None where
+    there isn't `min_window` days of history yet). A flat window (stdev 0) scores 0, not None -
+    the price IS exactly at its own flat mean, that's a real, defined state, not missing data."""
+    z = [None] * len(logp)
+    for i in range(len(logp)):
+        window = logp[max(0, i - trail + 1):i + 1]
+        if len(window) < min_window:
+            continue
+        m = statistics.mean(window)
+        sd = statistics.stdev(window) if len(window) > 1 else 0.0
+        z[i] = (logp[i] - m) / sd if sd > 1e-12 else 0.0
+    return z
+
+
+def empirical_touch(series: list[tuple[datetime.date, float]], y: float, rate: float,
+                     condition_on_state: bool = True, trail: int = 60, z_tol: float = 0.5,
+                     horizon: int | None = None) -> dict:
+    """Non-parametric first-passage stats for a target log-move `y` = ln(target_price/start_price),
+    estimated directly from how often and how fast this ticker's REAL price history has made moves
+    of that size - no random-walk/GBM assumption, so real mean-reversion shows up directly in the
+    result instead of being assumed away (see docs/decisions.md for the evidence that real prices
+    here do mean-revert).
+
+    Mean reversion means the answer depends on WHERE price is now relative to its own recent
+    trend, not just on the move size - a 20% rise is more/less likely depending on whether today
+    is already above or below the recent mean. With `condition_on_state` (default), only historical
+    start days whose z-score against a trailing `trail`-day mean are within `z_tol` of TODAY's
+    z-score (the series' last point) are used - keeping the estimate non-parametric (no fitted
+    mean-reversion speed/strength) while still making it state-aware. Falls back to using every
+    start day if there isn't enough history to compute today's z-score, or if the z_tol band is
+    empty (too little data to condition on, not a real "always/never" signal).
+
+    For each eligible historical day i, scan forward for the first day j>i where the cumulative
+    log-return reaches y (>= y if y>0, <= y if y<0). Contributes e^(-rho*dt) to the discount-factor
+    average, where dt is the elapsed calendar days and rho = ln(1+rate); a day that NEVER reaches y
+    within the observed history contributes 0 (not excluded - excluding right-censored samples
+    would bias the estimate toward only the easy cases, since the hard/slow/never cases are exactly
+    the ones that get cut off by the data running out).
+
+    CAVEAT: `series` is daily VWAP (one point per day), a discrete sample of a continuous price
+    path. Discrete monitoring can only miss touches, never invent them - an intraday spike that
+    crosses the target and reverts before the next day's VWAP is computed shows up as "not
+    touched" here even though a resting ask would genuinely have filled on it. So this estimator
+    is systematically conservative: real touch probability/hit_factor is at least this high, likely
+    somewhat higher, and the resulting "sell now" lean (tools/sell.py --tiers) is a bit stronger
+    than fully warranted. Standard discrete-monitoring barrier corrections (e.g. Broadie-Glasserman-
+    Kou) put the gap at roughly 0.58*sigma*sqrt(step) per sampling interval for a short horizon -
+    real at short horizons, shrinking as the horizon grows. Not corrected for here.
+
+    Without `horizon`, "never reached" means "not reached before the series ends" - which gives
+    start days near the end of history much less time to ever register a hit than start days near
+    the beginning, purely from running out of data, not economics. With `horizon` (days), only
+    start days with at least that much real history remaining are used, and the search for day j
+    is capped at `horizon` days past i for all of them - every sample gets the same fixed window,
+    so touched_frac/hit_factor are comparable across samples instead of recency-biased. Off by
+    default (every caller currently omits it) since it discards the most recent `horizon` days of
+    start-day samples entirely - a real tradeoff, not a strict improvement on small histories.
+
+    Returns: hit_factor (empirical E[e^-rho*tau], the discounted-value multiplier), touched_frac
+    (share of start days that ever reached y at all within their remaining history), median_days
+    (None if touched_frac < 0.5 - can't report a median that was never reached more than half the
+    time), n (sample count - note: these are heavily OVERLAPPING windows, not independent draws,
+    so n overstates the effective sample size; same caveat as the backtest this reuses).
+    """
+    n = len(series)
+    if y == 0 or n < 2:
+        return {"hit_factor": 1.0, "touched_frac": 1.0, "median_days": 0.0, "n": n}
+    dates = [d for d, _ in series]
+    logp = [math.log(p) for _, p in series if p > 0]
+    rho = math.log(1 + rate)
+
+    eligible = range(n - 1)
+    if horizon is not None:
+        eligible = [i for i in eligible if (dates[-1] - dates[i]).days >= horizon]
+    if condition_on_state:
+        z = _trailing_zscores(logp, trail)
+        z_today = z[-1]
+        if z_today is not None:
+            matched = [i for i in eligible if z[i] is not None and abs(z[i] - z_today) <= z_tol]
+            if matched:
+                eligible = matched
+
+    discount_sum, hit_days, count = 0.0, [], 0
+    for i in eligible:
+        target = logp[i] + y
+        found = None
+        for j in range(i + 1, n):
+            if horizon is not None and (dates[j] - dates[i]).days > horizon:
+                break
+            if (y > 0 and logp[j] >= target) or (y < 0 and logp[j] <= target):
+                found = j
+                break
+        count += 1
+        if found is not None:
+            dt = (dates[found] - dates[i]).days
+            discount_sum += math.exp(-rho * dt)
+            hit_days.append(dt)
+    touched_frac = len(hit_days) / count if count else 0.0
+    median_days = statistics.median(hit_days) if touched_frac >= 0.5 else None
+    return {"hit_factor": discount_sum / count if count else 0.0, "touched_frac": touched_frac,
+            "median_days": median_days, "n": count}

tests/test_history.py

@@ -0,0 +1,72 @@
+import datetime, math
+from puga.history import empirical_touch, _trailing_zscores
+
+
+def _series(jump_idx, n=20, jump_price=220.0, base_price=100.0):
+    base = datetime.date(2026, 1, 1)
+    return [(base + datetime.timedelta(days=i), jump_price if i >= jump_idx else base_price) for i in range(n)]
+
+
+def test_hit_factor_has_no_cliff_at_the_50pct_touched_threshold():
+    """median_days flips from a number to None the instant touched_frac crosses 50% - that's
+    fine as a reported diagnostic, but ranking by it (as sell.py's --tiers used to) creates a
+    fake cliff: a tier touched 47% of the time would be valued at exactly 0, one touched 53% of
+    the time at near-full value, even though hit_factor (the real E[e^-rho*tau]) moves smoothly
+    between them. This is what tools/sell.py's --tiers ranks by instead."""
+    y = math.log(2.0)
+    below = empirical_touch(_series(jump_idx=9), y, rate=0.05, condition_on_state=False)
+    above = empirical_touch(_series(jump_idx=10), y, rate=0.05, condition_on_state=False)
+    assert below["median_days"] is None and above["median_days"] is not None
+    assert below["touched_frac"] < 0.5 <= above["touched_frac"]
+    # hit_factor must not have fallen off a cliff just because median_days did
+    assert below["hit_factor"] > 0.5 * above["hit_factor"]
+
+
+def test_empirical_touch_trivial_for_zero_move():
+    r = empirical_touch(_series(jump_idx=5), 0.0, rate=0.05)
+    assert r == {"hit_factor": 1.0, "touched_frac": 1.0, "median_days": 0.0, "n": 20}
+
+
+def test_conditioning_on_state_uses_only_similar_regime_days():
+    """Mean reversion means the answer should depend on whether today looks like the ticker's
+    readily-rising early regime or its since-flattened plateau - not blend both in unconditionally.
+    Series: 20 flat days, then a steady climb, then 10 more flat days at the new high (today is
+    in the plateau, not the climb)."""
+    base = datetime.date(2026, 1, 1)
+    prices = [100] * 20 + [100 + i * 5 for i in range(1, 11)] + [150] * 10
+    series = [(base + datetime.timedelta(days=i), float(p)) for i, p in enumerate(prices)]
+    y = math.log(1.1)
+    uncond = empirical_touch(series, y, rate=0.05, condition_on_state=False)
+    cond = empirical_touch(series, y, rate=0.05, condition_on_state=True)
+    assert uncond["touched_frac"] > 0.5          # blends in the climb regime, looks like it moves readily
+    assert cond["touched_frac"] < uncond["touched_frac"]   # today's plateau regime moves less
+
+
+def test_trailing_zscores_flat_window_is_zero_not_none():
+    logp = [math.log(100.0)] * 15
+    z = _trailing_zscores(logp, trail=60, min_window=10)
+    assert z[:9] == [None] * 9        # not enough window yet
+    assert z[9:] == [0.0] * 6         # flat price is exactly at its own mean - a real, defined state
+
+
+def test_horizon_removes_recency_bias_from_right_censoring():
+    """Without a fixed horizon, start days near the end of history get an unfairly short (or
+    long) remaining window to register a touch, purely from running out of data. A flat series
+    with a late jump looks like it "touches" readily only because the jump happens to sit right
+    next to the end-of-history start days; a fixed horizon gives every sample the same window."""
+    base = datetime.date(2026, 1, 1)
+    prices = [100.0] * 30 + [220.0] * 5
+    series = [(base + datetime.timedelta(days=i), p) for i, p in enumerate(prices)]
+    y = math.log(2.0)
+    uncapped = empirical_touch(series, y, rate=0.05, condition_on_state=False)
+    capped = empirical_touch(series, y, rate=0.05, condition_on_state=False, horizon=5)
+    assert capped["touched_frac"] < uncapped["touched_frac"]
+    assert capped["n"] < uncapped["n"]   # the last `horizon` days of start days are dropped
+
+
+def test_empirical_touch_never_reached_is_zero_not_excluded():
+    # a move far beyond anything the series ever does should look like "basically never"
+    r = empirical_touch(_series(jump_idx=19, jump_price=101.0), math.log(50.0), rate=0.05)
+    assert r["touched_frac"] == 0.0
+    assert r["hit_factor"] == 0.0
+    assert r["median_days"] is None

tools/sell.py

@@ -10,11 +10,27 @@ having cash in time) matters more than a few points of margin.
 
   puga sell BHP 8 --cx AI1
   puga sell BHP 11 --tight        # cash-tight: bias the split toward the aggressive tranche
+  puga sell BHP 8 --tiers         # is pricing above the current market ever worth the wait?
+
+--tiers answers "should I price above the current market" with TWO combined mechanisms, since
+they are separate necessary conditions, not alternatives to average:
+  - queue-depth days: existing resting quantity ahead of you has to actually sell, at the real
+    historical trade-volume rate - valid only WITHIN the recently-traded price range (extrapolating
+    it past that would make a fat-finger outlier ask look "reachable").
+  - empirical first-passage probability: for a price with no resting quantity (above the whole
+    book, or outside the traded range), the market's own price level has to actually move up to
+    it - estimated directly from how often/how fast THIS ticker's real multi-year history has made
+    moves of that size (puga.history.empirical_touch), not assumed from a random-walk formula
+    (real prices mean-revert; see docs/decisions.md).
+The two combine as: hit_factor (the empirical probability-and-timing-weighted value) shifted by
+queue_days as a fixed extra delay (exp(-rho*queue_days)), since clearing the queue and waiting for
+the price move are sequential, not alternative, requirements.
 """
-import argparse, datetime, statistics, sys
+import argparse, datetime, math, statistics, sys
 from pathlib import Path
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 from puga import config, fio, market
+from puga.history import trading_days, trade_gaps_days, daily_prices, full_series, empirical_touch
 
 
 def quantile(xs: list[float], q: float) -> float:
@@ -28,10 +44,24 @@ def tranche_split(qty: float, trades: list[float], tight: bool) -> tuple[float,
     return aggressive, qty - aggressive
 
 
-def daily_traded(tk: str, cx: str, days: int = 30) -> list[float]:
+def tier_table(prices: list[float], qty: float, s0: float, series: list, rate: float) -> list[dict]:
-    rows = [(datetime.datetime.fromtimestamp(e["DateEpochMs"] / 1000, datetime.timezone.utc).date(), e["Traded"])
+    """Discounted value of posting at each candidate price, using the EMPIRICAL first-passage
-            for e in fio.cxpc(tk, cx) if e.get("Interval") == "DAY_ONE" and e.get("Traded")]
+    distribution of this ticker's own real multi-year price history (puga.history.empirical_touch),
-    return [t for _, t in rows[-days:]]
+    not an assumed random-walk formula (real prices mean-revert; see docs/decisions.md). Has no
+    built-in monotonicity: if the real history shows diminishing willingness to move further, a
+    genuine interior optimum can appear instead of being forced by the model's shape.
+    """
+    rows = []
+    for p in prices:
+        y = math.log(p / s0) if s0 else 0.0
+        if y <= 0:
+            stats = {"hit_factor": 1.0, "touched_frac": 1.0, "median_days": 0.0, "n": len(series)}
+        else:
+            stats = empirical_touch(series, y, rate)
+        rows.append({"price": p, "y": y, "hit_factor": stats["hit_factor"], "touch_frac": stats["touched_frac"],
+                     "median_days": stats["median_days"], "n": stats["n"],
+                     "revenue": p * qty, "discounted": p * qty * stats["hit_factor"]})
+    return rows
 
 
 def main():
@@ -42,6 +72,8 @@ def main():
     ap.add_argument("--undercut", type=float, default=10, help="AIC to undercut the current best ask by")
     ap.add_argument("--tight", action="store_true", help="cash-tight: bias the tranche split toward the aggressive (undercut) tranche")
     ap.add_argument("--show-bid", action="store_true", help="also show instant-bid revenue (usually worse than the tranche split; off by default)")
+    ap.add_argument("--rate", type=float, default=0.05, help="your REALIZED daily capital growth rate (opportunity cost of waiting), not a theoretical build ROI - default 0.05 matches this account's observed equity-growth curve as of 2026-10-10; pass your own if it's moved")
+    ap.add_argument("--tiers", action="store_true", help="show discounted value (price*qty discounted at --rate over the implied wait) across all existing ask tiers, to check whether climbing the book is ever worth it")
     a = ap.parse_args()
     t = a.ticker.upper()
     ob = fio.order_book(t, a.cx)
@@ -50,17 +82,29 @@ def main():
     post = round((best_ask - a.undercut) if best_ask else (ob.get("Ask") or 0))
     ahead = sum(u for p, u in asks if p < post)
 
-    trades = daily_traded(t, a.cx)
+    days_window = 30
+    td = trading_days(t, a.cx, days_window)
+    trades = [v for _, v in td]
     med = statistics.median(trades) if trades else 0
     lo = sorted(trades)[int(0.2 * (len(trades) - 1))] if trades else 0
     hi = sorted(trades)[int(0.8 * (len(trades) - 1))] if trades else 0
+    trade_frac = len(td) / days_window
+    gaps = trade_gaps_days([d for d, _ in td])
+    median_gap = statistics.median(gaps) if gaps else (days_window if len(td) <= 1 else 0)
 
     print(f"{t}.{a.cx}  best ask {best_ask}  bid {ob.get('Bid')}  vwap7 {ob.get('PriceAverage')}")
     print(f"\npost {a.qty:g} at {post:.0f}  ({ahead:.0f} units ahead of you at a lower price)")
     print(f"  revenue if filled: {post * a.qty:,.0f}")
     if med:
-        print(f"  expected clear time (last {len(trades)}d volume): busy day {24 * a.qty / hi:.1f}h | "
+        print(f"  trades on {len(td)}/{days_window} of the last {days_window} days ({100*trade_frac:.0f}%)"
-              f"median day {24 * a.qty / med:.1f}h | slow day {24 * a.qty / max(lo, 1):.1f}h")
+              + (f", median gap between trading days: {median_gap:.0f}d" if gaps else ""))
+        if trade_frac < 0.5:
+            print(f"  SPARSE TICKER: most days see no trade at all - don't expect a same-day fill by default. "
+                  f"Once a trade happens, it tends to move: busy day {24 * a.qty / hi:.1f}h | median day {24 * a.qty / med:.1f}h | "
+                  f"slow day {24 * a.qty / max(lo, 1):.1f}h for your quantity - but you may wait ~{median_gap:.0f}d first for a trade to happen at all")
+        else:
+            print(f"  expected clear time (last {days_window}d volume): busy day {24 * a.qty / hi:.1f}h | "
+                  f"median day {24 * a.qty / med:.1f}h | slow day {24 * a.qty / max(lo, 1):.1f}h")
     else:
         print("  no recent trade history to estimate clear time")
 
@@ -70,12 +114,61 @@ def main():
               + (f", SHORT: book only fills {hit['filled']:.0f}" if hit["short"] else "") + ")")
         print(f"  posting patiently gains {post * a.qty - hit['total']:,.0f} over this, at the cost of waiting")
 
+    if a.tiers:
+        dp = daily_prices(t, a.cx)
+        s0 = sorted(dp.items())[-1][1][0] if dp else (best_ask or ob.get("Ask") or 0)
+        series = full_series(t, a.cx)
+        rho = math.log(1 + a.rate)
+        tier_prices = sorted({p for p, _ in asks} | {post, s0})
+        rows = tier_table(tier_prices, a.qty, s0, series, a.rate)
+
+        # Queue-depth days: valid ONLY within the range this ticker has genuinely traded in -
+        # the mechanism it models (real buy flow eating through resting ahead-quantity) has no
+        # evidence to extrapolate beyond that (this is what made a fat-finger outlier ask look
+        # reachable before). Outside that range it's infinite (no constraint from this mechanism;
+        # the empirical median_days alone governs).
+        cutoff_date = datetime.datetime.now(datetime.timezone.utc).date() - datetime.timedelta(days=days_window)
+        recent_prices = [p for d, (p, _) in dp.items() if d >= cutoff_date]
+        cutoff_price = max(recent_prices + [best_ask or 0]) * 1.1 if recent_prices else float("inf")
+        med_vol = med or 1
+        for r in rows:
+            ahead = sum(u for p, u in asks if p < r["price"])
+            # Outside the recent-traded range the queue mechanism doesn't meaningfully apply (no
+            # real evidence that cumulative ahead-quantity, much of it itself unreachable junk
+            # asks, clears at the normal rate) - that means "not a binding constraint" (0), NOT
+            # "infinite constraint".
+            r["queue_days"] = (ahead + a.qty) / med_vol if r["price"] <= cutoff_price else 0.0
+            # Rank by hit_factor (the real empirical E[e^-rho*tau] from tier_table), with
+            # queue_days folded in as a FIXED, deterministic shift of tau: shifting a random delay
+            # by a constant multiplies its Laplace transform by exp(-rho*shift), which is exactly
+            # right when "clear the real resting queue, THEN wait for price to reach this tier"
+            # are sequential, not alternative, constraints. median_days (shown separately below)
+            # is not used for the value itself - plugging a point estimate into (1+rate)^-days
+            # would be biased by Jensen's inequality, and None (<50% touched) would wrongly zero
+            # out a tier that hit_factor still values smoothly.
+            r["combined_hit"] = r["hit_factor"] * math.exp(-rho * r["queue_days"])
+            r["discounted_combined"] = r["revenue"] * r["combined_hit"]
+
+        best = max(rows, key=lambda r: r["discounted_combined"]) if rows else None
+        print(f"\nTIER TABLE (last traded {s0:.0f}, {len(series)}d history, rate {100*a.rate:.1f}%/day - see --help):")
+        print(f"  {'price':>10} {'queue(d)':>9} {'touched':>8} {'median(d)':>10} {'hit%':>6} {'revenue':>10} {'discounted':>11}")
+        for r in rows:
+            flag = "  <- best" if best is r else ""
+            qd = f"{r['queue_days']:.1f}" if r["queue_days"] != float("inf") else "n/a"
+            md = f"{r['median_days']:.1f}" if r["median_days"] is not None else "never"
+            print(f"  {r['price']:>10.0f} {qd:>9} {100*r['touch_frac']:>7.0f}% {md:>10} {100*r['combined_hit']:>5.1f}% "
+                  f"{r['revenue']:>10,.0f} {r['discounted_combined']:>11,.0f}{flag}")
+        if best and best["price"] > s0:
+            print(f"  climbing to {best['price']:.0f} beats selling now")
+        else:
+            print(f"  selling at/near the current price ({s0:.0f}) dominates")
+
     aggr_qty, patient_qty = tranche_split(a.qty, trades, a.tight)
     if patient_qty > 0.01:
         tiers_above = sorted({p for p, u in asks if p > post})
         patient_price = round(tiers_above[0] - 1) if tiers_above else round(post + 2 * a.undercut)
         q_label = "80th pct (tight)" if a.tight else "median"
-        print(f"\nTRANCHE SPLIT ({q_label} volume quantile, {len(trades)}d):")
+        print(f"\nTRANCHE SPLIT ({q_label} volume quantile, {len(trades)} trading days in the last {days_window}d):")
         print(f"  aggressive: {aggr_qty:.1f} @ {post:.0f}  (confident to clear today)")
         print(f"  patient:    {patient_qty:.1f} @ {patient_price:.0f}  (waits behind the next tier, higher margin)")
         print(f"  vs posting all {a.qty:g} at {post:.0f}: gains {patient_qty*(patient_price-post):,.0f} extra if the patient tranche fills")