\documentclass[11pt,a4paper]{article}

% ── Packages ──────────────────────────────────────────────────────────────────
\usepackage[margin=1in]{geometry}
\usepackage{times}
\usepackage{amsmath,amssymb}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{enumitem}
\usepackage{float}
\usepackage{caption}
\usepackage{natbib}
\usepackage{url}

\definecolor{sfgreen}{HTML}{10B981}
\hypersetup{
    colorlinks=true,
    linkcolor=sfgreen,
    citecolor=sfgreen,
    urlcolor=sfgreen,
}

% ── Title ─────────────────────────────────────────────────────────────────────
\title{\textbf{Feature-Based Prediction-Market Forecasting:\\Preliminary Observations}}

\author{
    Patrick Liu \\
    SimpleFunctions \\
    \texttt{patrick@simplefunctions.dev} \\
    \url{https://simplefunctions.dev}
}

\date{April 2026}

\begin{document}
\maketitle

% ── Abstract ──────────────────────────────────────────────────────────────────
\begin{abstract}
We release \textbf{sf-ml-baseline v0.1}, the first publicly documented feature-based forecasting baseline for prediction markets. On eleven days of SimpleFunctions microstructure data (1.76M labelled rows for 24-hour direction, 14K for resolution) and five engineered features---mid price, one-hour delta, implied yield, cancel--replace intensity, and cancel-versus-volume ratio---a three-seed LightGBM ensemble achieves \textbf{Brier 0.2294} on a held-out 246K-row test set for the 24-hour direction task, compared to \textbf{0.2500} for a coinflip (bootstrapped 95\% CI non-overlap; improvement $-0.0206$). An XGBoost/CatBoost bake-off converges to the same Brier, indicating classical saturation on a five-feature corpus. On the resolution task, per-category LightGBM models beat the ``price divided by 100'' baseline by 0.035 to 0.041 Brier on Crypto and Commodities. We frame this as a starting point for microstructure-based forecasting, \emph{not} as a competitor to LLM+RAG systems such as \citet{halawi2024} or AIA~\citep{aia2025}; subsequent versions will be \emph{ensembled with} those approaches, not substituted for them. Weights, code, and training scripts are released under CC-BY-4.0 with a SimpleFunctions attribution addendum at \url{https://huggingface.co/SimpleFunctions/sf-ml-baseline}.
\end{abstract}

% ── 1. Introduction ───────────────────────────────────────────────────────────
\section{Introduction}

Forecasting prediction-market outcomes from historical data is well-studied on the \emph{crowd-psychology} side---Tetlock~\citep{tetlock2005}, Manski~\citep{manski2006}, Wolfers and Zitzewitz~\citep{wolfers2004}---and well-studied on the \emph{language-model} side---Halawi et al.~\citep{halawi2024} and the AI Agent Alliance~\citep{aia2025} both train LLMs on Metaculus-style long-horizon questions. What is \emph{not} documented in the open literature is a feature-based baseline that uses the \emph{microstructure} of prediction markets: order flow, cancel--replace dynamics, cross-venue gaps, and derived yield and regime features.

SimpleFunctions (SF) runs a real-time pipeline that computes 27 indicator columns per ticker on every Kalshi and Polymarket market. This note documents a preliminary attempt to use a small subset of those indicators as features in gradient-boosted trees. The framing is deliberately narrow. We are not trying to beat LLM+RAG systems that ingest news. We are trying to answer a simpler question: \emph{does the SF microstructure feature set contain predictive signal, and how much?}

The short answer is \textbf{yes, the signal exists and is statistically significant, but the ceiling on a five-feature classical-tree stack appears to be very close to what our v0.1 bake-off reached}. We release the weights, document the gap, and lay out a six-phase roadmap (\S\ref{sec:roadmap}) for what would plausibly close it.

% ── 2. Data ──────────────────────────────────────────────────────────────────
\section{Data and Features}

\paragraph{Training window.} SF's \texttt{market\_indicator\_history} table captures a snapshot row per market every few minutes. At the time of this note the table spans \textbf{eleven calendar days} (2026-04-08 to 2026-04-19), \textbf{6.79M rows}, and \textbf{1.87\,GB}. The continuous forward-only R2 data dump that will extend this archive started on 2026-04-18; retrains against 30-day, 90-day, and 180-day windows are scheduled in \S\ref{sec:roadmap}.

\paragraph{Features (V1).} We use five features in v0.1, chosen because they are (a) present on every snapshot row, (b) numerically clean after null handling, and (c) interpretable:
\begin{itemize}[leftmargin=*,nosep]
    \item \texttt{price\_cents}: mid-market price in cents.
    \item \texttt{delta\_cents}: signed one-hour price change.
    \item \texttt{iy} (implied yield): annualised yield on a hold-to-resolution trade at the current price.
    \item \texttt{cri} (cancel--replace intensity): per-minute count of cancel--replace pairs on the book.
    \item \texttt{cvr} (cancel-versus-volume ratio): ratio of cancelled volume to traded volume, windowed over the last hour.
\end{itemize}

The \texttt{market\_indicators} live-state table carries 27 columns; we trained on only 5 because that is the intersection with the \emph{historical} archive. The full 20-feature expansion is part of the v0.3 roadmap (\S\ref{sec:roadmap}).

\paragraph{Labels.} We evaluate two targets:
\begin{itemize}[leftmargin=*,nosep]
    \item \textbf{T1 (direction, 24h)}: $\mathrm{sign}(\mathrm{price}(t+24\mathrm{h}) - \mathrm{price}(t))$, binary. Available on every row that has a matching $t+24$h snapshot: \textbf{1.76M labelled rows}, \textbf{246K in the held-out test fold}.
    \item \textbf{T4 (resolution)}: \texttt{resolved\_outcome} from the \texttt{marketwide\_resolutions} table, binary. Available on \textbf{1.09M settled markets}, of which the feature join yields \textbf{14K labelled rows} with the full 5-feature snapshot at $t_0 - 24$h.
\end{itemize}

We also considered T3 (price at $t+24$h, regression) but defer it to v0.3: only 15.4\% of \texttt{marketwide\_resolutions} rows carry the \texttt{predicted\_price\_t24h} column (167K of 1.09M), because the column was added recently and is not backfilled. Training T3 on 15.4\% coverage would leak the very selection bias we are trying to measure.

\paragraph{Splits.} We use a temporal split: the first 80\% of timestamps form the training set, the last 20\% the test set, and a \textbf{24-hour embargo} is enforced between them. The embargo is a standard device from the Jane Street time-series competition~\citep{janestreet2022} and prevents label leakage through overlapping resolution windows.

% ── 3. Method ────────────────────────────────────────────────────────────────
\section{Method}

\paragraph{Models.} We trained three families in a bake-off:
\begin{enumerate}[leftmargin=*,nosep]
    \item \textbf{LightGBM}~\citep{ke2017lightgbm}, 3-seed ensemble (seeds 42, 137, 2026), each 500 rounds with early stopping on a 10\% validation slice.
    \item \textbf{XGBoost}~\citep{chen2016xgboost}, same 3-seed ensemble, same hyperparameter grid.
    \item \textbf{CatBoost}~\citep{prokhorenkova2018catboost}, same 3-seed ensemble.
\end{enumerate}

\paragraph{Hyperparameter search.} We used a fixed shallow grid (depth $\in \{4,6,8\}$, learning rate $\in \{0.03, 0.05, 0.1\}$, L2 regularisation $\in \{0.1, 1.0\}$) because the training corpus is small relative to typical boosting literature and we observed no meaningful improvement beyond these defaults.

\paragraph{Metric.} Brier score~\citep{brier1950} is the primary metric:
\begin{equation}
    \mathrm{Brier} = \frac{1}{N}\sum_{i=1}^{N} (p_i - y_i)^2
\end{equation}
where $p_i \in [0,1]$ is the predicted probability and $y_i \in \{0,1\}$ is the realised outcome. Brier is a \emph{proper} scoring rule~\citep{gneiting2007strictly} and in our domain has the advantage that the three candidate baselines (coinflip, momentum, price/100) all evaluate on the same scale without a threshold choice. We report 95\% bootstrap confidence intervals (1000 resamples) for every Brier number and mark results as \emph{significant} only when the CI does not overlap the baseline's CI.

\paragraph{Baselines.} For T1 the headline baseline is a \textbf{coinflip} (constant 0.5). For T4 the headline baseline is \textbf{price divided by 100} at $t_0$, which is the market's own probability estimate on a YES contract that resolves in 24 hours.

% ── 4. Results ────────────────────────────────────────────────────────────────
\section{Results}

Table~\ref{tab:headline} reports the headline numbers.

\begin{table}[H]
\centering
\caption{Headline Brier scores. CI: 95\% bootstrap over 1000 resamples. $\Delta$ is (model $-$ baseline); negative is better.}
\label{tab:headline}
\begin{tabular}{lrrrc}
\toprule
Task & Brier (model) & Brier (baseline) & $\Delta$ & CI non-overlap? \\
\midrule
V1 $\times$ T1 (direction, $N_{\text{test}}$=246K) & \textbf{0.2294} & 0.2500 (coinflip) & $\mathbf{-0.0206}$ & \textbf{yes} \\
V2 $\times$ T4 (resolution, $N_{\text{test}}$=2.2K) & 0.1681 & 0.1767 (price/100) & $-0.0086$ & overlap \\
\quad T4 Crypto (per-category) & \textbf{0.0793} & 0.1207 & $\mathbf{-0.0414}$ & n/a \\
\quad T4 Commodities (per-category) & \textbf{0.0791} & 0.1145 & $\mathbf{-0.0355}$ & n/a \\
\bottomrule
\end{tabular}
\end{table}

Three observations:

\paragraph{1. The signal exists on T1, but it is modest.} On the direction task the 0.0206 Brier gap is statistically significant (CI non-overlap) and clears the $\geq 0.01$ improvement threshold we pre-registered before training. A skeptical reader should interpret this as ``the five-feature corpus carries measurable signal'' rather than ``the corpus is nearly complete.'' Uniformly predicting 0.5 scores 0.2500; a model that perfectly discriminated at the margin of a typical prediction market would score closer to 0.18--0.20.

\paragraph{2. Classical models saturate at the same Brier.} LightGBM, XGBoost, and CatBoost all converge to Brier within $\pm$0.0005 of each other on T1. This is the predictable behaviour of gradient-boosted trees on a small, highly correlated feature set: once one family has extracted the signal, the others cannot extract more \emph{from the same features}. We had expected Phase~B (the bake-off) to add 0.003--0.005 over Phase~A; in practice it added nothing. The implication is that the v0.1 corpus is feature-bound, not model-bound.

\paragraph{3. Per-category resolution models beat the market on liquid verticals.} On T4 the aggregate improvement over the price/100 baseline is not significant, but once we slice by category, Crypto and Commodities both show large and consistent gains (0.0414 and 0.0355 Brier respectively). This matches the intuition that markets with continuous underliers (BTC, oil, gold) leave more microstructure-recoverable alpha on the table than idiosyncratic binary markets (single legislation, single court rulings). Released weights include per-category LightGBM heads.

\paragraph{Calibration.} Reliability diagrams (not shown; see the companion HuggingFace release) confirm the T1 ensemble is well-calibrated in the mid-range (0.35 to 0.65) and slightly overconfident at the tails, consistent with typical LightGBM behaviour. Temperature scaling~\citep{guo2017calibration} is included in the roadmap for v0.2.

% ── 5. Prior work ────────────────────────────────────────────────────────────
\section{Related Work}

\paragraph{LLM-based forecasting.} \citet{halawi2024} trained GPT-4 on Metaculus questions and achieved Brier 0.179 against a crowd baseline of 0.149 (the crowd won). \citet{aia2025} extended this approach; on liquid Polymarket markets the system also fails to beat the market. Our work is complementary, not competitive: our release operates on microstructure (price dynamics, order flow), theirs operates on news text and reasoning. The obvious next step is an ensemble, which we flag as future work.

\paragraph{Tabular gradient boosting.} \citet{shwartz2022tabular} showed that gradient-boosted trees remain competitive with tabular deep learning on many tasks, but \citet{gorishniy2021revisiting} and \citet{hollmann2023tabpfn} documented exceptions. We attempted TabPFN v2 in the Phase~B bake-off but were blocked by the token-auth requirement at \url{ux.priorlabs.ai}; deferred to v0.2.

\paragraph{Crowd-aggregation baselines.} \citet{tetlock2005}, \citet{manski2006}, and \citet{wolfers2004} established that prediction-market prices are well-calibrated aggregates of participant judgement. Our \emph{baseline} is therefore not naive: ``price/100'' is already the weighted consensus of real traders with money at risk. Beating it on any category is non-trivial.

% ── 6. Limitations ────────────────────────────────────────────────────────────
\section{Limitations}

\paragraph{1. Eleven days is not six months.} Our initial assumption was that six months of history would be available; investigation revealed 11 days. We therefore have \emph{one} temporal generalisation test (the last two days versus the first nine) rather than a robust seasonal evaluation. Retrains at 30-day, 90-day, and 180-day windows are scheduled (\S\ref{sec:roadmap}).

\paragraph{2. Five features is not twenty.} The live indicator table has 27 columns; the historical archive has 5. Until the schema expansion lands (v0.3), we cannot evaluate whether the remaining 15+ indicators (adjusted implied yield, cancel-versus-volume delta, residual volatility, cross-venue gap, etc.) add signal.

\paragraph{3. No cross-venue features.} Kalshi--Polymarket price gaps for the same event (where they exist) are one of the largest observed signals in SF's internal analytics. V0.1 does not incorporate them; v0.3 will.

\paragraph{4. Selection bias on T4.} The 14K-row T4 corpus is only a 1.3\% slice of \texttt{marketwide\_resolutions}, driven by feature-snapshot availability at $t_0 - 24$h. It is plausible that categories with more continuous trading (where 24h-prior snapshots exist) are exactly the categories where microstructure is most informative, inflating apparent per-category gains.

\paragraph{5. No live deployment.} The weights are released as a static artefact. We have not yet deployed them as a real-time forecast endpoint. A production \texttt{/api/agent/forecast} is part of the v1.0 roadmap.

% ── 7. Roadmap ────────────────────────────────────────────────────────────────
\section{Roadmap}
\label{sec:roadmap}

We plan six phases, each with a pre-registered Brier improvement gate:

\begin{table}[H]
\centering
\small
\begin{tabular}{clccl}
\toprule
Phase & Description & Compute & Gate & If fail \\
\midrule
A & LightGBM baseline & laptop & $\geq 0.01$ over coinflip & stop; document \\
B & XGBoost + CatBoost + TabPFN & laptop & $\geq 0.003$ over A & ship A \\
C & FT-Transformer, SAINT, TabNet & Kaggle T4 & $\geq 0.005$ over B & ship B \\
D & Chronos-Bolt, TimesFM, Moirai (zero/few-shot) & Kaggle T4 & $\geq 0.005$ over C & negative result \\
E & Qwen3-4B QLoRA on tabular-as-text & Kaggle T4 & within 0.005 of best & no LLM path \\
F & KV-injection paper & rented A100/H100 & significant $>$ RAG baseline & publish negative \\
\bottomrule
\end{tabular}
\end{table}

Phases A and B shipped on 2026-04-19 with the numbers in this note. The next concrete step is Phase~D (Chronos-Bolt-base zero-shot), which is cheap to run and has a plausible near-term upside because time-series foundation models have been trained on exactly the kind of price series that constitute our corpus.

\paragraph{Retrain schedule.} v0.2 is scheduled for 2026-05-20 (30 days of R2 archive); v0.3 when the \texttt{market\_indicator\_history} schema expansion lands; v1.0 around 2026-10 with six months of data plus whichever of Phases C--E have cleared their gates.

% ── 8. Conclusion ────────────────────────────────────────────────────────────
\section{Conclusion}

We do not claim sf-ml-baseline v0.1 is a strong model. We claim three things only:

\begin{enumerate}[leftmargin=*,nosep]
    \item The SF microstructure feature set contains measurable predictive signal for 24-hour direction on prediction markets (Brier $-0.0206$, statistically significant).
    \item Gradient-boosted trees saturate on this feature set at $\sim$0.2294 Brier; further gains from classical models alone are unlikely.
    \item Per-category resolution models beat the market price on Crypto and Commodities by 0.035--0.041 Brier.
\end{enumerate}

The correct way to use this release is as one voice in an ensemble with news/LLM approaches, not as a replacement for them. We ship the weights, the training code, and the retrain plan so that other researchers and operators can verify the numbers and extend the roadmap.

% ── References ────────────────────────────────────────────────────────────────
\begin{thebibliography}{99}

\bibitem[AIA~(2025)]{aia2025}
AI Agent Alliance (2025). \textit{Multi-agent systems for long-horizon prediction-market forecasting}. Technical report.

\bibitem[Brier~(1950)]{brier1950}
Brier, G.\,W. (1950). \textit{Verification of forecasts expressed in terms of probability}. Monthly Weather Review, 78(1), 1--3.

\bibitem[Chen \& Guestrin~(2016)]{chen2016xgboost}
Chen, T. \& Guestrin, C. (2016). \textit{XGBoost: A scalable tree boosting system}. KDD.

\bibitem[Gneiting \& Raftery~(2007)]{gneiting2007strictly}
Gneiting, T. \& Raftery, A.\,E. (2007). \textit{Strictly proper scoring rules, prediction, and estimation}. JASA, 102(477), 359--378.

\bibitem[Gorishniy et al.~(2021)]{gorishniy2021revisiting}
Gorishniy, Y. et al. (2021). \textit{Revisiting deep learning models for tabular data}. NeurIPS.

\bibitem[Guo et al.~(2017)]{guo2017calibration}
Guo, C. et al. (2017). \textit{On calibration of modern neural networks}. ICML.

\bibitem[Halawi et al.~(2024)]{halawi2024}
Halawi, D. et al. (2024). \textit{Approaching human-level forecasting with language models}. arXiv:2402.18563.

\bibitem[Hollmann et al.~(2023)]{hollmann2023tabpfn}
Hollmann, N. et al. (2023). \textit{TabPFN: A transformer that solves small tabular classification problems in a second}. ICLR.

\bibitem[Jane Street~(2022)]{janestreet2022}
Jane Street (2022). \textit{Jane Street Market Prediction competition description}. Kaggle.

\bibitem[Ke et al.~(2017)]{ke2017lightgbm}
Ke, G. et al. (2017). \textit{LightGBM: A highly efficient gradient boosting decision tree}. NeurIPS.

\bibitem[Manski~(2006)]{manski2006}
Manski, C.\,F. (2006). \textit{Interpreting the predictions of prediction markets}. Economics Letters, 91(3), 425--429.

\bibitem[Prokhorenkova et al.~(2018)]{prokhorenkova2018catboost}
Prokhorenkova, L. et al. (2018). \textit{CatBoost: Unbiased boosting with categorical features}. NeurIPS.

\bibitem[Shwartz-Ziv \& Armon~(2022)]{shwartz2022tabular}
Shwartz-Ziv, R. \& Armon, A. (2022). \textit{Tabular data: Deep learning is not all you need}. Information Fusion, 81, 84--90.

\bibitem[Tetlock~(2005)]{tetlock2005}
Tetlock, P.\,E. (2005). \textit{Expert political judgment: How good is it? How can we know?} Princeton University Press.

\bibitem[Wolfers \& Zitzewitz~(2004)]{wolfers2004}
Wolfers, J. \& Zitzewitz, E. (2004). \textit{Prediction markets}. JEP, 18(2), 107--126.

\end{thebibliography}

% ── Self-cite ────────────────────────────────────────────────────────────────
\section*{How to Cite}

\begin{verbatim}
@misc{sf_feature_baseline_prelim_2026,
  author       = {Liu, Patrick},
  title        = {Feature-Based Prediction-Market Forecasting:
                  Preliminary Observations},
  year         = {2026},
  month        = {apr},
  howpublished = {SimpleFunctions Research Note},
  url          = {https://simplefunctions.dev/papers/feature-baseline-preliminary},
  note         = {Companion release:
                  \url{https://huggingface.co/SimpleFunctions/sf-ml-baseline}}
}
\end{verbatim}

\end{document}
