feat: SQLAlchemy ORM models, filing cache incremental fetch, yfinance price cache
- Replace db/schema.sql + raw sqlite3 with SQLAlchemy ORM (db/models.py) - Filing, Signal, PriceCache models with proper indexes - db/db.py uses SQLAlchemy sessions throughout; no raw SQL strings - Add PriceCache table: stores daily close prices per ticker - backtest._fetch_prices checks DB first; skips yfinance for completed ranges - New data persisted via upsert_prices() - get_cached_prices() / upsert_prices() added to db.py - EDGAR poller incremental fetch: get_latest_filed_date() returns newest filed_date in DB; fetch_and_store_new_filings skips entries older than that cutoff before even checking accession_exists - Add get_signals_for_backtest() to db.py; backtest no longer opens its own sqlite3 connection - requirements.txt: add sqlalchemy>=2.0.0 Co-authored-by: dodox <dodox@users.noreply.local>
This commit is contained in:
dodox
+62
-58
@@ -3,44 +3,67 @@ import math
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import config
|
||||
from db.db import get_cached_prices, get_signals_for_backtest, upsert_prices
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_signals_from_db(db_path: str, min_score: float, min_cluster_size: int) -> list[dict]:
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT s.*, f.role FROM signals s
|
||||
LEFT JOIN filings f ON f.ticker = s.ticker AND f.transaction_date = s.trigger_date
|
||||
WHERE s.score >= ? AND s.cluster_size >= ?
|
||||
""",
|
||||
(min_score, min_cluster_size),
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
def _fetch_prices(ticker: str, start: datetime, end: datetime) -> dict[str, float]:
|
||||
try:
|
||||
import yfinance as yf
|
||||
except ImportError:
|
||||
raise ImportError("yfinance not installed. Run: pip install yfinance")
|
||||
|
||||
start_str = start.strftime("%Y-%m-%d")
|
||||
end_str = (end + timedelta(days=5)).strftime("%Y-%m-%d")
|
||||
|
||||
cached = get_cached_prices(ticker, start_str, end_str)
|
||||
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
range_is_complete = end_str < today
|
||||
|
||||
if range_is_complete and cached:
|
||||
return cached
|
||||
|
||||
data = yf.download(
|
||||
ticker,
|
||||
start=start_str,
|
||||
end=end_str,
|
||||
progress=False,
|
||||
auto_adjust=True,
|
||||
)
|
||||
if data.empty:
|
||||
return cached
|
||||
|
||||
fetched: dict[str, float] = {}
|
||||
for ts, close_val in data["Close"].items():
|
||||
date_key = ts.to_pydatetime().replace(tzinfo=None).strftime("%Y-%m-%d")
|
||||
fetched[date_key] = float(close_val)
|
||||
|
||||
new_prices = {k: v for k, v in fetched.items() if k not in cached}
|
||||
if new_prices:
|
||||
upsert_prices(ticker, new_prices)
|
||||
|
||||
cached.update(fetched)
|
||||
return cached
|
||||
|
||||
|
||||
def _first_close_on_or_after(price_data, target_date: datetime) -> float:
|
||||
"""Return the closing price on the first trading day on or after target_date."""
|
||||
for ts, row in price_data["Close"].items():
|
||||
ts_date = ts.to_pydatetime().replace(tzinfo=None)
|
||||
if ts_date.date() >= target_date.date():
|
||||
return float(row)
|
||||
raise ValueError(f"No price data on or after {target_date.date()}")
|
||||
def _first_close_on_or_after(prices: dict[str, float], target: datetime) -> float:
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
for date_str in sorted(prices):
|
||||
if date_str >= target_str:
|
||||
return prices[date_str]
|
||||
raise ValueError(f"No price data on or after {target_str}")
|
||||
|
||||
|
||||
def _first_close_before(price_data, target_date: datetime) -> float:
|
||||
"""Return the closing price on the last trading day before or on target_date."""
|
||||
def _first_close_before(prices: dict[str, float], target: datetime) -> float:
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
result = None
|
||||
for ts, row in price_data["Close"].items():
|
||||
ts_date = ts.to_pydatetime().replace(tzinfo=None)
|
||||
if ts_date.date() <= target_date.date():
|
||||
result = float(row)
|
||||
for date_str in sorted(prices):
|
||||
if date_str <= target_str:
|
||||
result = prices[date_str]
|
||||
if result is None:
|
||||
raise ValueError(f"No price data on or before {target_date.date()}")
|
||||
raise ValueError(f"No price data on or before {target_str}")
|
||||
return result
|
||||
|
||||
|
||||
@@ -50,22 +73,15 @@ def run_backtest(
|
||||
min_score: float = 0.0,
|
||||
min_cluster_size: int = 1,
|
||||
) -> dict:
|
||||
try:
|
||||
import yfinance as yf
|
||||
except ImportError:
|
||||
raise ImportError("yfinance not installed. Run: pip install yfinance")
|
||||
|
||||
db_path = db_path or config.DB_PATH
|
||||
holding_days = holding_days or config.HOLDING_PERIOD_DAYS
|
||||
|
||||
signals = _load_signals_from_db(db_path, min_score, min_cluster_size)
|
||||
signals = get_signals_for_backtest(min_score, min_cluster_size)
|
||||
|
||||
if not signals:
|
||||
logger.warning("No signals found matching criteria")
|
||||
return {}
|
||||
|
||||
results = []
|
||||
spy_cache: dict[tuple, float] = {}
|
||||
spy_cache: dict[str, float] = {}
|
||||
|
||||
for signal in signals:
|
||||
ticker = signal["ticker"]
|
||||
@@ -79,38 +95,26 @@ def run_backtest(
|
||||
exit_date = entry_date + timedelta(days=holding_days)
|
||||
|
||||
try:
|
||||
stock_data = yf.download(
|
||||
ticker,
|
||||
start=entry_date.strftime("%Y-%m-%d"),
|
||||
end=(exit_date + timedelta(days=5)).strftime("%Y-%m-%d"),
|
||||
progress=False,
|
||||
auto_adjust=True,
|
||||
)
|
||||
if stock_data.empty:
|
||||
prices = _fetch_prices(ticker, entry_date, exit_date)
|
||||
if not prices:
|
||||
logger.debug(f"No price data for {ticker}")
|
||||
continue
|
||||
|
||||
entry_price = _first_close_on_or_after(stock_data, entry_date)
|
||||
exit_price = _first_close_before(stock_data, exit_date)
|
||||
entry_price = _first_close_on_or_after(prices, entry_date)
|
||||
exit_price = _first_close_before(prices, exit_date)
|
||||
stock_return = (exit_price - entry_price) / entry_price
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get data for {ticker}: {e}")
|
||||
continue
|
||||
|
||||
period_key = (entry_date_str, holding_days)
|
||||
period_key = entry_date_str
|
||||
if period_key not in spy_cache:
|
||||
try:
|
||||
spy_data = yf.download(
|
||||
"SPY",
|
||||
start=entry_date.strftime("%Y-%m-%d"),
|
||||
end=(exit_date + timedelta(days=5)).strftime("%Y-%m-%d"),
|
||||
progress=False,
|
||||
auto_adjust=True,
|
||||
)
|
||||
if not spy_data.empty:
|
||||
spy_entry = _first_close_on_or_after(spy_data, entry_date)
|
||||
spy_exit = _first_close_before(spy_data, exit_date)
|
||||
spy_prices = _fetch_prices("SPY", entry_date, exit_date)
|
||||
if spy_prices:
|
||||
spy_entry = _first_close_on_or_after(spy_prices, entry_date)
|
||||
spy_exit = _first_close_before(spy_prices, exit_date)
|
||||
spy_cache[period_key] = (spy_exit - spy_entry) / spy_entry
|
||||
else:
|
||||
spy_cache[period_key] = 0.0
|
||||
|
||||
Reference in New Issue
Block a user