feat: add PLAN.md and insider copytrade POC implementation

- PLAN.md: full implementation plan from issue - config.py: configurable thresholds, API keys via .env - ingestion/: EDGAR RSS poller + Form 4 XML parser - db/: SQLite schema + interface (WAL mode) - signals/: filter engine (buy/10b5-1/value/role) + cluster detector - alerts/: Slack webhook alert with score gating - broker/: Alpaca paper/live trade execution - backtest/: historical signal backtesting with yfinance - main.py: CLI entrypoint (run | fetch-once | backtest)
2026-05-04 16:15:22 +00:00
parent 7ddf89ebfb
commit 7e9221a914
20 changed files with 1244 additions and 0 deletions
@@ -0,0 +1,138 @@
+import time
+import os
+import logging
+from datetime import datetime
+from typing import Optional
+import requests
+from lxml import etree
+
+import config
+from ingestion.form4_parser import parse_form4
+from db.db import insert_filing, accession_exists
+
+logger = logging.getLogger(__name__)
+
+HEADERS = {
+    "User-Agent": "insider-copytrade-poc contact@example.com",
+    "Accept-Encoding": "gzip, deflate",
+}
+
+EDGAR_FULL_INDEX = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&type=4&dateb=&owner=include&count=40&output=atom"
+
+
+def _fetch(url: str, timeout: int = 30) -> requests.Response:
+    resp = requests.get(url, headers=HEADERS, timeout=timeout)
+    resp.raise_for_status()
+    return resp
+
+
+def _get_filing_urls() -> list[tuple[str, str, str]]:
+    resp = _fetch(EDGAR_FULL_INDEX)
+    root = etree.fromstring(resp.content)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    entries = root.findall("atom:entry", ns)
+    results = []
+    for entry in entries:
+        filing_href = entry.find("atom:link", ns)
+        if filing_href is None:
+            continue
+        url = filing_href.get("href", "")
+        updated = (entry.findtext("atom:updated", namespaces=ns) or "")[:10]
+        accession = url.rstrip("/").split("/")[-1].replace("-index.htm", "")
+        accession = accession.replace("-", "")
+        if len(accession) == 18:
+            accession = f"{accession[:10]}-{accession[10:12]}-{accession[12:]}"
+        results.append((url, accession, updated))
+    return results
+
+
+def _get_xml_url_from_index(index_url: str) -> Optional[str]:
+    try:
+        resp = _fetch(index_url)
+    except Exception:
+        return None
+    root = etree.fromstring(resp.content)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    for link in root.findall("atom:link", ns):
+        href = link.get("href", "")
+        if href.endswith(".xml") and "form4" in href.lower():
+            return href
+    for link in root.findall(".//filing-href"):
+        if link.text and link.text.endswith(".xml"):
+            return link.text.strip()
+    return None
+
+
+def _save_raw_xml(accession: str, xml_bytes: bytes):
+    os.makedirs(config.DATA_DIR, exist_ok=True)
+    path = os.path.join(config.DATA_DIR, f"{accession}.xml")
+    if not os.path.exists(path):
+        with open(path, "wb") as f:
+            f.write(xml_bytes)
+
+
+def fetch_and_store_new_filings() -> list[dict]:
+    new_filings = []
+    try:
+        entries = _get_filing_urls()
+    except Exception as e:
+        logger.error(f"Failed to fetch EDGAR index: {e}")
+        return new_filings
+
+    for index_url, accession, filed_date in entries:
+        if accession_exists(accession):
+            continue
+
+        xml_url = _resolve_xml_url(index_url, accession)
+        if not xml_url:
+            logger.warning(f"No XML found for {accession}")
+            continue
+
+        try:
+            xml_resp = _fetch(xml_url)
+            xml_bytes = xml_resp.content
+        except Exception as e:
+            logger.error(f"Failed to fetch XML for {accession}: {e}")
+            continue
+
+        _save_raw_xml(accession, xml_bytes)
+        parsed = parse_form4(xml_bytes, accession, filed_date)
+
+        for filing in parsed:
+            inserted = insert_filing(filing)
+            if inserted:
+                new_filings.append(filing)
+
+    return new_filings
+
+
+def _resolve_xml_url(index_url: str, accession: str) -> Optional[str]:
+    accession_path = accession.replace("-", "")
+    cik = accession_path[:10].lstrip("0")
+    base = f"{config.EDGAR_BASE_URL}/Archives/edgar/data/{cik}/{accession_path}/"
+    candidate = f"{base}{accession}-index.htm"
+    try:
+        resp = _fetch(candidate)
+        root = etree.fromstring(resp.content)
+        for node in root.iter():
+            text = (node.text or "").strip()
+            if text.endswith(".xml") and ("4" in text or "form" in text.lower()):
+                return base + text
+    except Exception:
+        pass
+    return None
+
+
+def run_poller(on_new_filing=None):
+    logger.info("EDGAR poller started")
+    while True:
+        logger.info("Polling EDGAR for new Form 4 filings...")
+        new = fetch_and_store_new_filings()
+        logger.info(f"Found {len(new)} new filings")
+        if on_new_filing:
+            for filing in new:
+                try:
+                    on_new_filing(filing)
+                except Exception as e:
+                    logger.error(f"Error in on_new_filing callback: {e}")
+        time.sleep(config.EDGAR_POLL_INTERVAL)
@@ -0,0 +1,95 @@
+import re
+from lxml import etree
+from typing import Optional
+
+
+_10B51_PATTERNS = [
+    r"10b5-1",
+    r"rule 10b5",
+    r"adopted a plan",
+    r"10b5\(1\)",
+]
+
+
+def _is_10b51(text: str) -> bool:
+    text_lower = text.lower()
+    return any(re.search(p, text_lower) for p in _10B51_PATTERNS)
+
+
+def _text(el, tag: str) -> Optional[str]:
+    node = el.find(".//" + tag)
+    if node is not None and node.text:
+        return node.text.strip()
+    return None
+
+
+def _float(el, tag: str) -> Optional[float]:
+    val = _text(el, tag)
+    if val is None:
+        return None
+    try:
+        return float(val.replace(",", ""))
+    except ValueError:
+        return None
+
+
+def parse_form4(xml_bytes: bytes, accession_number: str, filed_date: str) -> list[dict]:
+    try:
+        root = etree.fromstring(xml_bytes)
+    except etree.XMLSyntaxError:
+        return []
+
+    ticker = _text(root, "issuerTradingSymbol") or ""
+    cik = _text(root, "issuerCik") or ""
+    insider_name = _text(root, "rptOwnerName") or ""
+    role = _text(root, "officerTitle") or _text(root, "isDirector") or ""
+
+    footnotes_text = " ".join(
+        (node.text or "") for node in root.findall(".//footnote")
+    )
+    global_10b51 = _is_10b51(footnotes_text)
+
+    transactions = root.findall(".//nonDerivativeTransaction")
+    results = []
+
+    for tx in transactions:
+        flag = _text(tx, "transactionAcquiredDisposedCode")
+        if not flag:
+            continue
+
+        shares = _float(tx, "transactionShares")
+        price = _float(tx, "transactionPricePerShare")
+        total_value = _float(tx, "transactionTotalValue")
+        if total_value is None and shares is not None and price is not None:
+            total_value = shares * price
+        post_tx_shares = _float(tx, "sharesOwnedFollowingTransaction")
+        tx_date = _text(tx, "transactionDate") or filed_date
+
+        tx_footnote_ids = [
+            fn.get("id", "") for fn in tx.findall(".//footnoteId")
+        ]
+        tx_footnote_text = " ".join(
+            (root.find(f".//footnote[@id='{fid}']") or etree.Element("x")).text or ""
+            for fid in tx_footnote_ids
+        )
+        is_10b51 = int(global_10b51 or _is_10b51(tx_footnote_text))
+
+        results.append(
+            {
+                "accession_number": accession_number,
+                "ticker": ticker.upper(),
+                "cik": cik,
+                "insider_name": insider_name,
+                "role": role,
+                "transaction_date": tx_date,
+                "filed_date": filed_date,
+                "shares": shares,
+                "price": price,
+                "total_value": total_value,
+                "flag": flag.upper(),
+                "is_10b51": is_10b51,
+                "post_tx_shares": post_tx_shares,
+            }
+        )
+
+    return results