fix: address sanity-check issues + rebrand to Smaug
Co-authored-by: dodox <dodox@users.noreply.local>
This commit is contained in:
dodox
+33
-48
@@ -1,10 +1,9 @@
|
||||
import time
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
import requests
|
||||
from lxml import etree
|
||||
from lxml import etree, html
|
||||
|
||||
import config
|
||||
from ingestion.form4_parser import parse_form4
|
||||
@@ -13,11 +12,14 @@ from db.db import insert_filing, accession_exists
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "insider-copytrade-poc contact@example.com",
|
||||
"User-Agent": "smaug-insider-monitor contact@example.com",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
}
|
||||
|
||||
EDGAR_FULL_INDEX = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&type=4&dateb=&owner=include&count=40&output=atom"
|
||||
EDGAR_ATOM_URL = (
|
||||
"https://www.sec.gov/cgi-bin/browse-edgar"
|
||||
"?action=getcurrent&type=4&dateb=&owner=include&count=40&output=atom"
|
||||
)
|
||||
|
||||
|
||||
def _fetch(url: str, timeout: int = 30) -> requests.Response:
|
||||
@@ -27,39 +29,40 @@ def _fetch(url: str, timeout: int = 30) -> requests.Response:
|
||||
|
||||
|
||||
def _get_filing_urls() -> list[tuple[str, str, str]]:
|
||||
resp = _fetch(EDGAR_FULL_INDEX)
|
||||
resp = _fetch(EDGAR_ATOM_URL)
|
||||
root = etree.fromstring(resp.content)
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
entries = root.findall("atom:entry", ns)
|
||||
results = []
|
||||
for entry in entries:
|
||||
filing_href = entry.find("atom:link", ns)
|
||||
if filing_href is None:
|
||||
for entry in root.findall("atom:entry", ns):
|
||||
link = entry.find("atom:link", ns)
|
||||
if link is None:
|
||||
continue
|
||||
url = filing_href.get("href", "")
|
||||
url = link.get("href", "")
|
||||
updated = (entry.findtext("atom:updated", namespaces=ns) or "")[:10]
|
||||
accession = url.rstrip("/").split("/")[-1].replace("-index.htm", "")
|
||||
accession = accession.replace("-", "")
|
||||
if len(accession) == 18:
|
||||
accession = f"{accession[:10]}-{accession[10:12]}-{accession[12:]}"
|
||||
raw = url.rstrip("/").split("/")[-1].replace("-index.htm", "")
|
||||
raw = raw.replace("-", "")
|
||||
if len(raw) == 18:
|
||||
accession = f"{raw[:10]}-{raw[10:12]}-{raw[12:]}"
|
||||
else:
|
||||
accession = raw
|
||||
results.append((url, accession, updated))
|
||||
return results
|
||||
|
||||
|
||||
def _get_xml_url_from_index(index_url: str) -> Optional[str]:
|
||||
def _resolve_xml_url(accession: str) -> Optional[str]:
|
||||
accession_path = accession.replace("-", "")
|
||||
cik = accession_path[:10].lstrip("0")
|
||||
base = f"{config.EDGAR_BASE_URL}/Archives/edgar/data/{cik}/{accession_path}/"
|
||||
index_url = f"{base}{accession}-index.htm"
|
||||
try:
|
||||
resp = _fetch(index_url)
|
||||
except Exception:
|
||||
return None
|
||||
root = etree.fromstring(resp.content)
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
for link in root.findall("atom:link", ns):
|
||||
href = link.get("href", "")
|
||||
if href.endswith(".xml") and "form4" in href.lower():
|
||||
return href
|
||||
for link in root.findall(".//filing-href"):
|
||||
if link.text and link.text.endswith(".xml"):
|
||||
return link.text.strip()
|
||||
doc = html.fromstring(resp.content)
|
||||
for link in doc.cssselect("table.tableFile a[href]"):
|
||||
href = link.get("href", "")
|
||||
if href.lower().endswith(".xml") and not href.lower().endswith("-index.htm"):
|
||||
return config.EDGAR_BASE_URL + href if href.startswith("/") else base + href
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not resolve XML URL for {accession}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -79,11 +82,11 @@ def fetch_and_store_new_filings() -> list[dict]:
|
||||
logger.error(f"Failed to fetch EDGAR index: {e}")
|
||||
return new_filings
|
||||
|
||||
for index_url, accession, filed_date in entries:
|
||||
for _index_url, accession, filed_date in entries:
|
||||
if accession_exists(accession):
|
||||
continue
|
||||
|
||||
xml_url = _resolve_xml_url(index_url, accession)
|
||||
xml_url = _resolve_xml_url(accession)
|
||||
if not xml_url:
|
||||
logger.warning(f"No XML found for {accession}")
|
||||
continue
|
||||
@@ -99,30 +102,12 @@ def fetch_and_store_new_filings() -> list[dict]:
|
||||
parsed = parse_form4(xml_bytes, accession, filed_date)
|
||||
|
||||
for filing in parsed:
|
||||
inserted = insert_filing(filing)
|
||||
if inserted:
|
||||
if insert_filing(filing):
|
||||
new_filings.append(filing)
|
||||
|
||||
return new_filings
|
||||
|
||||
|
||||
def _resolve_xml_url(index_url: str, accession: str) -> Optional[str]:
|
||||
accession_path = accession.replace("-", "")
|
||||
cik = accession_path[:10].lstrip("0")
|
||||
base = f"{config.EDGAR_BASE_URL}/Archives/edgar/data/{cik}/{accession_path}/"
|
||||
candidate = f"{base}{accession}-index.htm"
|
||||
try:
|
||||
resp = _fetch(candidate)
|
||||
root = etree.fromstring(resp.content)
|
||||
for node in root.iter():
|
||||
text = (node.text or "").strip()
|
||||
if text.endswith(".xml") and ("4" in text or "form" in text.lower()):
|
||||
return base + text
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def run_poller(on_new_filing=None):
|
||||
logger.info("EDGAR poller started")
|
||||
while True:
|
||||
@@ -134,5 +119,5 @@ def run_poller(on_new_filing=None):
|
||||
try:
|
||||
on_new_filing(filing)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_new_filing callback: {e}")
|
||||
logger.error(f"Error processing filing {filing.get('accession_number')}: {e}")
|
||||
time.sleep(config.EDGAR_POLL_INTERVAL)
|
||||
|
||||
Reference in New Issue
Block a user