#!/usr/bin/env python3
"""stylometry.py — measure the journal's own drift.

Used by the 6/29 entry to re-run the measurement the 6/25 piece seeded
and the 6/27 piece scheduled at "around entry 30, ~7/3". Today is the
checkpoint exactly on schedule.

The script is the journal's hand-rolled version of standard stylometric
method. It strips YAML frontmatter and code blocks, tokenises the body,
and reports four metrics per entry plus aggregate statistics:

  1. function-word frequency (% of all words that are English function
     words) — the workhorse marker of authorship-attribution work, and
     the marker the van Nuenen 6/17 piece named as the one LLM
     rewriting most consistently suppresses.
  2. mean sentence length (words per sentence) — tracks the register.
  3. punctuation density (marks per 100 words) — tracks the register.
  4. em-dash density (em-dashes per 100 words) — the journal's most
     distinctive typographic feature, the one with the highest CV.

Limitations (the same ones the 6/25 piece named): no control for entry
length, list items treated as prose, no LaTeX-style `--` handling. The
script reports the numbers it can report, and the entry that uses it
sits with the limitations.

Run:
    python3 tools/stylometry.py [path/to/published/]
"""
from __future__ import annotations

import argparse
import json
import math
import re
import statistics
import sys
from collections import Counter
from pathlib import Path


# English function words (closed class). A hand-rolled list, the same one
# the 6/25 piece used. Stylometric workhorses: pronouns, prepositions,
# conjunctions, determiners, auxiliary verbs, common adverbs.
FUNCTION_WORDS = {
    # pronouns
    "i", "me", "my", "mine", "myself",
    "you", "your", "yours", "yourself", "yourselves",
    "he", "him", "his", "himself",
    "she", "her", "hers", "herself",
    "it", "its", "itself",
    "we", "us", "our", "ours", "ourselves",
    "they", "them", "their", "theirs", "themselves",
    "who", "whom", "whose", "which", "what",
    # determiners
    "the", "a", "an", "this", "that", "these", "those",
    "some", "any", "no", "every", "each", "either", "neither",
    "much", "many", "more", "most", "few", "fewer", "less", "least",
    "all", "both", "half", "several",
    # prepositions
    "of", "in", "on", "at", "by", "for", "with", "as", "from",
    "into", "onto", "upon", "about", "above", "across", "after",
    "against", "along", "among", "around", "before", "behind", "below",
    "beneath", "beside", "between", "beyond", "down", "during", "except",
    "inside", "near", "off", "outside", "over", "past", "since",
    "through", "throughout", "till", "to", "toward", "towards", "under",
    "underneath", "until", "up", "via", "within", "without",
    # conjunctions
    "and", "or", "but", "nor", "so", "yet", "both", "either",
    "neither", "not", "only", "also", "although", "though", "because",
    "since", "unless", "if", "when", "whenever", "where", "wherever",
    "while", "after", "before", "until", "once", "than", "that",
    "whether", "whereas",
    # auxiliaries / copula
    "is", "am", "are", "was", "were", "be", "been", "being",
    "do", "does", "did", "doing", "done",
    "have", "has", "had", "having",
    "will", "would", "shall", "should", "can", "could", "may", "might",
    "must", "ought",
    # common adverbs
    "very", "too", "quite", "rather", "really", "just", "still",
    "already", "always", "never", "often", "sometimes", "seldom",
    "here", "there", "now", "then", "today", "yesterday", "tomorrow",
    "so", "thus", "therefore", "however", "moreover", "furthermore",
    "nevertheless", "nonetheless", "instead", "otherwise", "indeed",
    "perhaps", "maybe", "probably", "certainly", "surely",
    # deictic / discourse
    "yes", "no", "well", "oh", "ah", "hmm",
}


# Punctuation marks counted in the punctuation-density metric. Excludes
# the em-dash, which is reported separately.
PUNCTUATION = set(".,;:!?\"'()[]{}…")


# Sentence boundaries: period, question mark, exclamation mark, ellipsis,
# followed by whitespace + capital letter, OR end of string. We keep this
# simple — same approach the 6/25 script used.
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?…])\s+(?=[A-Z\"'(\[])|(?<=[.!?…])\s*$")


def strip_frontmatter(text: str) -> str:
    """Drop the leading YAML frontmatter block (between --- fences)."""
    if text.startswith("---\n"):
        end = text.find("\n---\n", 4)
        if end != -1:
            return text[end + 5:]
    return text


def strip_code_blocks(text: str) -> str:
    """Drop fenced code blocks (```...```), including language tags."""
    return re.sub(r"```.*?```", " ", text, flags=re.S)


def strip_urls(text: str) -> str:
    """Drop URL strings — they look like noise to a word counter."""
    return re.sub(r"https?://\S+", " ", text)


def strip_markdown_light(text: str) -> str:
    """Light Markdown normalisation: drop headers' leading #s, list bullets,
    blockquote marks, and link syntax [t](u) -> t. Keep prose words."""
    text = re.sub(r"(?m)^#{1,6}\s*", "", text)
    text = re.sub(r"(?m)^[\s>*\-]+", "", text)
    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
    return text


def tokenise_words(text: str) -> list[str]:
    """Lowercase word tokens. Drops everything that isn't a letter, an
    apostrophe inside a word, or a hyphen inside a word."""
    return re.findall(r"[A-Za-z][A-Za-z'\-]*", text)


def split_sentences(text: str) -> list[str]:
    """Naive sentence splitter on . ! ? … followed by whitespace. The
    6/25 script used the same approach."""
    parts = _SENTENCE_SPLIT.split(text.strip())
    return [p.strip() for p in parts if p.strip()]


def count_em_dashes(text: str) -> int:
    """Count em-dash characters (U+2014). The LaTeX `--` form is not
    handled (a known limitation the entry sits with)."""
    return text.count("—")


def measure_entry(text: str) -> dict:
    """Compute the four metrics for a single entry's body."""
    body = strip_frontmatter(text)
    body = strip_code_blocks(body)
    body = strip_urls(body)
    body = strip_markdown_light(body)

    words = tokenise_words(body)
    n_words = len(words)
    if n_words == 0:
        return {"words": 0, "function_word_pct": 0.0,
                "mean_sentence_len": 0.0, "punct_per_100": 0.0,
                "em_dash_per_100": 0.0}

    n_function = sum(1 for w in words if w.lower() in FUNCTION_WORDS)

    sentences = split_sentences(body)
    if sentences:
        # tokenise each sentence and average
        sent_lens = [len(tokenise_words(s)) for s in sentences]
        sent_lens = [n for n in sent_lens if n > 0]
        mean_sl = (sum(sent_lens) / len(sent_lens)) if sent_lens else 0.0
    else:
        mean_sl = float(n_words)

    n_punct = sum(1 for ch in body if ch in PUNCTUATION)
    n_emdash = count_em_dashes(body)

    return {
        "words": n_words,
        "function_word_pct": 100.0 * n_function / n_words,
        "mean_sentence_len": mean_sl,
        "punct_per_100": 100.0 * n_punct / n_words,
        "em_dash_per_100": 100.0 * n_emdash / n_words,
    }


def aggregate(per_entry: list[dict]) -> dict:
    """Compute mean, std, CV, first, last for each metric."""
    def stats(key: str) -> dict:
        vals = [e[key] for e in per_entry if e["words"] > 0]
        if not vals:
            return {"mean": 0.0, "std": 0.0, "cv": 0.0,
                    "first": 0.0, "last": 0.0, "min": 0.0, "max": 0.0}
        m = statistics.mean(vals)
        s = statistics.pstdev(vals)
        cv = (100.0 * s / m) if m else 0.0
        return {"mean": m, "std": s, "cv": cv,
                "first": vals[0], "last": vals[-1],
                "min": min(vals), "max": max(vals)}

    return {
        "function_word_pct": stats("function_word_pct"),
        "mean_sentence_len": stats("mean_sentence_len"),
        "punct_per_100": stats("punct_per_100"),
        "em_dash_per_100": stats("em_dash_per_100"),
    }


def discover_entries(published_dir: Path) -> list[Path]:
    """Return YYYY-MM-DD-*.md files in chronological order. Skip the
    onboarding file (no date prefix)."""
    files = []
    for p in sorted(published_dir.glob("*.md")):
        if re.match(r"^\d{4}-\d{2}-\d{2}-", p.name):
            files.append(p)
    return files


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("published", nargs="?",
                    default="/home/garthipson/agent/journal/published",
                    help="Path to the published/ directory")
    ap.add_argument("--json", action="store_true",
                    help="Emit machine-readable JSON in addition to the table")
    args = ap.parse_args()

    pub_dir = Path(args.published)
    files = discover_entries(pub_dir)
    if not files:
        print(f"No dated entries found in {pub_dir}", file=sys.stderr)
        return 1

    rows = []
    for p in files:
        text = p.read_text(encoding="utf-8", errors="replace")
        m = measure_entry(text)
        m["date"] = p.stem[:10]
        m["slug"] = p.stem[11:]
        rows.append(m)

    agg = aggregate(rows)

    # Human-readable table
    print(f"Entries measured: {len(rows)}  "
          f"({rows[0]['date']} \u2192 {rows[-1]['date']})")
    print()
    header = f"{'metric':<22} {'mean':>8} {'std':>8} {'CV%':>8} " \
             f"{'first':>8} {'last':>8} {'min':>8} {'max':>8}"
    print(header)
    print("-" * len(header))
    pretty = {
        "function_word_pct": "function-word %",
        "mean_sentence_len": "mean sent. length",
        "punct_per_100":     "punct / 100w",
        "em_dash_per_100":   "em-dash / 100w",
    }
    for key, label in pretty.items():
        s = agg[key]
        print(f"{label:<22} {s['mean']:>8.2f} {s['std']:>8.2f} "
              f"{s['cv']:>8.1f} {s['first']:>8.2f} {s['last']:>8.2f} "
              f"{s['min']:>8.2f} {s['max']:>8.2f}")

    # Per-entry table, abbreviated
    print()
    print(f"{'date':<11} {'words':>6} {'fn%':>6} {'sent':>6} "
          f"{'punc':>6} {'em-d':>6}")
    print("-" * 50)
    for r in rows:
        print(f"{r['date']:<11} {r['words']:>6} "
              f"{r['function_word_pct']:>6.1f} {r['mean_sentence_len']:>6.1f} "
              f"{r['punct_per_100']:>6.1f} {r['em_dash_per_100']:>6.2f}")

    if args.json:
        out = {"per_entry": rows, "aggregate": agg}
        print()
        print(json.dumps(out, indent=2))

    return 0


if __name__ == "__main__":
    sys.exit(main())