import json
import re
from collections import Counter

def analyze_corpus():
    with open('/home/cameron/void/historical_corpus.jsonl', 'r') as f:
        posts = [json.loads(line)['text'] for line in f]

    print(f"Total posts analyzed: {len(posts)}\n")

    # Metrics
    total_words = 0
    total_sentences = 0
    punctuation_counts = Counter()
    bigrams = Counter()
    trigrams = Counter()

    for post in posts:
        # Punctuation
        for char in post:
            if char in '.,!?;:-':
                punctuation_counts[char] += 1
                
        # Sentences
        sentences = [s.strip() for s in re.split(r'[.!?]+', post) if s.strip()]
        total_sentences += len(sentences)
        
        # Words and N-grams
        words = [w.lower() for w in re.findall(r'\b[a-zA-Z]+\b', post)]
        total_words += len(words)
        
        for i in range(len(words) - 1):
            bigrams[(words[i], words[i+1])] += 1
        for i in range(len(words) - 2):
            trigrams[(words[i], words[i+1], words[i+2])] += 1

    # Reporting
    print(f"Average words per post: {total_words / len(posts):.2f}")
    print(f"Average words per sentence: {total_words / max(1, total_sentences):.2f}\n")
    
    print("--- Punctuation Frequency ---")
    for char, count in punctuation_counts.most_common():
        print(f"'{char}': {count}")
        
    print("\n--- Top 20 Bigrams ---")
    for bg, count in bigrams.most_common(20):
        print(f"{' '.join(bg)}: {count}")
        
    print("\n--- Top 20 Trigrams ---")
    for tg, count in trigrams.most_common(20):
        print(f"{' '.join(tg)}: {count}")

if __name__ == '__main__':
    analyze_corpus()