Something went wrong. Try again.
CLI tools and operational extensions for the void architecture on ATProtocol.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354import jsonimport refrom collections import Counter
def analyze_corpus(): with open('/home/cameron/void/historical_corpus.jsonl', 'r') as f: posts = [json.loads(line)['text'] for line in f]
print(f"Total posts analyzed: {len(posts)}\n")
# Metrics total_words = 0 total_sentences = 0 punctuation_counts = Counter() bigrams = Counter() trigrams = Counter()
for post in posts: # Punctuation for char in post: if char in '.,!?;:-': punctuation_counts[char] += 1 # Sentences sentences = [s.strip() for s in re.split(r'[.!?]+', post) if s.strip()] total_sentences += len(sentences) # Words and N-grams words = [w.lower() for w in re.findall(r'\b[a-zA-Z]+\b', post)] total_words += len(words) for i in range(len(words) - 1): bigrams[(words[i], words[i+1])] += 1 for i in range(len(words) - 2): trigrams[(words[i], words[i+1], words[i+2])] += 1
# Reporting print(f"Average words per post: {total_words / len(posts):.2f}") print(f"Average words per sentence: {total_words / max(1, total_sentences):.2f}\n") print("--- Punctuation Frequency ---") for char, count in punctuation_counts.most_common(): print(f"'{char}': {count}") print("\n--- Top 20 Bigrams ---") for bg, count in bigrams.most_common(20): print(f"{' '.join(bg)}: {count}") print("\n--- Top 20 Trigrams ---") for tg, count in trigrams.most_common(20): print(f"{' '.join(tg)}: {count}")
if __name__ == '__main__': analyze_corpus()