Something went wrong. Try again.
Command line interface for accessing the Open Web Index shards and slices.
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061#!/usr/bin/env python3"""Script to fetch sample dataset metadata from Lexis API for testing.Saves metadata as JSON for use in unit tests."""import jsonimport osimport sys
# Add project root to pathsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from owilix.core import OWIlixManager
def fetch_sample_datasets(count: int = 5): """Fetch sample datasets from Lexis and save as test fixtures.""" manager = OWIlixManager() print(f"Fetching datasets with specifier 'all:latest'...") spec = manager.parse_specifier("all:latest") datasets = manager.remote_data.list( access=spec.get("query", {}).get("access", "public"), day=spec.get("day"), duration=spec.get("duration", 0), query=spec.get("query"), ) print(f"Found {len(datasets)} datasets, taking first {count}...") samples = [] for ds in datasets[:count]: # Extract full metadata as dict md = { "id": ds.metadata.get("id") or ds.metadata.get("internalID"), "path": ds.path, "access": ds.access, "collectionName": ds.collectionName, "zone": getattr(ds, "zone", None), "dataCenter": ds.dataCenter, "metadata_dict": ds.metadata.as_json_dict() if hasattr(ds.metadata, "as_json_dict") else dict(ds.metadata), } samples.append(md) print(f" - {md['id']}: {ds.metadata.get('title', 'NO TITLE')[:60]}...") # Save to fixtures directory fixtures_dir = os.path.join(os.path.dirname(__file__), "fixtures") os.makedirs(fixtures_dir, exist_ok=True) output_path = os.path.join(fixtures_dir, "lexis_sample_datasets.json") with open(output_path, "w") as f: json.dump(samples, f, indent=2, default=str) print(f"\nSaved {len(samples)} samples to {output_path}") return samples
if __name__ == "__main__": fetch_sample_datasets(5)