Something went wrong. Try again.
small gleam coding and (not yet) persistent agent daemon with a detachable cli forked from okami.mom/albedo
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327"""Every SQLite table the daemon creates has one transfer classification.
A session move copies only the tables owner `transfer.gleam` modules declare.A new table that holds session or workspace rows but has no `transfer.Table`or `transfer.Excluded` declaration would silently stay behind on the sourcedaemon. No end-to-end run installs and fills every owner's schema, so thisreads the schemas straight from the Gleam sources, without a build or daemon."""
from __future__ import annotations
import reimport unittestfrom dataclasses import dataclassfrom pathlib import Path
ROOT = Path(__file__).resolve().parents[2]SOURCE = ROOT / "src"
# Literals are joined with NUL so a table name never spans `<>` concatenation:# `"CREATE TABLE " <> name <> "_scoped("` (operations.scope_display_metadata,# a rebuild renamed back to its original table) is not a table of its own._JOIN = "\x00"_CREATE = re.compile( r"CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?\s+([A-Za-z_]\w*)\s*\(", re.I)_CREATE_ANY = re.compile(r"CREATE\s+TABLE\b", re.I)_ALTER = re.compile( r"ALTER\s+TABLE\s+([A-Za-z_]\w*)\s+ADD\s+(?:COLUMN\s+)?([A-Za-z_]\w*)", re.I)_ADD_COLUMNS = re.compile(r'store\.add_columns\(\s*\w+\s*,\s*"([A-Za-z_]\w*)"\s*,\s*\[')_PAIR = re.compile(r'#\(\s*"([A-Za-z_]\w*)"\s*,')_CONSTRAINTS = {"PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "CONSTRAINT"}
@dataclass(frozen=True)class Schema: name: str columns: tuple[str, ...] source: str line: int
def literals(source: str) -> list[tuple[str, int]]: """Each Gleam string literal's text and starting line, comments skipped.""" found, index, line = [], 0, 1 while index < len(source): char = source[index] if char == "\n": line += 1 elif source.startswith("//", index): index = source.find("\n", index) if index < 0: break continue elif char == '"': start_line, index, text = line, index + 1, [] while source[index] != '"': if source[index] == "\\": text.append(source[index : index + 2]) index += 2 continue line += source[index] == "\n" text.append(source[index]) index += 1 found.append(("".join(text).replace('\\"', '"'), start_line)) index += 1 return found
def _body(sql: str, opening: int) -> str: depth, quote = 0, None for index in range(opening, len(sql)): char = sql[index] if quote: quote = None if char == quote else quote elif char in "'\"": quote = char elif char == "(": depth += 1 elif char == ")": depth -= 1 if depth == 0: return sql[opening + 1 : index] raise AssertionError("unbalanced CREATE TABLE: " + sql[opening : opening + 80])
def _columns(body: str) -> list[str]: columns, depth, start = [], 0, 0 for index, char in enumerate(body + ","): depth += (char == "(") - (char == ")") if char == "," and depth == 0: word = re.match(r"\s*([A-Za-z_]\w*)", body[start:index]) if word and word.group(1).upper() not in _CONSTRAINTS: columns.append(word.group(1)) start = index + 1 return columns
def schemas() -> dict[str, Schema]: """Every table a source file creates, with the columns later code adds.""" created: dict[str, tuple[list[str], str, int]] = {} added: dict[str, list[str]] = {} for path in sorted(SOURCE.rglob("*.gleam")): source = path.read_text() parts = literals(source) sql = _JOIN.join(text for text, _ in parts) starts, offset = [], 0 for text, line in parts: starts.append((offset, line)) offset += len(text) + 1 relative = str(path.relative_to(ROOT)) for match in _CREATE.finditer(sql): line = max(at for offset, at in starts if offset <= match.start()) columns = _columns(_body(sql, match.end() - 1)) prior = created.get(match.group(1)) if prior and prior[0] != columns: raise AssertionError(f"{match.group(1)} is created twice differently") created.setdefault(match.group(1), (columns, relative, line)) for match in _ALTER.finditer(sql): added.setdefault(match.group(1), []).append(match.group(2)) for match in _ADD_COLUMNS.finditer(source): end = source.index("])", match.end()) pairs = _PAIR.findall(source[match.end() : end]) added.setdefault(match.group(1), []).extend(pairs) dynamic = len(_CREATE_ANY.findall(sql)) - len(_CREATE.findall(sql)) if dynamic and relative != "src/albedo/daemon/operations.gleam": raise AssertionError(f"{relative} builds a table name at runtime") result = {} for name, (columns, source, line) in created.items(): extra = [c for c in added.get(name, []) if c not in columns] result[name] = Schema(name, tuple(columns + extra), source, line) return result
_TOKEN = re.compile( r'\s+|//[^\n]*|"(?:[^"\\]|\\.)*"|<>|\d+|[A-Za-z_][\w]*(?:/[A-Za-z_]\w*)*|.', re.S)_TYPES = SOURCE / "albedo" / "transfer.gleam"
def _tokens(source: str) -> list[str]: return [ token for token in _TOKEN.findall(source) if token.strip() and not token.startswith("//") ]
def _imported(tokens: list[str]) -> set[str]: """Constructors `import albedo/transfer.{...}` brings in unqualified.""" names: set[str] = set() for index, token in enumerate(tokens): if token == "albedo/transfer" and tokens[index - 1] == "import": if tokens[index + 1 : index + 3] == [".", "{"]: at = index + 3 while tokens[at] != "}": if tokens[at] in ("Table", "Excluded") and tokens[at - 1] != "type": names.add(tokens[at]) at += 1 return names
class _Declarations: """Reads Table/Excluded calls whose arguments are literals, same-module string consts, `<>` joins of those, and ints. Anything else fails, so a declaration the test cannot read is never skipped."""
def __init__(self, path: Path): self.path = str(path.relative_to(ROOT)) self.tokens = _tokens(path.read_text()) self.consts: dict[str, str] = {} self.at = 0 starts = { self.tokens[index + 1]: index + 2 for index, token in enumerate(self.tokens) if token == "const" } # Passes until nothing new resolves: a const may name a later one. while True: resolved = len(self.consts) for name, start in starts.items(): self.at = start + 2 if self.tokens[start] == ":" else start if name in self.consts or self.tokens[self.at] != "=": continue self.at += 1 try: self.consts[name] = self._string() except AssertionError: continue if len(self.consts) == resolved: break
def _fail(self, wanted: str) -> AssertionError: found = " ".join(self.tokens[self.at : self.at + 6]) return AssertionError(f"{self.path}: expected {wanted}, found {found!r}")
def _take(self, token: str) -> None: if self.tokens[self.at] != token: raise self._fail(repr(token)) self.at += 1
def _string(self) -> str: parts = [] while True: token = self.tokens[self.at] if token.startswith('"'): parts.append(token[1:-1].replace('\\"', '"').replace("\\\\", "\\")) elif token in self.consts: parts.append(self.consts[token]) else: raise self._fail("a string literal or same-module string const") self.at += 1 if self.tokens[self.at] != "<>": return "".join(parts) self.at += 1
def _label(self, name: str) -> None: if self.tokens[self.at] == name and self.tokens[self.at + 1] == ":": self.at += 2
def _argument(self, name: str, read): self._label(name) value = read() if self.tokens[self.at] == ",": self.at += 1 return value
def _int(self) -> int: token = self.tokens[self.at] if not token.isdigit(): raise self._fail("an int literal") self.at += 1 return int(token)
def calls(self): unqualified = _imported(self.tokens) for index, token in enumerate(self.tokens): if token not in ("Table", "Excluded") or self.tokens[index + 1] != "(": continue qualified = self.tokens[index - 2 : index] == ["transfer", "."] if not qualified and token not in unqualified: raise AssertionError(f"{self.path}: {token}( is not albedo/transfer's") self.at = index + 2 if token == "Table": entry = ( self._argument("name", self._string), self._argument("version", self._int), self._argument("filter_sql", self._string), ) else: entry = ( self._argument("name", self._string), self._argument("reason", self._string), ) self._take(")") yield token, entry
def declarations() -> tuple[dict[str, list], dict[str, list]]: tables: dict[str, list] = {} excluded: dict[str, list] = {} for path in sorted(SOURCE.rglob("*.gleam")): if path == _TYPES: continue if path.name != "transfer.gleam": tokens = _tokens(path.read_text()) built = any( tokens[index : index + 3] == ["transfer", "."] + [name] and tokens[index + 3 : index + 4] == ["("] for index in range(len(tokens)) for name in ("Table", "Excluded") ) if built or _imported(tokens): raise AssertionError( f"{path.relative_to(ROOT)} declares transfer tables outside a transfer.gleam" ) continue reader = _Declarations(path) for kind, entry in reader.calls(): if kind == "Table": name, version, filter_sql = entry tables.setdefault(name, []).append((version, filter_sql, reader.path)) else: excluded.setdefault(entry[0], []).append((entry[1], reader.path)) return tables, excluded
class TransferCoverageTest(unittest.TestCase): def test_every_table_is_declared_once(self): schema = schemas() tables, excluded = declarations() declared = list(tables) + list(excluded) twice = sorted( name for name in set(declared) if len(tables.get(name, [])) + len(excluded.get(name, [])) > 1 ) self.assertEqual(twice, [], "tables declared more than once") missing = sorted(set(schema) - set(declared)) self.assertEqual( missing, [], "tables with no transfer.Table or transfer.Excluded" ) unknown = sorted(set(declared) - set(schema)) self.assertEqual( unknown, [], "transfer declarations for tables nothing creates" )
def test_filters_are_scoped(self): tables, _ = declarations() for name, entries in tables.items(): _, filter_sql, source = entries[0] with self.subTest(table=name, source=source): self.assertRegex(filter_sql, r"\btransfer_(?:scope|workspace)\b") self.assertNotRegex(filter_sql, r"'[0-9a-f]{8,}'", "literal ids")
def test_reasons_are_given(self): _, excluded = declarations() for name, entries in excluded.items(): self.assertTrue( entries[0][0].strip(), f"{name} is excluded without a reason" )
if __name__ == "__main__": unittest.main()