Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422#!/usr/bin/env nix-shell#! nix-shell -I nixpkgs=../../../.. -i python3 -p bundix bundler nix-update nix python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.packaging prefetch-yarn-deps git
import clickimport click_logimport reimport loggingimport subprocessimport jsonimport pathlibimport tempfilefrom packaging.version import Versionfrom typing import Iterable
import requests
NIXPKGS_PATH = pathlib.Path(__file__).parent / "../../../../"GITLAB_DIR = pathlib.Path(__file__).parent
logger = logging.getLogger(__name__)click_log.basic_config(logger)
class GitLabRepo: version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?(\-gitlab)?")
def __init__(self, owner: str = "gitlab-org", repo: str = "gitlab"): self.owner = owner self.repo = repo
@property def url(self): return f"https://gitlab.com/{self.owner}/{self.repo}"
@property def tags(self) -> Iterable[str]: """Returns a sorted list of repository tags""" r = requests.get(self.url + "/refs?sort=updated_desc&ref=master").json() tags = r.get("Tags", [])
# filter out versions not matching version_regex versions = list(filter(self.version_regex.match, tags))
# sort, but ignore v, -ee and -gitlab for sorting comparisons versions.sort( key=lambda x: Version( x.replace("v", "").replace("-ee", "").replace("-gitlab", "") ), reverse=True, ) return versions def get_git_hash(self, rev: str): return ( subprocess.check_output( [ "nix-prefetch-url", "--unpack", f"https://gitlab.com/{self.owner}/{self.repo}/-/archive/{rev}/{self.repo}-{rev}.tar.gz", ] ) .decode("utf-8") .strip() )
def get_yarn_hash(self, rev: str): with tempfile.TemporaryDirectory() as tmp_dir: with open(tmp_dir + "/yarn.lock", "w") as f: f.write(self.get_file("yarn.lock", rev)) return ( subprocess.check_output(["prefetch-yarn-deps", tmp_dir + "/yarn.lock"]) .decode("utf-8") .strip() )
@staticmethod def rev2version(tag: str) -> str: """ normalize a tag to a version number. This obviously isn't very smart if we don't pass something that looks like a tag :param tag: the tag to normalize :return: a normalized version number """ # strip v prefix version = re.sub(r"^v", "", tag) # strip -ee and -gitlab suffixes return re.sub(r"-(ee|gitlab)$", "", version)
def get_file(self, filepath, rev): """ returns file contents at a given rev :param filepath: the path to the file, relative to the repo root :param rev: the rev to fetch at :return: """ return requests.get(self.url + f"/raw/{rev}/{filepath}").text
def get_data(self, rev): version = self.rev2version(rev)
passthru = { v: self.get_file(v, rev).strip() for v in [ "GITALY_SERVER_VERSION", "GITLAB_PAGES_VERSION", "GITLAB_SHELL_VERSION", "GITLAB_ELASTICSEARCH_INDEXER_VERSION", ] } passthru["GITLAB_WORKHORSE_VERSION"] = version
return dict( version=self.rev2version(rev), repo_hash=self.get_git_hash(rev), yarn_hash=self.get_yarn_hash(rev), owner=self.owner, repo=self.repo, rev=rev, passthru=passthru, )
def _get_data_json(): data_file_path = pathlib.Path(__file__).parent / "data.json" with open(data_file_path, "r") as f: return json.load(f)
def _call_nix_update(pkg, version): """calls nix-update from nixpkgs root dir""" return subprocess.check_output( ["nix-update", pkg, "--version", version], cwd=NIXPKGS_PATH )
@click_log.simple_verbosity_option(logger)@click.group()def cli(): pass
@cli.command("update-data")@click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")def update_data(rev: str): """Update data.json""" logger.info("Updating data.json")
repo = GitLabRepo() if rev == "latest": # filter out pre and rc releases rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
data_file_path = pathlib.Path(__file__).parent / "data.json"
data = repo.get_data(rev)
with open(data_file_path.as_posix(), "w") as f: json.dump(data, f, indent=2) f.write("\n")
@cli.command("update-rubyenv")def update_rubyenv(): """Update rubyEnv""" logger.info("Updating gitlab") repo = GitLabRepo() rubyenv_dir = pathlib.Path(__file__).parent / "rubyEnv"
# load rev from data.json data = _get_data_json() rev = data["rev"] version = data["version"]
for fn in ["Gemfile.lock", "Gemfile"]: with open(rubyenv_dir / fn, "w") as f: f.write(repo.get_file(fn, rev))
# update to 1.2.9 to include https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/commit/5d77f3f3e048834250589b416c6b3d4bba65a570 subprocess.check_output( ["sed", "-i", "s:'prometheus-client-mmap', '~> 1.2.8':'prometheus-client-mmap', '~> 1.2.9':g", "Gemfile"], cwd=rubyenv_dir, )
# Un-vendor sidekiq # # The sidekiq dependency was vendored to maintain compatibility with Redis 6.0 (as # stated in this [comment]) but unfortunately, it seems to cause a crash in the # application, as noted in this [upstream issue]. # # We can safely swap out the dependency, as our Redis release in nixpkgs is >= 7.0. # # [comment]: https://gitlab.com/gitlab-org/gitlab/-/issues/468435#note_1979750600 # [upstream issue]: https://gitlab.com/gitlab-org/gitlab/-/issues/468435 subprocess.check_output( ["sed", "-i", "s|gem 'sidekiq', path: 'vendor/gems/sidekiq', require: 'sidekiq'|gem 'sidekiq', '~> 7.3.9'|g", "Gemfile"], cwd=rubyenv_dir, )
# Fetch vendored dependencies temporarily in order to build the gemset.nix subprocess.check_output(["mkdir", "-p", "vendor/gems", "gems"], cwd=rubyenv_dir) subprocess.check_output( [ "sh", "-c", f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=vendor/gems | tar -xj --strip-components=3", ], cwd=f"{rubyenv_dir}/vendor/gems", ) subprocess.check_output( [ "sh", "-c", f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=gems | tar -xj --strip-components=2", ], cwd=f"{rubyenv_dir}/gems", )
# Undo our gemset.nix patches so that bundix runs through subprocess.check_output( ["sed", "-i", "-e", "s|\\${src}/||g", "gemset.nix"], cwd=rubyenv_dir ) subprocess.check_output( ["sed", "-i", "-e", "s|^src:[[:space:]]||g", "gemset.nix"], cwd=rubyenv_dir )
subprocess.check_output(["bundle", "lock"], cwd=rubyenv_dir) subprocess.check_output(["bundix"], cwd=rubyenv_dir)
subprocess.check_output( [ "sed", "-i", "-e", "1c\\src: {", "-e", 's:path = \\(vendor/[^;]*\\);:path = "${src}/\\1";:g', "-e", 's:path = \\(gems/[^;]*\\);:path = "${src}/\\1";:g', "gemset.nix", ], cwd=rubyenv_dir, ) subprocess.check_output(["rm", "-rf", "vendor", "gems"], cwd=rubyenv_dir)
# Reformat gemset.nix subprocess.check_output(["nix-shell", "--run", "treefmt pkgs/by-name/gi/gitlab"], cwd=NIXPKGS_PATH)
@cli.command("update-gitaly")def update_gitaly(): """Update gitaly""" logger.info("Updating gitaly") data = _get_data_json() gitaly_server_version = data['passthru']['GITALY_SERVER_VERSION'] repo = GitLabRepo(repo="gitaly") gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'
makefile = repo.get_file("Makefile", f"v{gitaly_server_version}") makefile += "\nprint-%:;@echo $($*)\n"
git_version = subprocess.run(["make", "-f", "-", "print-GIT_VERSION"], check=True, input=makefile, text=True, capture_output=True).stdout.strip()
_call_nix_update("gitaly", gitaly_server_version) _call_nix_update("gitaly.git", git_version)
@cli.command("update-gitlab-pages")def update_gitlab_pages(): """Update gitlab-pages""" logger.info("Updating gitlab-pages") data = _get_data_json() gitlab_pages_version = data["passthru"]["GITLAB_PAGES_VERSION"] _call_nix_update("gitlab-pages", gitlab_pages_version)
def get_container_registry_version() -> str: """Returns the version attribute of gitlab-container-registry""" return subprocess.check_output( [ "nix", "--experimental-features", "nix-command", "eval", "-f", ".", "--raw", "gitlab-container-registry.version", ], cwd=NIXPKGS_PATH, ).decode("utf-8")
@cli.command("update-gitlab-shell")def update_gitlab_shell(): """Update gitlab-shell""" logger.info("Updating gitlab-shell") data = _get_data_json() gitlab_shell_version = data["passthru"]["GITLAB_SHELL_VERSION"] _call_nix_update("gitlab-shell", gitlab_shell_version)
@cli.command("update-gitlab-workhorse")def update_gitlab_workhorse(): """Update gitlab-workhorse""" logger.info("Updating gitlab-workhorse") data = _get_data_json() gitlab_workhorse_version = data["passthru"]["GITLAB_WORKHORSE_VERSION"] _call_nix_update("gitlab-workhorse", gitlab_workhorse_version)
@cli.command("update-gitlab-container-registry")@click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")@click.option( "--commit", is_flag=True, default=False, help="Commit the changes for you")def update_gitlab_container_registry(rev: str, commit: bool): """Update gitlab-container-registry""" logger.info("Updading gitlab-container-registry") repo = GitLabRepo(repo="container-registry") old_container_registry_version = get_container_registry_version()
if rev == "latest": rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
version = repo.rev2version(rev) _call_nix_update("gitlab-container-registry", version) if commit: new_container_registry_version = get_container_registry_version() commit_container_registry( old_container_registry_version, new_container_registry_version )
@cli.command('update-gitlab-elasticsearch-indexer')def update_gitlab_elasticsearch_indexer(): """Update gitlab-elasticsearch-indexer""" data = _get_data_json() gitlab_elasticsearch_indexer_version = data['passthru']['GITLAB_ELASTICSEARCH_INDEXER_VERSION'] _call_nix_update('gitlab-elasticsearch-indexer', gitlab_elasticsearch_indexer_version)
@cli.command("update-all")@click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")@click.option( "--commit", is_flag=True, default=False, help="Commit the changes for you")@click.pass_contextdef update_all(ctx, rev: str, commit: bool): """Update all gitlab components to the latest stable release""" old_data_json = _get_data_json() old_container_registry_version = get_container_registry_version()
ctx.invoke(update_data, rev=rev)
new_data_json = _get_data_json()
ctx.invoke(update_rubyenv) ctx.invoke(update_gitaly) ctx.invoke(update_gitlab_pages) ctx.invoke(update_gitlab_shell) ctx.invoke(update_gitlab_workhorse) ctx.invoke(update_gitlab_elasticsearch_indexer) if commit: commit_gitlab( old_data_json["version"], new_data_json["version"], new_data_json["rev"] )
ctx.invoke(update_gitlab_container_registry) if commit: new_container_registry_version = get_container_registry_version() commit_container_registry( old_container_registry_version, new_container_registry_version )
def commit_gitlab(old_version: str, new_version: str, new_rev: str) -> None: """Commits the gitlab changes for you""" subprocess.run( [ "git", "add", "pkgs/by-name/gi/gitlab", "pkgs/by-name/gi/gitaly", "pkgs/by-name/gi/gitlab-elasticsearch-indexer", "pkgs/by-name/gi/gitlab-pages", ], cwd=NIXPKGS_PATH, ) subprocess.run( [ "git", "commit", "--message", f"""gitlab: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/gitlab/-/blob/{new_rev}/CHANGELOG.md""", ], cwd=NIXPKGS_PATH, )
def commit_container_registry(old_version: str, new_version: str) -> None: """Commits the gitlab-container-registry changes for you""" subprocess.run( [ "git", "add", "pkgs/by-name/gi/gitlab-container-registry" ], cwd=NIXPKGS_PATH, ) subprocess.run( [ "git", "commit", "--message", f"gitlab-container-registry: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/container-registry/-/blob/v{new_version}-gitlab/CHANGELOG.md", ], cwd=NIXPKGS_PATH, )
if __name__ == "__main__": cli()