Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
18 kB · 627 lines
Ruby
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628#!/usr/bin/env nix-shell#!nix-shell -i ruby -p "ruby.withPackages (ps: with ps; [ slop curb nokogiri ])"
require 'json'require 'rubygems'require 'shellwords'require 'erb'require 'uri'require 'stringio'require 'slop'require 'curb'require 'nokogiri'
# Returns a repo URL for a given package name.def repo_url value if value && value.start_with?('http') value elsif value "https://dl.google.com/android/repository/#{value}" else nil endend
# Returns a system image URL for a given system image name.def image_url value, dir if dir == "default" dir = "android" end if value && value.start_with?('http') value elsif value "https://dl.google.com/android/repository/sys-img/#{dir}/#{value}" else nil endend
# Runs a GET with curl.def _curl_get url curl = Curl::Easy.new(url) do |http| http.headers['User-Agent'] = 'nixpkgs androidenv update bot' yield http if block_given? end STDERR.print "GET #{url}" curl.perform STDERR.puts "... #{curl.response_code}"
StringIO.new(curl.body_str)end
# Retrieves a repo from the filesystem or a URL.def get location uri = URI.parse(location) case uri.scheme when 'repo' _curl_get repo_url("#{uri.host}#{uri.fragment}.xml") when 'image' _curl_get image_url("sys-img#{uri.fragment}.xml", uri.host) else if File.exist?(uri.path) File.open(uri.path, 'rt') else raise "Repository #{uri} was neither a file nor a repo URL" end endend
# Returns a JSON with the data and structure of the input XMLdef to_json_collector doc json = {} index = 0 doc.element_children.each { |node| if node.children.length == 1 and node.children.first.text? json["#{node.name}:#{index}"] ||= node.content index += 1 next end json["#{node.name}:#{index}"] ||= to_json_collector node index += 1 } element_attributes = {} doc.attribute_nodes.each do |attr| if attr.name == "type" type = attr.value.split(':', 2).last case attr.value when 'generic:genericDetailsType' element_attributes["xsi:type"] ||= "ns5:#{type}" when 'addon:extraDetailsType' element_attributes["xsi:type"] ||= "ns8:#{type}" when 'addon:mavenType' element_attributes["xsi:type"] ||= "ns8:#{type}" when 'sdk:platformDetailsType' element_attributes["xsi:type"] ||= "ns11:#{type}" when 'sdk:sourceDetailsType' element_attributes["xsi:type"] ||= "ns11:#{type}" when 'sys-img:sysImgDetailsType' element_attributes["xsi:type"] ||= "ns12:#{type}" when 'addon:addonDetailsType' then element_attributes["xsi:type"] ||= "ns8:#{type}" end else element_attributes[attr.name] ||= attr.value end end if !element_attributes.empty? json['element-attributes'] ||= element_attributes end jsonend
# Returns a tuple of [type, revision, revision components] for a package node.def package_revision package type_details = package.at_css('> type-details') type = type_details.attributes['type'] type &&= type.value
revision = nil components = nil
case type when 'generic:genericDetailsType', 'addon:extraDetailsType', 'addon:mavenType' major = text package.at_css('> revision > major') minor = text package.at_css('> revision > minor') micro = text package.at_css('> revision > micro') preview = text package.at_css('> revision > preview')
revision = '' components = [] unless empty?(major) revision << major components << major end
unless empty?(minor) revision << ".#{minor}" components << minor end
unless empty?(micro) revision << ".#{micro}" components << micro end
unless empty?(preview) revision << "-rc#{preview}" components << preview end when 'sdk:platformDetailsType' codename = text type_details.at_css('> codename') api_level = text type_details.at_css('> api-level') revision = empty?(codename) ? api_level : codename components = [revision] when 'sdk:sourceDetailsType' api_level = text type_details.at_css('> api-level') revision, components = api_level, [api_level] when 'sys-img:sysImgDetailsType' codename = text type_details.at_css('> codename') api_level = text type_details.at_css('> api-level') id = text type_details.at_css('> tag > id') abi = text type_details.at_css('> abi')
revision = '' components = [] if empty?(codename) revision << api_level components << api_level else revision << codename components << codename end
unless empty?(id) revision << "-#{id}" components << id end
unless empty?(abi) revision << "-#{abi}" components << abi end when 'addon:addonDetailsType' then api_level = text type_details.at_css('> api-level') id = text type_details.at_css('> tag > id') revision = api_level components = [api_level, id] end
[type, revision, components]end
# Returns a hash of archives for the specified package node.def package_archives package archives = {} package.css('> archives > archive').each do |archive| host_os = text archive.at_css('> host-os') host_arch = text archive.at_css('> host-arch') host_os = 'all' if empty?(host_os) host_arch = 'all' if empty?(host_arch) archives[host_os + host_arch] = { 'os' => host_os, 'arch' => host_arch, 'size' => Integer(text(archive.at_css('> complete > size'))), 'sha1' => text(archive.at_css('> complete > checksum')), 'url' => yield(text(archive.at_css('> complete > url'))) } end archivesend
# Returns the text from a node, or nil.def text node node ? node.text : nilend
# Nil or empty helper.def empty? value !value || value.empty?end
# Fixes up returned hashes by converting archives like# (e.g. {'linux' => {'sha1' => ...}, 'macosx' => ...} to# [{'os' => 'linux', 'sha1' => ...}, {'os' => 'macosx', ...}, ...].def fixup value Hash[value.map do |k, v| if k == 'archives' && v.is_a?(Hash) [k, v.map do |os, archive| fixup(archive) end] elsif v.is_a?(Hash) [k, fixup(v)] else [k, v] end end]end
# Today since Unix Epoch, January 1, 1970.def today Time.now.utc.to_i / 24 / 60 / 60end
# The expiration strategy. Expire if the last available day was before the `oldest_valid_day`.def expire_records record, oldest_valid_day if record.is_a?(Hash) if record.has_key?('last-available-day') && record['last-available-day'] < oldest_valid_day return nil end update = {} # This should only happen in the first run of this scrip after adding the `expire_record` function. if record.has_key?('displayName') && !record.has_key?('last-available-day') update['last-available-day'] = today end record.each {|key, value| v = expire_records value, oldest_valid_day update[key] = v if v } update else record endend
# Normalize the specified license text.# See: https://brash-snapper.glitch.me/ for how the munging works.def normalize_license license license = license.dup license.gsub!(/([^\n])\n([^\n])/m, '\1 \2') license.gsub!(/ +/, ' ') license.strip! licenseend
# Gets all license texts, deduplicating them.def get_licenses doc licenses = {} doc.css('license[type="text"]').each do |license_node| license_id = license_node['id'] if license_id licenses[license_id] ||= [] licenses[license_id] |= [normalize_license(text(license_node))] end end licensesend
def parse_package_xml doc licenses = get_licenses doc packages = {} # check https://github.com/NixOS/nixpkgs/issues/373785 extras = {}
doc.css('remotePackage').each do |package| name, _, version = package['path'].partition(';') next if version == 'latest'
is_extras = name == 'extras' if is_extras name = package['path'].tr(';', '-') end
type, revision, _ = package_revision(package) next unless revision
path = package['path'].tr(';', '/') display_name = text package.at_css('> display-name') uses_license = package.at_css('> uses-license') uses_license &&= uses_license['ref'] obsolete ||= package['obsolete'] type_details = to_json_collector package.at_css('> type-details') revision_details = to_json_collector package.at_css('> revision') archives = package_archives(package) {|url| repo_url url} dependencies_xml = package.at_css('> dependencies') dependencies = to_json_collector dependencies_xml if dependencies_xml
if is_extras target = extras component = package['path'] target = (target[component] ||= {}) else target = (packages[name] ||= {}) target = (target[revision] ||= {}) end
target['name'] ||= name target['path'] ||= path target['revision'] ||= revision target['displayName'] ||= display_name target['license'] ||= uses_license if uses_license target['obsolete'] ||= obsolete if obsolete == 'true' target['type-details'] ||= type_details target['revision-details'] ||= revision_details target['dependencies'] ||= dependencies if dependencies target['archives'] ||= {} merge target['archives'], archives target['last-available-day'] = today end
[licenses, packages, extras]end
def parse_image_xml doc licenses = get_licenses doc images = {}
doc.css('remotePackage[path^="system-images;"]').each do |package| type, revision, components = package_revision(package) next unless revision
path = package['path'].tr(';', '/') display_name = text package.at_css('> display-name') uses_license = package.at_css('> uses-license') uses_license &&= uses_license['ref'] obsolete &&= package['obsolete'] type_details = to_json_collector package.at_css('> type-details') revision_details = to_json_collector package.at_css('> revision') archives = package_archives(package) {|url| image_url url, components[-2]} dependencies_xml = package.at_css('> dependencies') dependencies = to_json_collector dependencies_xml if dependencies_xml
target = images components.each do |component| target[component] ||= {} target = target[component] end
target['name'] ||= "system-image-#{revision}" target['path'] ||= path target['revision'] ||= revision target['displayName'] ||= display_name target['license'] ||= uses_license if uses_license target['obsolete'] ||= obsolete if obsolete target['type-details'] ||= type_details target['revision-details'] ||= revision_details target['dependencies'] ||= dependencies if dependencies target['archives'] ||= {} merge target['archives'], archives target['last-available-day'] = today end
[licenses, images]end
def parse_addon_xml doc licenses = get_licenses doc addons, extras = {}, {}
doc.css('remotePackage').each do |package| type, revision, components = package_revision(package) next unless revision
path = package['path'].tr(';', '/') display_name = text package.at_css('> display-name') uses_license = package.at_css('> uses-license') uses_license &&= uses_license['ref'] obsolete &&= package['obsolete'] type_details = to_json_collector package.at_css('> type-details') revision_details = to_json_collector package.at_css('> revision') archives = package_archives(package) {|url| repo_url url} dependencies_xml = package.at_css('> dependencies') dependencies = to_json_collector dependencies_xml if dependencies_xml
case type when 'addon:addonDetailsType' name = components.last target = addons
# Hack for Google APIs 25 r1, which displays as 23 for some reason archive_name = text package.at_css('> archives > archive > complete > url') if archive_name == 'google_apis-25_r1.zip' path = 'add-ons/addon-google_apis-google-25' revision = '25' components = [revision, components.last] end when 'addon:extraDetailsType', 'addon:mavenType' name = package['path'].tr(';', '-') components = [package['path']] target = extras end
components.each do |component| target = (target[component] ||= {}) end
target['name'] ||= name target['path'] ||= path target['revision'] ||= revision target['displayName'] ||= display_name target['license'] ||= uses_license if uses_license target['obsolete'] ||= obsolete if obsolete target['type-details'] ||= type_details target['revision-details'] ||= revision_details target['dependencies'] ||= dependencies if dependencies target['archives'] ||= {} merge target['archives'], archives target['last-available-day'] = today end
[licenses, addons, extras]end
# Make the clean diff by always sorting the result before puting it in the stdout.def sort_recursively value if value.is_a?(Hash) Hash[ value.map do |k, v| [k, sort_recursively(v)] end.sort_by {|(k, v)| k } ] elsif value.is_a?(Array) value.map do |v| sort_recursively(v) end else value endend
def merge_recursively a, b a.merge!(b) {|key, a_item, b_item| if a_item.is_a?(Hash) && b_item.is_a?(Hash) merge_recursively(a_item, b_item) elsif b_item != nil b_item end } aend
def merge dest, src merge_recursively dest, srcend
opts = Slop.parse do |o| o.array '-p', '--packages', 'packages repo XMLs to parse', default: %w[repo://repository#2-3] o.array '-i', '--images', 'system image repo XMLs to parse', default: %w[ image://android#2-3 image://android-tv#2-3 image://android-wear#2-3 image://android-wear-cn#2-3 image://android-automotive#2-3 image://google_apis#2-3 image://google_apis_playstore#2-3 ] o.array '-a', '--addons', 'addon repo XMLs to parse', default: %w[repo://addon#2-3] o.string '-I', '--input', 'input JSON file for repo', default: File.join(__dir__, 'repo.json') o.string '-O', '--output', 'output JSON file for repo', default: File.join(__dir__, 'repo.json')end
result = {}result['licenses'] = {}result['packages'] = {}result['images'] = {}result['addons'] = {}result['extras'] = {}
opts[:packages].each do |filename| licenses, packages, extras = parse_package_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks }) merge result['licenses'], licenses merge result['packages'], packages merge result['extras'], extrasend
opts[:images].each do |filename| licenses, images = parse_image_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks }) merge result['licenses'], licenses merge result['images'], imagesend
opts[:addons].each do |filename| licenses, addons, extras = parse_addon_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks }) merge result['licenses'], licenses merge result['addons'], addons merge result['extras'], extrasend
result['latest'] = {}result['packages'].each do |name, versions| max_version = Gem::Version.new('0') versions.each do |version, package| if package['license'] == 'android-sdk-license' && Gem::Version.correct?(package['revision']) package_version = Gem::Version.new(package['revision']) max_version = package_version if package_version > max_version end end result['latest'][name] = max_version.to_send
# As we keep the old packages in the repo JSON file, we should have# a strategy to remove them at some point!# So with this variable we claim it's okay to remove them from the# JSON after two years that they are not available.two_years_ago = today - 2 * 365
input = {}prev_latest = {}begin input_json = if File.exist?(opts[:input]) STDERR.puts "Reading #{opts[:input]}" File.read(opts[:input]) else STDERR.puts "Creating new repo" "{}" end
if input_json != nil && !input_json.empty? input = expire_records(JSON.parse(input_json), two_years_ago)
# Just create a new set of latest packages. prev_latest = input['latest'] || {} input['latest'] = {} endrescue JSON::ParserError => e STDERR.write(e.message) returnend
fixup_result = fixup(result)
# Regular installation of Android SDK would keep the previously installed packages even if they are not# in the uptodate XML files, so here we try to support this logic by keeping un-available packages,# therefore the old packages will work as long as the links are working on the Google servers.output = merge input, fixup_result
# Write the repository. Append a \n to keep nixpkgs Github Actions happy.STDERR.puts "Writing #{opts[:output]}"File.write opts[:output], (JSON.pretty_generate(sort_recursively(output)) + "\n")
# Output metadata for the nixpkgs update script.if ENV['UPDATE_NIX_ATTR_PATH'] # See if there are any changes in the latest versions. cur_latest = output['latest'] || {}
old_versions = [] new_versions = [] changes = [] changed = false
cur_latest.each do |k, v| prev = prev_latest[k] if prev && prev != v old_versions << "#{k}:#{prev}" new_versions << "#{k}:#{v}" changes << "#{k}: #{prev} -> #{v}" changed = true end end
changed_paths = [] if changed # Instantiate it. test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build #{Shellwords.escape(File.realpath(File.join(__dir__, '..', '..', '..', '..', 'default.nix')))} -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1` test_status = $?.exitstatus
template = ERB.new(<<-EOF, trim_mode: '<>-')androidenv: <%= changes.join('; ') %>
Performed the following automatic androidenv updates:
<% changes.each do |change| %>- <%= change -%><% end %>
Tests exited with status: <%= test_status -%>
<% if !test_result.empty? %>Last 100 lines of output:```<%= test_result.lines.last(100).join -%>```<% end %>EOF
changed_paths << { attrPath: 'androidenv.androidPkgs.androidsdk', oldVersion: old_versions.join('; '), newVersion: new_versions.join('; '), files: [ opts[:output] ], commitMessage: template.result(binding) } end
# nix-update info is on stdout STDOUT.puts JSON.pretty_generate(changed_paths)end