Something went wrong. Try again.
Git fork
Something went wrong. Try again.
git commit-graph.c
79 kB · 2921 lines
C
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922#define DISABLE_SIGN_COMPARE_WARNINGS
#include "git-compat-util.h"#include "config.h"#include "csum-file.h"#include "environment.h"#include "gettext.h"#include "hex.h"#include "lockfile.h"#include "packfile.h"#include "commit.h"#include "object.h"#include "refs.h"#include "hash-lookup.h"#include "commit-graph.h"#include "odb.h"#include "oid-array.h"#include "path.h"#include "alloc.h"#include "hashmap.h"#include "replace-object.h"#include "progress.h"#include "bloom.h"#include "commit-slab.h"#include "shallow.h"#include "json-writer.h"#include "trace2.h"#include "tree.h"#include "chunk-format.h"
void git_test_write_commit_graph_or_die(struct odb_source *source){ int flags = 0; if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) return;
if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS;
if (write_commit_graph_reachable(source, flags, NULL)) die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH");}
#define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */#define GRAPH_CHUNKID_GENERATION_DATA 0x47444132 /* "GDA2" */#define GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW 0x47444f32 /* "GDO2" */#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
#define GRAPH_VERSION_1 0x1#define GRAPH_VERSION GRAPH_VERSION_1
#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000#define GRAPH_EDGE_LAST_MASK 0x7fffffff#define GRAPH_PARENT_NONE 0x70000000
#define GRAPH_LAST_EDGE 0x80000000
#define GRAPH_HEADER_SIZE 8#define GRAPH_FANOUT_SIZE (4 * 256)
#define CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW (1ULL << 31)
/* Remember to update object flag allocation in object.h */#define REACHABLE (1u<<15)
define_commit_slab(topo_level_slab, uint32_t);
/* Keep track of the order in which commits are added to our list. */define_commit_slab(commit_pos, int);static struct commit_pos commit_pos = COMMIT_SLAB_INIT(1, commit_pos);
static size_t graph_data_width(const struct git_hash_algo *algop){ return algop->rawsz + 16;}
static size_t graph_min_size(const struct git_hash_algo *algop){ return GRAPH_HEADER_SIZE + 4 * CHUNK_TOC_ENTRY_SIZE + GRAPH_FANOUT_SIZE + algop->rawsz;}
static void set_commit_pos(struct repository *r, const struct object_id *oid){ static int32_t max_pos; struct commit *commit = lookup_commit(r, oid);
if (!commit) return; /* should never happen, but be lenient */
*commit_pos_at(&commit_pos, commit) = max_pos++;}
static int commit_pos_cmp(const void *va, const void *vb){ const struct commit *a = *(const struct commit **)va; const struct commit *b = *(const struct commit **)vb; return commit_pos_at(&commit_pos, a) - commit_pos_at(&commit_pos, b);}
define_commit_slab(commit_graph_data_slab, struct commit_graph_data);static struct commit_graph_data_slab commit_graph_data_slab = COMMIT_SLAB_INIT(1, commit_graph_data_slab);
static int get_configured_generation_version(struct repository *r){ int version = 2; repo_config_get_int(r, "commitgraph.generationversion", &version); return version;}
uint32_t commit_graph_position(const struct commit *c){ struct commit_graph_data *data = commit_graph_data_slab_peek(&commit_graph_data_slab, c);
return data ? data->graph_pos : COMMIT_NOT_FROM_GRAPH;}
timestamp_t commit_graph_generation(const struct commit *c){ struct commit_graph_data *data = commit_graph_data_slab_peek(&commit_graph_data_slab, c);
if (data && data->generation) return data->generation;
return GENERATION_NUMBER_INFINITY;}
static timestamp_t commit_graph_generation_from_graph(const struct commit *c){ struct commit_graph_data *data = commit_graph_data_slab_peek(&commit_graph_data_slab, c);
if (!data || data->graph_pos == COMMIT_NOT_FROM_GRAPH) return GENERATION_NUMBER_INFINITY; return data->generation;}
static struct commit_graph_data *commit_graph_data_at(const struct commit *c){ unsigned int i, nth_slab; struct commit_graph_data *data = commit_graph_data_slab_peek(&commit_graph_data_slab, c);
if (data) return data;
nth_slab = c->index / commit_graph_data_slab.slab_size; data = commit_graph_data_slab_at(&commit_graph_data_slab, c);
/* * commit-slab initializes elements with zero, overwrite this with * COMMIT_NOT_FROM_GRAPH for graph_pos. * * We avoid initializing generation with checking if graph position * is not COMMIT_NOT_FROM_GRAPH. */ for (i = 0; i < commit_graph_data_slab.slab_size; i++) { commit_graph_data_slab.slab[nth_slab][i].graph_pos = COMMIT_NOT_FROM_GRAPH; }
return data;}
/* * Should be used only while writing commit-graph as it compares * generation value of commits by directly accessing commit-slab. */static int commit_gen_cmp(const void *va, const void *vb){ const struct commit *a = *(const struct commit **)va; const struct commit *b = *(const struct commit **)vb;
const timestamp_t generation_a = commit_graph_data_at(a)->generation; const timestamp_t generation_b = commit_graph_data_at(b)->generation; /* lower generation commits first */ if (generation_a < generation_b) return -1; else if (generation_a > generation_b) return 1;
/* use date as a heuristic when generations are equal */ if (a->date < b->date) return -1; else if (a->date > b->date) return 1; return 0;}
char *get_commit_graph_filename(struct odb_source *source){ return xstrfmt("%s/info/commit-graph", source->path);}
static char *get_split_graph_filename(struct odb_source *source, const char *oid_hex){ return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path, oid_hex);}
char *get_commit_graph_chain_filename(struct odb_source *source){ return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path);}
static struct commit_graph *alloc_commit_graph(void){ struct commit_graph *g = xcalloc(1, sizeof(*g));
return g;}
static int commit_graph_compatible(struct repository *r){ if (!r->gitdir) return 0;
if (replace_refs_enabled(r)) { prepare_replace_object(r); if (oidmap_get_size(&r->objects->replace_map)) return 0; }
prepare_commit_graft(r); if (r->parsed_objects && (r->parsed_objects->grafts_nr || r->parsed_objects->substituted_parent)) return 0; if (is_repository_shallow(r)) return 0;
return 1;}
int open_commit_graph(const char *graph_file, int *fd, struct stat *st){ *fd = git_open(graph_file); if (*fd < 0) return 0; if (fstat(*fd, st)) { close(*fd); return 0; } return 1;}
struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, int fd, struct stat *st){ void *graph_map; size_t graph_size; struct commit_graph *ret;
graph_size = xsize_t(st->st_size);
if (graph_size < graph_min_size(source->odb->repo->hash_algo)) { close(fd); error(_("commit-graph file is too small")); return NULL; } graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd);
ret = parse_commit_graph(source->odb->repo, graph_map, graph_size); if (ret) ret->odb_source = source; else munmap(graph_map, graph_size);
return ret;}
static int graph_read_oid_fanout(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data; int i;
if (chunk_size != 256 * sizeof(uint32_t)) return error(_("commit-graph oid fanout chunk is wrong size")); g->chunk_oid_fanout = (const uint32_t *)chunk_start; g->num_commits = ntohl(g->chunk_oid_fanout[255]);
for (i = 0; i < 255; i++) { uint32_t oid_fanout1 = ntohl(g->chunk_oid_fanout[i]); uint32_t oid_fanout2 = ntohl(g->chunk_oid_fanout[i + 1]);
if (oid_fanout1 > oid_fanout2) { error(_("commit-graph fanout values out of order")); return 1; } }
return 0;}
static int graph_read_oid_lookup(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data; g->chunk_oid_lookup = chunk_start; if (chunk_size / g->hash_algo->rawsz != g->num_commits) return error(_("commit-graph OID lookup chunk is the wrong size")); return 0;}
static int graph_read_commit_data(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data; if (chunk_size / graph_data_width(g->hash_algo) != g->num_commits) return error(_("commit-graph commit data chunk is wrong size")); g->chunk_commit_data = chunk_start; return 0;}
static int graph_read_generation_data(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data; if (chunk_size / sizeof(uint32_t) != g->num_commits) return error(_("commit-graph generations chunk is wrong size")); g->chunk_generation_data = chunk_start; return 0;}
static int graph_read_bloom_index(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data; if (chunk_size / 4 != g->num_commits) { warning(_("commit-graph changed-path index chunk is too small")); return -1; } g->chunk_bloom_indexes = chunk_start; return 0;}
static int graph_read_bloom_data(const unsigned char *chunk_start, size_t chunk_size, void *data){ struct commit_graph *g = data;
if (chunk_size < BLOOMDATA_CHUNK_HEADER_SIZE) { warning(_("ignoring too-small changed-path chunk" " (%"PRIuMAX" < %"PRIuMAX") in commit-graph file"), (uintmax_t)chunk_size, (uintmax_t)BLOOMDATA_CHUNK_HEADER_SIZE); return -1; }
g->chunk_bloom_data = chunk_start; g->chunk_bloom_data_size = chunk_size;
g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings)); g->bloom_filter_settings->hash_version = get_be32(chunk_start); g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4); g->bloom_filter_settings->bits_per_entry = get_be32(chunk_start + 8); g->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES;
return 0;}
struct commit_graph *parse_commit_graph(struct repository *r, void *graph_map, size_t graph_size){ const unsigned char *data; struct commit_graph *graph; uint32_t graph_signature; unsigned char graph_version, hash_version; struct chunkfile *cf = NULL;
if (!graph_map) return NULL;
if (graph_size < graph_min_size(r->hash_algo)) return NULL;
data = (const unsigned char *)graph_map;
graph_signature = get_be32(data); if (graph_signature != GRAPH_SIGNATURE) { error(_("commit-graph signature %X does not match signature %X"), graph_signature, GRAPH_SIGNATURE); return NULL; }
graph_version = *(unsigned char*)(data + 4); if (graph_version != GRAPH_VERSION) { error(_("commit-graph version %X does not match version %X"), graph_version, GRAPH_VERSION); return NULL; }
hash_version = *(unsigned char*)(data + 5); if (hash_version != oid_version(r->hash_algo)) { error(_("commit-graph hash version %X does not match version %X"), hash_version, oid_version(r->hash_algo)); return NULL; }
graph = alloc_commit_graph();
graph->hash_algo = r->hash_algo; graph->num_chunks = *(unsigned char*)(data + 6); graph->data = graph_map; graph->data_len = graph_size;
if (graph_size < GRAPH_HEADER_SIZE + (graph->num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE + GRAPH_FANOUT_SIZE + r->hash_algo->rawsz) { error(_("commit-graph file is too small to hold %u chunks"), graph->num_chunks); free(graph); return NULL; }
cf = init_chunkfile(NULL);
if (read_table_of_contents(cf, graph->data, graph_size, GRAPH_HEADER_SIZE, graph->num_chunks, 1)) goto free_and_return;
if (read_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, graph_read_oid_fanout, graph)) { error(_("commit-graph required OID fanout chunk missing or corrupted")); goto free_and_return; } if (read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph)) { error(_("commit-graph required OID lookup chunk missing or corrupted")); goto free_and_return; } if (read_chunk(cf, GRAPH_CHUNKID_DATA, graph_read_commit_data, graph)) { error(_("commit-graph required commit data chunk missing or corrupted")); goto free_and_return; }
pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges, &graph->chunk_extra_edges_size); pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs, &graph->chunk_base_graphs_size);
prepare_repo_settings(r);
if (r->settings.commit_graph_generation_version >= 2) { read_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, graph_read_generation_data, graph); pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, &graph->chunk_generation_data_overflow, &graph->chunk_generation_data_overflow_size);
if (graph->chunk_generation_data) graph->read_generation_data = 1; }
if (r->settings.commit_graph_changed_paths_version) { read_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, graph_read_bloom_index, graph); read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, graph_read_bloom_data, graph); }
if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) { init_bloom_filters(); } else { /* We need both the bloom chunks to exist together. Else ignore the data */ graph->chunk_bloom_indexes = NULL; graph->chunk_bloom_data = NULL; FREE_AND_NULL(graph->bloom_filter_settings); }
oidread(&graph->oid, graph->data + graph->data_len - graph->hash_algo->rawsz, r->hash_algo);
free_chunkfile(cf); return graph;
free_and_return: free_chunkfile(cf); free(graph->bloom_filter_settings); free(graph); return NULL;}
static struct commit_graph *load_commit_graph_one(struct odb_source *source, const char *graph_file){ struct stat st; int fd; struct commit_graph *g; int open_ok = open_commit_graph(graph_file, &fd, &st);
if (!open_ok) return NULL;
g = load_commit_graph_one_fd_st(source, fd, &st); if (g) g->filename = xstrdup(graph_file);
return g;}
static struct commit_graph *load_commit_graph_v1(struct odb_source *source){ char *graph_name = get_commit_graph_filename(source); struct commit_graph *g = load_commit_graph_one(source, graph_name); free(graph_name);
return g;}
/* * returns 1 if and only if all graphs in the chain have * corrected commit dates stored in the generation_data chunk. */static int validate_mixed_generation_chain(struct commit_graph *g){ int read_generation_data = 1; struct commit_graph *p = g;
while (read_generation_data && p) { read_generation_data = p->read_generation_data; p = p->base_graph; }
if (read_generation_data) return 1;
while (g) { g->read_generation_data = 0; g = g->base_graph; }
return 0;}
static void validate_mixed_bloom_settings(struct commit_graph *g){ struct bloom_filter_settings *settings = NULL; for (; g; g = g->base_graph) { if (!g->bloom_filter_settings) continue; if (!settings) { settings = g->bloom_filter_settings; continue; }
if (g->bloom_filter_settings->bits_per_entry != settings->bits_per_entry || g->bloom_filter_settings->num_hashes != settings->num_hashes || g->bloom_filter_settings->hash_version != settings->hash_version) { g->chunk_bloom_indexes = NULL; g->chunk_bloom_data = NULL; FREE_AND_NULL(g->bloom_filter_settings);
warning(_("disabling Bloom filters for commit-graph " "layer '%s' due to incompatible settings"), oid_to_hex(&g->oid)); } }}
static int add_graph_to_chain(struct commit_graph *g, struct commit_graph *chain, struct object_id *oids, int n){ struct commit_graph *cur_g = chain;
if (n && !g->chunk_base_graphs) { warning(_("commit-graph has no base graphs chunk")); return 0; }
if (g->chunk_base_graphs_size / g->hash_algo->rawsz < n) { warning(_("commit-graph base graphs chunk is too small")); return 0; }
while (n) { n--;
if (!cur_g || !oideq(&oids[n], &cur_g->oid) || !hasheq(oids[n].hash, g->chunk_base_graphs + st_mult(g->hash_algo->rawsz, n), g->hash_algo)) { warning(_("commit-graph chain does not match")); return 0; }
cur_g = cur_g->base_graph; }
if (chain) { if (unsigned_add_overflows(chain->num_commits, chain->num_commits_in_base)) { warning(_("commit count in base graph too high: %"PRIuMAX), (uintmax_t)chain->num_commits_in_base); return 0; } g->num_commits_in_base = chain->num_commits + chain->num_commits_in_base; }
g->base_graph = chain;
return 1;}
int open_commit_graph_chain(const char *chain_file, int *fd, struct stat *st, const struct git_hash_algo *hash_algo){ *fd = git_open(chain_file); if (*fd < 0) return 0; if (fstat(*fd, st)) { close(*fd); return 0; } if (st->st_size < hash_algo->hexsz) { close(*fd); if (!st->st_size) { /* treat empty files the same as missing */ errno = ENOENT; } else { warning(_("commit-graph chain file too small")); errno = EINVAL; } return 0; } return 1;}
struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, int fd, struct stat *st, int *incomplete_chain){ struct commit_graph *graph_chain = NULL; struct strbuf line = STRBUF_INIT; struct object_id *oids; int i = 0, valid = 1, count; FILE *fp = xfdopen(fd, "r");
count = st->st_size / (odb->repo->hash_algo->hexsz + 1); CALLOC_ARRAY(oids, count);
odb_prepare_alternates(odb);
for (i = 0; i < count; i++) { struct odb_source *source;
if (strbuf_getline_lf(&line, fp) == EOF) break;
if (get_oid_hex_algop(line.buf, &oids[i], odb->repo->hash_algo)) { warning(_("invalid commit-graph chain: line '%s' not a hash"), line.buf); valid = 0; break; }
valid = 0; for (source = odb->sources; source; source = source->next) { char *graph_name = get_split_graph_filename(source, line.buf); struct commit_graph *g = load_commit_graph_one(source, graph_name);
free(graph_name);
if (g) { if (add_graph_to_chain(g, graph_chain, oids, i)) { graph_chain = g; valid = 1; } else { free_commit_graph(g); }
break; } }
if (!valid) { warning(_("unable to find all commit-graph files")); break; } }
validate_mixed_generation_chain(graph_chain); validate_mixed_bloom_settings(graph_chain);
free(oids); fclose(fp); strbuf_release(&line);
*incomplete_chain = !valid; return graph_chain;}
static struct commit_graph *load_commit_graph_chain(struct odb_source *source){ char *chain_file = get_commit_graph_chain_filename(source); struct stat st; int fd; struct commit_graph *g = NULL;
if (open_commit_graph_chain(chain_file, &fd, &st, source->odb->repo->hash_algo)) { int incomplete; /* ownership of fd is taken over by load function */ g = load_commit_graph_chain_fd_st(source->odb, fd, &st, &incomplete); }
free(chain_file); return g;}
struct commit_graph *read_commit_graph_one(struct odb_source *source){ struct commit_graph *g = load_commit_graph_v1(source);
if (!g) g = load_commit_graph_chain(source);
return g;}
/* * Return 1 if commit_graph is non-NULL, and 0 otherwise. * * On the first invocation, this function attempts to load the commit * graph if the repository is configured to have one. */static struct commit_graph *prepare_commit_graph(struct repository *r){ struct odb_source *source;
/* * Early return if there is no git dir or if the commit graph is * disabled. * * This must come before the "already attempted?" check below, because * we want to disable even an already-loaded graph file. */ if (!r->gitdir || r->commit_graph_disabled) return NULL;
if (r->objects->commit_graph_attempted) return r->objects->commit_graph; r->objects->commit_graph_attempted = 1;
prepare_repo_settings(r);
if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && r->settings.core_commit_graph != 1) /* * This repository is not configured to use commit graphs, so * do not load one. (But report commit_graph_attempted anyway * so that commit graph loading is not attempted again for this * repository.) */ return NULL;
if (!commit_graph_compatible(r)) return NULL;
odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { r->objects->commit_graph = read_commit_graph_one(source); if (r->objects->commit_graph) break; }
return r->objects->commit_graph;}
int generation_numbers_enabled(struct repository *r){ uint32_t first_generation; struct commit_graph *g;
g = prepare_commit_graph(r); if (!g || !g->num_commits) return 0;
first_generation = get_be32(g->chunk_commit_data + g->hash_algo->rawsz + 8) >> 2;
return !!first_generation;}
int corrected_commit_dates_enabled(struct repository *r){ struct commit_graph *g;
g = prepare_commit_graph(r); if (!g || !g->num_commits) return 0;
return g->read_generation_data;}
struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r){ struct commit_graph *g;
if (!prepare_commit_graph(r)) return NULL;
g = r->objects->commit_graph; while (g) { if (g->bloom_filter_settings) return g->bloom_filter_settings; g = g->base_graph; } return NULL;}
void close_commit_graph(struct object_database *o){ if (!o->commit_graph) return;
clear_commit_graph_data_slab(&commit_graph_data_slab); deinit_bloom_filters(); free_commit_graph(o->commit_graph); o->commit_graph = NULL;}
static int bsearch_graph(struct commit_graph *g, const struct object_id *oid, uint32_t *pos){ return bsearch_hash(oid->hash, g->chunk_oid_fanout, g->chunk_oid_lookup, g->hash_algo->rawsz, pos);}
static void load_oid_from_graph(struct commit_graph *g, uint32_t pos, struct object_id *oid){ uint32_t lex_index;
while (g && pos < g->num_commits_in_base) g = g->base_graph;
if (!g) BUG("NULL commit-graph");
if (pos >= g->num_commits + g->num_commits_in_base) die(_("invalid commit position. commit-graph is likely corrupt"));
lex_index = pos - g->num_commits_in_base;
oidread(oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, lex_index), g->hash_algo);}
static struct commit_list **insert_parent_or_die(struct commit_graph *g, uint32_t pos, struct commit_list **pptr){ struct commit *c; struct object_id oid;
if (pos >= g->num_commits + g->num_commits_in_base) die("invalid parent position %"PRIu32, pos);
load_oid_from_graph(g, pos, &oid); c = lookup_commit(g->odb_source->odb->repo, &oid); if (!c) die(_("could not find commit %s"), oid_to_hex(&oid)); commit_graph_data_at(c)->graph_pos = pos; return &commit_list_insert(c, pptr)->next;}
static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos){ const unsigned char *commit_data; struct commit_graph_data *graph_data; uint32_t lex_index, offset_pos; uint64_t date_high, date_low, offset;
while (pos < g->num_commits_in_base) g = g->base_graph;
if (pos >= g->num_commits + g->num_commits_in_base) die(_("invalid commit position. commit-graph is likely corrupt"));
lex_index = pos - g->num_commits_in_base; commit_data = g->chunk_commit_data + st_mult(graph_data_width(g->hash_algo), lex_index);
graph_data = commit_graph_data_at(item); graph_data->graph_pos = pos;
date_high = get_be32(commit_data + g->hash_algo->rawsz + 8) & 0x3; date_low = get_be32(commit_data + g->hash_algo->rawsz + 12); item->date = (timestamp_t)((date_high << 32) | date_low);
if (g->read_generation_data) { offset = (timestamp_t)get_be32(g->chunk_generation_data + st_mult(sizeof(uint32_t), lex_index));
if (offset & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) { if (!g->chunk_generation_data_overflow) die(_("commit-graph requires overflow generation data but has none"));
offset_pos = offset ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW; if (g->chunk_generation_data_overflow_size / sizeof(uint64_t) <= offset_pos) die(_("commit-graph overflow generation data is too small")); graph_data->generation = item->date + get_be64(g->chunk_generation_data_overflow + sizeof(uint64_t) * offset_pos); } else graph_data->generation = item->date + offset; } else graph_data->generation = get_be32(commit_data + g->hash_algo->rawsz + 8) >> 2;
if (g->topo_levels) *topo_level_slab_at(g->topo_levels, item) = get_be32(commit_data + g->hash_algo->rawsz + 8) >> 2;}
static inline void set_commit_tree(struct commit *c, struct tree *t){ c->maybe_tree = t;}
static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos){ uint32_t edge_value; uint32_t parent_data_pos; struct commit_list **pptr; const unsigned char *commit_data; uint32_t lex_index;
while (pos < g->num_commits_in_base) g = g->base_graph;
fill_commit_graph_info(item, g, pos);
lex_index = pos - g->num_commits_in_base; commit_data = g->chunk_commit_data + st_mult(g->hash_algo->rawsz + 16, lex_index);
item->object.parsed = 1;
set_commit_tree(item, NULL);
pptr = &item->parents;
edge_value = get_be32(commit_data + g->hash_algo->rawsz); if (edge_value == GRAPH_PARENT_NONE) return 1; pptr = insert_parent_or_die(g, edge_value, pptr);
edge_value = get_be32(commit_data + g->hash_algo->rawsz + 4); if (edge_value == GRAPH_PARENT_NONE) return 1; if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) { pptr = insert_parent_or_die(g, edge_value, pptr); return 1; }
parent_data_pos = edge_value & GRAPH_EDGE_LAST_MASK; do { if (g->chunk_extra_edges_size / sizeof(uint32_t) <= parent_data_pos) { error(_("commit-graph extra-edges pointer out of bounds")); free_commit_list(item->parents); item->parents = NULL; item->object.parsed = 0; return 0; } edge_value = get_be32(g->chunk_extra_edges + sizeof(uint32_t) * parent_data_pos); pptr = insert_parent_or_die(g, edge_value & GRAPH_EDGE_LAST_MASK, pptr); parent_data_pos++; } while (!(edge_value & GRAPH_LAST_EDGE));
return 1;}
static int search_commit_pos_in_graph(const struct object_id *id, struct commit_graph *g, uint32_t *pos){ struct commit_graph *cur_g = g; uint32_t lex_index;
while (cur_g && !bsearch_graph(cur_g, id, &lex_index)) cur_g = cur_g->base_graph;
if (cur_g) { *pos = lex_index + cur_g->num_commits_in_base; return 1; }
return 0;}
static int find_commit_pos_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos){ uint32_t graph_pos = commit_graph_position(item); if (graph_pos != COMMIT_NOT_FROM_GRAPH) { *pos = graph_pos; return 1; } else { return search_commit_pos_in_graph(&item->object.oid, g, pos); }}
struct commit_graph *repo_find_commit_pos_in_graph(struct repository *r, struct commit *c, uint32_t *pos){ struct commit_graph *g = prepare_commit_graph(r); if (!g) return NULL; if (!find_commit_pos_in_graph(c, g, pos)) return NULL; return g;}
struct commit *lookup_commit_in_graph(struct repository *repo, const struct object_id *id){ static int commit_graph_paranoia = -1; struct commit_graph *g; struct commit *commit; uint32_t pos;
if (commit_graph_paranoia == -1) commit_graph_paranoia = git_env_bool(GIT_COMMIT_GRAPH_PARANOIA, 0);
g = prepare_commit_graph(repo); if (!g) return NULL; if (!search_commit_pos_in_graph(id, g, &pos)) return NULL; if (commit_graph_paranoia && !odb_has_object(repo->objects, id, 0)) return NULL;
commit = lookup_commit(repo, id); if (!commit) return NULL; if (commit->object.parsed) return commit;
if (!fill_commit_in_graph(commit, g, pos)) return NULL;
return commit;}
static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item){ uint32_t pos;
if (item->object.parsed) return 1;
if (find_commit_pos_in_graph(item, g, &pos)) return fill_commit_in_graph(item, g, pos);
return 0;}
int parse_commit_in_graph(struct repository *r, struct commit *item){ static int checked_env = 0; struct commit_graph *g;
if (!checked_env && git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE, 0)) die("dying as requested by the '%s' variable on commit-graph parse!", GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE); checked_env = 1;
g = prepare_commit_graph(r); if (!g) return 0; return parse_commit_in_graph_one(g, item);}
void load_commit_graph_info(struct repository *r, struct commit *item){ struct commit_graph *g; uint32_t pos;
g = repo_find_commit_pos_in_graph(r, item, &pos); if (g) fill_commit_graph_info(item, g, pos);}
static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c){ struct object_id oid; const unsigned char *commit_data; uint32_t graph_pos = commit_graph_position(c);
while (graph_pos < g->num_commits_in_base) g = g->base_graph;
commit_data = g->chunk_commit_data + st_mult(graph_data_width(g->hash_algo), graph_pos - g->num_commits_in_base);
oidread(&oid, commit_data, g->hash_algo); set_commit_tree(c, lookup_tree(g->odb_source->odb->repo, &oid));
return c->maybe_tree;}
static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g, const struct commit *c){ if (c->maybe_tree) return c->maybe_tree; if (commit_graph_position(c) == COMMIT_NOT_FROM_GRAPH) BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
return load_tree_for_commit(g, (struct commit *)c);}
struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c){ return get_commit_tree_in_graph_one(r->objects->commit_graph, c);}
struct packed_commit_list { struct commit **list; size_t nr; size_t alloc;};
struct write_commit_graph_context { struct repository *r; struct odb_source *odb_source; char *graph_name; struct oid_array oids; struct packed_commit_list commits; int num_extra_edges; int num_generation_data_overflows; unsigned long approx_nr_objects; struct progress *progress; int progress_done; uint64_t progress_cnt;
char *base_graph_name; int num_commit_graphs_before; int num_commit_graphs_after; char **commit_graph_filenames_before; char **commit_graph_filenames_after; char **commit_graph_hash_after; uint32_t new_num_commits_in_base; struct commit_graph *new_base_graph;
unsigned append:1, report_progress:1, split:1, changed_paths:1, order_by_pack:1, write_generation_data:1, trust_generation_numbers:1;
struct topo_level_slab *topo_levels; const struct commit_graph_opts *opts; size_t total_bloom_filter_data_size; const struct bloom_filter_settings *bloom_settings;
int count_bloom_filter_computed; int count_bloom_filter_not_computed; int count_bloom_filter_trunc_empty; int count_bloom_filter_trunc_large; int count_bloom_filter_upgraded;};
static int write_graph_chunk_fanout(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; int i, count = 0; struct commit **list = ctx->commits.list;
/* * Write the first-level table (the list is sorted, * but we use a 256-entry lookup to be able to avoid * having to do eight extra binary search iterations). */ for (i = 0; i < 256; i++) { while (count < ctx->commits.nr) { if ((*list)->object.oid.hash[0] != i) break; display_progress(ctx->progress, ++ctx->progress_cnt); count++; list++; }
hashwrite_be32(f, count); }
return 0;}
static int write_graph_chunk_oids(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; struct commit **list = ctx->commits.list; int count; for (count = 0; count < ctx->commits.nr; count++, list++) { display_progress(ctx->progress, ++ctx->progress_cnt); hashwrite(f, (*list)->object.oid.hash, f->algop->rawsz); }
return 0;}
static const struct object_id *commit_to_oid(size_t index, const void *table){ const struct commit * const *commits = table; return &commits[index]->object.oid;}
static int write_graph_chunk_data(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; struct commit **list = ctx->commits.list; struct commit **last = ctx->commits.list + ctx->commits.nr; uint32_t num_extra_edges = 0;
while (list < last) { struct commit_list *parent; struct object_id *tree; int edge_value; uint32_t packedDate[2]; display_progress(ctx->progress, ++ctx->progress_cnt);
if (repo_parse_commit_no_graph(ctx->r, *list)) die(_("unable to parse commit %s"), oid_to_hex(&(*list)->object.oid)); tree = get_commit_tree_oid(*list); hashwrite(f, tree->hash, ctx->r->hash_algo->rawsz);
parent = (*list)->parents;
if (!parent) edge_value = GRAPH_PARENT_NONE; else { edge_value = oid_pos(&parent->item->object.oid, ctx->commits.list, ctx->commits.nr, commit_to_oid);
if (edge_value >= 0) edge_value += ctx->new_num_commits_in_base; else if (ctx->new_base_graph) { uint32_t pos; if (find_commit_pos_in_graph(parent->item, ctx->new_base_graph, &pos)) edge_value = pos; }
if (edge_value < 0) BUG("missing parent %s for commit %s", oid_to_hex(&parent->item->object.oid), oid_to_hex(&(*list)->object.oid)); }
hashwrite_be32(f, edge_value);
if (parent) parent = parent->next;
if (!parent) edge_value = GRAPH_PARENT_NONE; else if (parent->next) edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges; else { edge_value = oid_pos(&parent->item->object.oid, ctx->commits.list, ctx->commits.nr, commit_to_oid);
if (edge_value >= 0) edge_value += ctx->new_num_commits_in_base; else if (ctx->new_base_graph) { uint32_t pos; if (find_commit_pos_in_graph(parent->item, ctx->new_base_graph, &pos)) edge_value = pos; }
if (edge_value < 0) BUG("missing parent %s for commit %s", oid_to_hex(&parent->item->object.oid), oid_to_hex(&(*list)->object.oid)); }
hashwrite_be32(f, edge_value);
if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) { do { num_extra_edges++; parent = parent->next; } while (parent); }
if (sizeof((*list)->date) > 4) packedDate[0] = htonl(((*list)->date >> 32) & 0x3); else packedDate[0] = 0;
packedDate[0] |= htonl(*topo_level_slab_at(ctx->topo_levels, *list) << 2);
packedDate[1] = htonl((*list)->date); hashwrite(f, packedDate, 8);
list++; }
return 0;}
static int write_graph_chunk_generation_data(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; int i, num_generation_data_overflows = 0;
for (i = 0; i < ctx->commits.nr; i++) { struct commit *c = ctx->commits.list[i]; timestamp_t offset; repo_parse_commit(ctx->r, c); offset = commit_graph_data_at(c)->generation - c->date; display_progress(ctx->progress, ++ctx->progress_cnt);
if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) { offset = CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW | num_generation_data_overflows; num_generation_data_overflows++; }
hashwrite_be32(f, offset); }
return 0;}
static int write_graph_chunk_generation_data_overflow(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; int i; for (i = 0; i < ctx->commits.nr; i++) { struct commit *c = ctx->commits.list[i]; timestamp_t offset = commit_graph_data_at(c)->generation - c->date; display_progress(ctx->progress, ++ctx->progress_cnt);
if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) { hashwrite_be32(f, offset >> 32); hashwrite_be32(f, (uint32_t) offset); } }
return 0;}
static int write_graph_chunk_extra_edges(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; struct commit **list = ctx->commits.list; struct commit **last = ctx->commits.list + ctx->commits.nr; struct commit_list *parent;
while (list < last) { int num_parents = 0;
display_progress(ctx->progress, ++ctx->progress_cnt);
for (parent = (*list)->parents; num_parents < 3 && parent; parent = parent->next) num_parents++;
if (num_parents <= 2) { list++; continue; }
/* Since num_parents > 2, this initializer is safe. */ for (parent = (*list)->parents->next; parent; parent = parent->next) { int edge_value = oid_pos(&parent->item->object.oid, ctx->commits.list, ctx->commits.nr, commit_to_oid);
if (edge_value >= 0) edge_value += ctx->new_num_commits_in_base; else if (ctx->new_base_graph) { uint32_t pos; if (find_commit_pos_in_graph(parent->item, ctx->new_base_graph, &pos)) edge_value = pos; }
if (edge_value < 0) BUG("missing parent %s for commit %s", oid_to_hex(&parent->item->object.oid), oid_to_hex(&(*list)->object.oid)); else if (!parent->next) edge_value |= GRAPH_LAST_EDGE;
hashwrite_be32(f, edge_value); }
list++; }
return 0;}
static int write_graph_chunk_bloom_indexes(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; struct commit **list = ctx->commits.list; struct commit **last = ctx->commits.list + ctx->commits.nr; uint32_t cur_pos = 0;
while (list < last) { struct bloom_filter *filter = get_bloom_filter(ctx->r, *list); size_t len = filter ? filter->len : 0; cur_pos += len; display_progress(ctx->progress, ++ctx->progress_cnt); hashwrite_be32(f, cur_pos); list++; }
return 0;}
static void trace2_bloom_filter_settings(struct write_commit_graph_context *ctx){ struct json_writer jw = JSON_WRITER_INIT;
jw_object_begin(&jw, 0); jw_object_intmax(&jw, "hash_version", ctx->bloom_settings->hash_version); jw_object_intmax(&jw, "num_hashes", ctx->bloom_settings->num_hashes); jw_object_intmax(&jw, "bits_per_entry", ctx->bloom_settings->bits_per_entry); jw_object_intmax(&jw, "max_changed_paths", ctx->bloom_settings->max_changed_paths); jw_end(&jw);
trace2_data_json("bloom", ctx->r, "settings", &jw);
jw_release(&jw);}
static int write_graph_chunk_bloom_data(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; struct commit **list = ctx->commits.list; struct commit **last = ctx->commits.list + ctx->commits.nr;
trace2_bloom_filter_settings(ctx);
hashwrite_be32(f, ctx->bloom_settings->hash_version); hashwrite_be32(f, ctx->bloom_settings->num_hashes); hashwrite_be32(f, ctx->bloom_settings->bits_per_entry);
while (list < last) { struct bloom_filter *filter = get_bloom_filter(ctx->r, *list); size_t len = filter ? filter->len : 0;
display_progress(ctx->progress, ++ctx->progress_cnt); if (len) hashwrite(f, filter->data, len * sizeof(unsigned char)); list++; }
return 0;}
static int add_packed_commits(const struct object_id *oid, struct packed_git *pack, uint32_t pos, void *data){ struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data; enum object_type type; off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT;
if (ctx->progress) display_progress(ctx->progress, ++ctx->progress_done);
oi.typep = &type; if (packed_object_info(ctx->r, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid));
if (type != OBJ_COMMIT) return 0;
oid_array_append(&ctx->oids, oid); set_commit_pos(ctx->r, oid);
return 0;}
static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit){ struct commit_list *parent; for (parent = commit->parents; parent; parent = parent->next) { if (!(parent->item->object.flags & REACHABLE)) { oid_array_append(&ctx->oids, &parent->item->object.oid); parent->item->object.flags |= REACHABLE; } }}
static void close_reachable(struct write_commit_graph_context *ctx){ int i; struct commit *commit; enum commit_graph_split_flags flags = ctx->opts ? ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Loading known commits in commit graph"), ctx->oids.nr); for (i = 0; i < ctx->oids.nr; i++) { display_progress(ctx->progress, i + 1); commit = lookup_commit(ctx->r, &ctx->oids.oid[i]); if (commit) commit->object.flags |= REACHABLE; } stop_progress(&ctx->progress);
/* * As this loop runs, ctx->oids.nr may grow, but not more * than the number of missing commits in the reachable * closure. */ if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Expanding reachable commits in commit graph"), 0); for (i = 0; i < ctx->oids.nr; i++) { display_progress(ctx->progress, i + 1); commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
if (!commit) continue; if (ctx->split) { if ((!repo_parse_commit(ctx->r, commit) && commit_graph_position(commit) == COMMIT_NOT_FROM_GRAPH) || flags == COMMIT_GRAPH_SPLIT_REPLACE) add_missing_parents(ctx, commit); } else if (!repo_parse_commit_no_graph(ctx->r, commit)) add_missing_parents(ctx, commit); } stop_progress(&ctx->progress);
if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Clearing commit marks in commit graph"), ctx->oids.nr); for (i = 0; i < ctx->oids.nr; i++) { display_progress(ctx->progress, i + 1); commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
if (commit) commit->object.flags &= ~REACHABLE; } stop_progress(&ctx->progress);}
struct compute_generation_info { struct repository *r; struct packed_commit_list *commits; struct progress *progress; int progress_cnt;
timestamp_t (*get_generation)(struct commit *c, void *data); void (*set_generation)(struct commit *c, timestamp_t gen, void *data); void *data;};
static timestamp_t compute_generation_from_max(struct commit *c, timestamp_t max_gen, int generation_version){ switch (generation_version) { case 1: /* topological levels */ if (max_gen > GENERATION_NUMBER_V1_MAX - 1) max_gen = GENERATION_NUMBER_V1_MAX - 1; return max_gen + 1;
case 2: /* corrected commit date */ if (c->date && c->date > max_gen) max_gen = c->date - 1; return max_gen + 1;
default: BUG("attempting unimplemented version"); }}
static void compute_reachable_generation_numbers( struct compute_generation_info *info, int generation_version){ int i; struct commit_list *list = NULL;
for (i = 0; i < info->commits->nr; i++) { struct commit *c = info->commits->list[i]; timestamp_t gen; repo_parse_commit(info->r, c); gen = info->get_generation(c, info->data); display_progress(info->progress, ++info->progress_cnt);
if (gen != GENERATION_NUMBER_ZERO && gen != GENERATION_NUMBER_INFINITY) continue;
commit_list_insert(c, &list); while (list) { struct commit *current = list->item; struct commit_list *parent; int all_parents_computed = 1; uint32_t max_gen = 0;
for (parent = current->parents; parent; parent = parent->next) { repo_parse_commit(info->r, parent->item); gen = info->get_generation(parent->item, info->data);
if (gen == GENERATION_NUMBER_ZERO) { all_parents_computed = 0; commit_list_insert(parent->item, &list); break; }
if (gen > max_gen) max_gen = gen; }
if (all_parents_computed) { pop_commit(&list); gen = compute_generation_from_max( current, max_gen, generation_version); info->set_generation(current, gen, info->data); } } }}
static timestamp_t get_topo_level(struct commit *c, void *data){ struct write_commit_graph_context *ctx = data; return *topo_level_slab_at(ctx->topo_levels, c);}
static void set_topo_level(struct commit *c, timestamp_t t, void *data){ struct write_commit_graph_context *ctx = data; *topo_level_slab_at(ctx->topo_levels, c) = (uint32_t)t;}
static void compute_topological_levels(struct write_commit_graph_context *ctx){ struct compute_generation_info info = { .r = ctx->r, .commits = &ctx->commits, .get_generation = get_topo_level, .set_generation = set_topo_level, .data = ctx, };
if (ctx->report_progress) info.progress = ctx->progress = start_delayed_progress( ctx->r, _("Computing commit graph topological levels"), ctx->commits.nr);
compute_reachable_generation_numbers(&info, 1);
stop_progress(&ctx->progress);}
static timestamp_t get_generation_from_graph_data(struct commit *c, void *data UNUSED){ return commit_graph_data_at(c)->generation;}
static void set_generation_v2(struct commit *c, timestamp_t t, void *data UNUSED){ struct commit_graph_data *g = commit_graph_data_at(c); g->generation = t;}
static void compute_generation_numbers(struct write_commit_graph_context *ctx){ int i; struct compute_generation_info info = { .r = ctx->r, .commits = &ctx->commits, .get_generation = get_generation_from_graph_data, .set_generation = set_generation_v2, };
if (ctx->report_progress) info.progress = ctx->progress = start_delayed_progress( ctx->r, _("Computing commit graph generation numbers"), ctx->commits.nr);
if (!ctx->trust_generation_numbers) { for (i = 0; i < ctx->commits.nr; i++) { struct commit *c = ctx->commits.list[i]; repo_parse_commit(ctx->r, c); commit_graph_data_at(c)->generation = GENERATION_NUMBER_ZERO; } }
compute_reachable_generation_numbers(&info, 2);
for (i = 0; i < ctx->commits.nr; i++) { struct commit *c = ctx->commits.list[i]; timestamp_t offset = commit_graph_data_at(c)->generation - c->date; if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) ctx->num_generation_data_overflows++; } stop_progress(&ctx->progress);}
static void set_generation_in_graph_data(struct commit *c, timestamp_t t, void *data UNUSED){ commit_graph_data_at(c)->generation = t;}
/* * After this method, all commits reachable from those in the given * list will have non-zero, non-infinite generation numbers. */void ensure_generations_valid(struct repository *r, struct commit **commits, size_t nr){ int generation_version = get_configured_generation_version(r); struct packed_commit_list list = { .list = commits, .alloc = nr, .nr = nr, }; struct compute_generation_info info = { .r = r, .commits = &list, .get_generation = get_generation_from_graph_data, .set_generation = set_generation_in_graph_data, };
compute_reachable_generation_numbers(&info, generation_version);}
static void trace2_bloom_filter_write_statistics(struct write_commit_graph_context *ctx){ trace2_data_intmax("commit-graph", ctx->r, "filter-computed", ctx->count_bloom_filter_computed); trace2_data_intmax("commit-graph", ctx->r, "filter-not-computed", ctx->count_bloom_filter_not_computed); trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-empty", ctx->count_bloom_filter_trunc_empty); trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-large", ctx->count_bloom_filter_trunc_large); trace2_data_intmax("commit-graph", ctx->r, "filter-upgraded", ctx->count_bloom_filter_upgraded);}
static void compute_bloom_filters(struct write_commit_graph_context *ctx){ int i; struct progress *progress = NULL; struct commit **sorted_commits; int max_new_filters;
init_bloom_filters();
if (ctx->report_progress) progress = start_delayed_progress( ctx->r, _("Computing commit changed paths Bloom filters"), ctx->commits.nr);
DUP_ARRAY(sorted_commits, ctx->commits.list, ctx->commits.nr);
if (ctx->order_by_pack) QSORT(sorted_commits, ctx->commits.nr, commit_pos_cmp); else QSORT(sorted_commits, ctx->commits.nr, commit_gen_cmp);
max_new_filters = ctx->opts && ctx->opts->max_new_filters >= 0 ? ctx->opts->max_new_filters : ctx->commits.nr;
for (i = 0; i < ctx->commits.nr; i++) { enum bloom_filter_computed computed = 0; struct commit *c = sorted_commits[i]; struct bloom_filter *filter = get_or_compute_bloom_filter( ctx->r, c, ctx->count_bloom_filter_computed < max_new_filters, ctx->bloom_settings, &computed); if (computed & BLOOM_COMPUTED) { ctx->count_bloom_filter_computed++; if (computed & BLOOM_TRUNC_EMPTY) ctx->count_bloom_filter_trunc_empty++; if (computed & BLOOM_TRUNC_LARGE) ctx->count_bloom_filter_trunc_large++; } else if (computed & BLOOM_UPGRADED) { ctx->count_bloom_filter_upgraded++; } else if (computed & BLOOM_NOT_COMPUTED) ctx->count_bloom_filter_not_computed++; ctx->total_bloom_filter_data_size += filter ? sizeof(unsigned char) * filter->len : 0; display_progress(progress, i + 1); }
if (trace2_is_enabled()) trace2_bloom_filter_write_statistics(ctx);
free(sorted_commits); stop_progress(&progress);}
struct refs_cb_data { struct repository *repo; struct oidset *commits; struct progress *progress;};
static int add_ref_to_set(const char *refname UNUSED, const char *referent UNUSED, const struct object_id *oid, int flags UNUSED, void *cb_data){ struct object_id peeled; struct refs_cb_data *data = (struct refs_cb_data *)cb_data;
if (!peel_iterated_oid(data->repo, oid, &peeled)) oid = &peeled; if (odb_read_object_info(data->repo->objects, oid, NULL) == OBJ_COMMIT) oidset_insert(data->commits, oid);
display_progress(data->progress, oidset_size(data->commits));
return 0;}
int write_commit_graph_reachable(struct odb_source *source, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts){ struct oidset commits = OIDSET_INIT; struct refs_cb_data data; int result;
memset(&data, 0, sizeof(data)); data.repo = source->odb->repo; data.commits = &commits;
if (flags & COMMIT_GRAPH_WRITE_PROGRESS) data.progress = start_delayed_progress( source->odb->repo, _("Collecting referenced commits"), 0);
refs_for_each_ref(get_main_ref_store(source->odb->repo), add_ref_to_set, &data);
stop_progress(&data.progress);
result = write_commit_graph(source, NULL, &commits, flags, opts);
oidset_clear(&commits); return result;}
static int fill_oids_from_packs(struct write_commit_graph_context *ctx, const struct string_list *pack_indexes){ uint32_t i; struct strbuf progress_title = STRBUF_INIT; struct strbuf packname = STRBUF_INIT; int dirlen; int ret = 0;
strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path); dirlen = packname.len; if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Finding commits for commit graph in %"PRIuMAX" pack", "Finding commits for commit graph in %"PRIuMAX" packs", pack_indexes->nr), (uintmax_t)pack_indexes->nr); ctx->progress = start_delayed_progress(ctx->r, progress_title.buf, 0); ctx->progress_done = 0; } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; strbuf_setlen(&packname, dirlen); strbuf_addstr(&packname, pack_indexes->items[i].string); p = add_packed_git(ctx->r, packname.buf, packname.len, 1); if (!p) { ret = error(_("error adding pack %s"), packname.buf); goto cleanup; } if (open_pack_index(p)) { ret = error(_("error opening index for %s"), packname.buf); close_pack(p); free(p); goto cleanup; } for_each_object_in_pack(p, add_packed_commits, ctx, FOR_EACH_OBJECT_PACK_ORDER); close_pack(p); free(p); }
cleanup: stop_progress(&ctx->progress); strbuf_release(&progress_title); strbuf_release(&packname);
return ret;}
static int fill_oids_from_commits(struct write_commit_graph_context *ctx, struct oidset *commits){ struct oidset_iter iter; struct object_id *oid;
if (!oidset_size(commits)) return 0;
oidset_iter_init(commits, &iter); while ((oid = oidset_iter_next(&iter))) { oid_array_append(&ctx->oids, oid); }
return 0;}
static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx){ if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Finding commits for commit graph among packed objects"), ctx->approx_nr_objects); for_each_packed_object(ctx->r, add_packed_commits, ctx, FOR_EACH_OBJECT_PACK_ORDER); if (ctx->progress_done < ctx->approx_nr_objects) display_progress(ctx->progress, ctx->approx_nr_objects); stop_progress(&ctx->progress);}
static void copy_oids_to_commits(struct write_commit_graph_context *ctx){ uint32_t i; enum commit_graph_split_flags flags = ctx->opts ? ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
ctx->num_extra_edges = 0; if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Finding extra edges in commit graph"), ctx->oids.nr); oid_array_sort(&ctx->oids); for (i = 0; i < ctx->oids.nr; i = oid_array_next_unique(&ctx->oids, i)) { unsigned int num_parents;
display_progress(ctx->progress, i + 1);
ALLOC_GROW(ctx->commits.list, ctx->commits.nr + 1, ctx->commits.alloc); ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.oid[i]);
if (ctx->split && flags != COMMIT_GRAPH_SPLIT_REPLACE && commit_graph_position(ctx->commits.list[ctx->commits.nr]) != COMMIT_NOT_FROM_GRAPH) continue;
if (ctx->split && flags == COMMIT_GRAPH_SPLIT_REPLACE) repo_parse_commit(ctx->r, ctx->commits.list[ctx->commits.nr]); else repo_parse_commit_no_graph(ctx->r, ctx->commits.list[ctx->commits.nr]);
num_parents = commit_list_count(ctx->commits.list[ctx->commits.nr]->parents); if (num_parents > 2) ctx->num_extra_edges += num_parents - 1;
ctx->commits.nr++; } stop_progress(&ctx->progress);}
static int write_graph_chunk_base_1(struct hashfile *f, struct commit_graph *g){ int num = 0;
if (!g) return 0;
num = write_graph_chunk_base_1(f, g->base_graph); hashwrite(f, g->oid.hash, g->hash_algo->rawsz); return num + 1;}
static int write_graph_chunk_base(struct hashfile *f, void *data){ struct write_commit_graph_context *ctx = data; int num = write_graph_chunk_base_1(f, ctx->new_base_graph);
if (num != ctx->num_commit_graphs_after - 1) { error(_("failed to write correct number of base graph ids")); return -1; }
return 0;}
static int write_commit_graph_file(struct write_commit_graph_context *ctx){ uint32_t i; struct hashfile *f; struct tempfile *graph_layer; /* when ctx->split is non-zero */ struct lock_file lk = LOCK_INIT; const unsigned hashsz = ctx->r->hash_algo->rawsz; struct strbuf progress_title = STRBUF_INIT; struct chunkfile *cf; unsigned char file_hash[GIT_MAX_RAWSZ];
if (ctx->split) { struct strbuf tmp_file = STRBUF_INIT;
strbuf_addf(&tmp_file, "%s/info/commit-graphs/tmp_graph_XXXXXX", ctx->odb_source->path); ctx->graph_name = strbuf_detach(&tmp_file, NULL); } else { ctx->graph_name = get_commit_graph_filename(ctx->odb_source); }
if (safe_create_leading_directories(ctx->r, ctx->graph_name)) { error(_("unable to create leading directories of %s"), ctx->graph_name); return -1; }
if (ctx->split) { char *lock_name = get_commit_graph_chain_filename(ctx->odb_source);
hold_lock_file_for_update_mode(&lk, lock_name, LOCK_DIE_ON_ERROR, 0444); free(lock_name);
graph_layer = mks_tempfile_m(ctx->graph_name, 0444); if (!graph_layer) { error(_("unable to create temporary graph layer")); return -1; }
if (adjust_shared_perm(ctx->r, get_tempfile_path(graph_layer))) { error(_("unable to adjust shared permissions for '%s'"), get_tempfile_path(graph_layer)); return -1; }
f = hashfd(ctx->r->hash_algo, get_tempfile_fd(graph_layer), get_tempfile_path(graph_layer)); } else { hold_lock_file_for_update_mode(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR, 0444); f = hashfd(ctx->r->hash_algo, get_lock_file_fd(&lk), get_lock_file_path(&lk)); }
cf = init_chunkfile(f);
add_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, GRAPH_FANOUT_SIZE, write_graph_chunk_fanout); add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, st_mult(hashsz, ctx->commits.nr), write_graph_chunk_oids); add_chunk(cf, GRAPH_CHUNKID_DATA, st_mult(hashsz + 16, ctx->commits.nr), write_graph_chunk_data);
if (ctx->write_generation_data) add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, st_mult(sizeof(uint32_t), ctx->commits.nr), write_graph_chunk_generation_data); if (ctx->num_generation_data_overflows) add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, st_mult(sizeof(timestamp_t), ctx->num_generation_data_overflows), write_graph_chunk_generation_data_overflow); if (ctx->num_extra_edges) add_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, st_mult(4, ctx->num_extra_edges), write_graph_chunk_extra_edges); if (ctx->changed_paths) { add_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, st_mult(sizeof(uint32_t), ctx->commits.nr), write_graph_chunk_bloom_indexes); add_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, st_add(sizeof(uint32_t) * 3, ctx->total_bloom_filter_data_size), write_graph_chunk_bloom_data); } if (ctx->num_commit_graphs_after > 1) add_chunk(cf, GRAPH_CHUNKID_BASE, st_mult(hashsz, ctx->num_commit_graphs_after - 1), write_graph_chunk_base);
hashwrite_be32(f, GRAPH_SIGNATURE);
hashwrite_u8(f, GRAPH_VERSION); hashwrite_u8(f, oid_version(ctx->r->hash_algo)); hashwrite_u8(f, get_num_chunks(cf)); hashwrite_u8(f, ctx->num_commit_graphs_after - 1);
if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Writing out commit graph in %d pass", "Writing out commit graph in %d passes", get_num_chunks(cf)), get_num_chunks(cf)); ctx->progress = start_delayed_progress( ctx->r, progress_title.buf, st_mult(get_num_chunks(cf), ctx->commits.nr)); }
write_chunkfile(cf, ctx);
stop_progress(&ctx->progress); strbuf_release(&progress_title);
if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) { char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid)); char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash);
free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]); free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]); ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2] = new_base_name; ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2] = new_base_hash; }
close_commit_graph(ctx->r->objects); finalize_hashfile(f, file_hash, FSYNC_COMPONENT_COMMIT_GRAPH, CSUM_HASH_IN_STREAM | CSUM_FSYNC); free_chunkfile(cf);
if (ctx->split) { FILE *chainf = fdopen_lock_file(&lk, "w"); char *final_graph_name; int result;
if (!chainf) { error(_("unable to open commit-graph chain file")); return -1; }
if (ctx->base_graph_name) { const char *dest; int idx = ctx->num_commit_graphs_after - 1; if (ctx->num_commit_graphs_after > 1) idx--;
dest = ctx->commit_graph_filenames_after[idx];
if (strcmp(ctx->base_graph_name, dest)) { result = rename(ctx->base_graph_name, dest);
if (result) { error(_("failed to rename base commit-graph file")); return -1; } } } else { char *graph_name = get_commit_graph_filename(ctx->odb_source); unlink(graph_name); free(graph_name); }
free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(hash_to_hex_algop(file_hash, ctx->r->hash_algo)); final_graph_name = get_split_graph_filename(ctx->odb_source, ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name;
result = rename_tempfile(&graph_layer, final_graph_name);
for (i = 0; i < ctx->num_commit_graphs_after; i++) fprintf(get_lock_file_fp(&lk), "%s\n", ctx->commit_graph_hash_after[i]);
if (result) { error(_("failed to rename temporary commit-graph file")); return -1; } }
commit_lock_file(&lk);
return 0;}
static void split_graph_merge_strategy(struct write_commit_graph_context *ctx, struct commit_graph *graph_to_merge){ struct commit_graph *g; uint32_t num_commits; enum commit_graph_split_flags flags = COMMIT_GRAPH_SPLIT_UNSPECIFIED; uint32_t i;
int max_commits = 0; int size_mult = 2;
if (ctx->opts) { max_commits = ctx->opts->max_commits;
if (ctx->opts->size_multiple) size_mult = ctx->opts->size_multiple;
flags = ctx->opts->split_flags; }
g = graph_to_merge; num_commits = ctx->commits.nr; if (flags == COMMIT_GRAPH_SPLIT_REPLACE) ctx->num_commit_graphs_after = 1; else ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
if (flags != COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED && flags != COMMIT_GRAPH_SPLIT_REPLACE) { while (g && (g->num_commits <= st_mult(size_mult, num_commits) || (max_commits && num_commits > max_commits))) { if (g->odb_source != ctx->odb_source) break;
if (unsigned_add_overflows(num_commits, g->num_commits)) die(_("cannot merge graphs with %"PRIuMAX", " "%"PRIuMAX" commits"), (uintmax_t)num_commits, (uintmax_t)g->num_commits); num_commits += g->num_commits; g = g->base_graph;
ctx->num_commit_graphs_after--; } }
if (flags != COMMIT_GRAPH_SPLIT_REPLACE) ctx->new_base_graph = g; else if (ctx->num_commit_graphs_after != 1) BUG("split_graph_merge_strategy: num_commit_graphs_after " "should be 1 with --split=replace");
if (ctx->num_commit_graphs_after == 2) { char *old_graph_name = get_commit_graph_filename(g->odb_source);
if (!strcmp(g->filename, old_graph_name) && g->odb_source != ctx->odb_source) { ctx->num_commit_graphs_after = 1; ctx->new_base_graph = NULL; }
free(old_graph_name); }
CALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after); CALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);
for (i = 0; i < ctx->num_commit_graphs_after && i < ctx->num_commit_graphs_before; i++) ctx->commit_graph_filenames_after[i] = xstrdup(ctx->commit_graph_filenames_before[i]);
i = ctx->num_commit_graphs_before - 1; g = graph_to_merge;
while (g) { if (i < ctx->num_commit_graphs_after) ctx->commit_graph_hash_after[i] = xstrdup(oid_to_hex(&g->oid));
/* * If the topmost remaining layer has generation data chunk, the * resultant layer also has generation data chunk. */ if (i == ctx->num_commit_graphs_after - 2) ctx->write_generation_data = !!g->chunk_generation_data;
i--; g = g->base_graph; }}
static void merge_commit_graph(struct write_commit_graph_context *ctx, struct commit_graph *g){ uint32_t i; uint32_t offset = g->num_commits_in_base;
if (unsigned_add_overflows(ctx->commits.nr, g->num_commits)) die(_("cannot merge graph %s, too many commits: %"PRIuMAX), oid_to_hex(&g->oid), (uintmax_t)st_add(ctx->commits.nr, g->num_commits));
ALLOC_GROW(ctx->commits.list, ctx->commits.nr + g->num_commits, ctx->commits.alloc);
for (i = 0; i < g->num_commits; i++) { struct object_id oid; struct commit *result;
display_progress(ctx->progress, i + 1);
load_oid_from_graph(g, i + offset, &oid);
/* only add commits if they still exist in the repo */ result = lookup_commit_reference_gently(ctx->r, &oid, 1);
if (result) { ctx->commits.list[ctx->commits.nr] = result; ctx->commits.nr++; } }}
static int commit_compare(const void *_a, const void *_b){ const struct commit *a = *(const struct commit **)_a; const struct commit *b = *(const struct commit **)_b; return oidcmp(&a->object.oid, &b->object.oid);}
static void sort_and_scan_merged_commits(struct write_commit_graph_context *ctx){ uint32_t i, dedup_i = 0;
if (ctx->report_progress) ctx->progress = start_delayed_progress( ctx->r, _("Scanning merged commits"), ctx->commits.nr);
QSORT(ctx->commits.list, ctx->commits.nr, commit_compare);
ctx->num_extra_edges = 0; for (i = 0; i < ctx->commits.nr; i++) { display_progress(ctx->progress, i + 1);
if (i && oideq(&ctx->commits.list[i - 1]->object.oid, &ctx->commits.list[i]->object.oid)) { /* * Silently ignore duplicates. These were likely * created due to a commit appearing in multiple * layers of the chain, which is unexpected but * not invalid. We should make sure there is a * unique copy in the new layer. */ } else { unsigned int num_parents;
ctx->commits.list[dedup_i] = ctx->commits.list[i]; dedup_i++;
num_parents = commit_list_count(ctx->commits.list[i]->parents); if (num_parents > 2) ctx->num_extra_edges += num_parents - 1; } }
ctx->commits.nr = dedup_i;
stop_progress(&ctx->progress);}
static void merge_commit_graphs(struct write_commit_graph_context *ctx, struct commit_graph *g){ uint32_t current_graph_number = ctx->num_commit_graphs_before;
while (g && current_graph_number >= ctx->num_commit_graphs_after) { current_graph_number--;
if (ctx->report_progress) ctx->progress = start_delayed_progress(ctx->r, _("Merging commit-graph"), 0);
merge_commit_graph(ctx, g); stop_progress(&ctx->progress);
g = g->base_graph; }
if (g) { ctx->new_base_graph = g; ctx->new_num_commits_in_base = g->num_commits + g->num_commits_in_base; }
if (ctx->new_base_graph) ctx->base_graph_name = xstrdup(ctx->new_base_graph->filename);
sort_and_scan_merged_commits(ctx);}
static void mark_commit_graphs(struct write_commit_graph_context *ctx){ uint32_t i; time_t now = time(NULL);
for (i = ctx->num_commit_graphs_after - 1; i < ctx->num_commit_graphs_before; i++) { struct stat st; struct utimbuf updated_time;
if (stat(ctx->commit_graph_filenames_before[i], &st) < 0) continue;
updated_time.actime = st.st_atime; updated_time.modtime = now; utime(ctx->commit_graph_filenames_before[i], &updated_time); }}
static void expire_commit_graphs(struct write_commit_graph_context *ctx){ struct strbuf path = STRBUF_INIT; DIR *dir; struct dirent *de; size_t dirnamelen; timestamp_t expire_time = time(NULL);
if (ctx->opts && ctx->opts->expire_time) expire_time = ctx->opts->expire_time; if (!ctx->split) { char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source); unlink(chain_file_name); free(chain_file_name); ctx->num_commit_graphs_after = 0; }
strbuf_addstr(&path, ctx->odb_source->path); strbuf_addstr(&path, "/info/commit-graphs"); dir = opendir(path.buf);
if (!dir) goto out;
strbuf_addch(&path, '/'); dirnamelen = path.len; while ((de = readdir(dir)) != NULL) { struct stat st; uint32_t i, found = 0;
strbuf_setlen(&path, dirnamelen); strbuf_addstr(&path, de->d_name);
if (stat(path.buf, &st) < 0) continue;
if (st.st_mtime > expire_time) continue; if (path.len < 6 || strcmp(path.buf + path.len - 6, ".graph")) continue;
for (i = 0; i < ctx->num_commit_graphs_after; i++) { if (!strcmp(ctx->commit_graph_filenames_after[i], path.buf)) { found = 1; break; } }
if (!found) unlink(path.buf); }
out: if(dir) closedir(dir); strbuf_release(&path);}
int write_commit_graph(struct odb_source *source, const struct string_list *const pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts){ struct repository *r = source->odb->repo; struct write_commit_graph_context ctx = { .r = r, .odb_source = source, .append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0, .report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0, .split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0, .opts = opts, .total_bloom_filter_data_size = 0, .write_generation_data = (get_configured_generation_version(r) == 2), .num_generation_data_overflows = 0, }; uint32_t i; int res = 0; int replace = 0; struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS; struct topo_level_slab topo_levels; struct commit_graph *g;
prepare_repo_settings(r); if (!r->settings.core_commit_graph) { warning(_("attempting to write a commit-graph, but 'core.commitGraph' is disabled")); return 0; } if (!commit_graph_compatible(r)) return 0; if (r->settings.commit_graph_changed_paths_version < -1 || r->settings.commit_graph_changed_paths_version > 2) { warning(_("attempting to write a commit-graph, but " "'commitGraph.changedPathsVersion' (%d) is not supported"), r->settings.commit_graph_changed_paths_version); return 0; }
bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version; bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY", bloom_settings.bits_per_entry); bloom_settings.num_hashes = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_NUM_HASHES", bloom_settings.num_hashes); bloom_settings.max_changed_paths = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS", bloom_settings.max_changed_paths); ctx.bloom_settings = &bloom_settings;
init_topo_level_slab(&topo_levels); ctx.topo_levels = &topo_levels;
g = prepare_commit_graph(ctx.r); for (struct commit_graph *chain = g; chain; chain = chain->base_graph) g->topo_levels = &topo_levels;
if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS) ctx.changed_paths = 1; if (!(flags & COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS)) { /* We have changed-paths already. Keep them in the next graph */ if (g && g->bloom_filter_settings) { ctx.changed_paths = 1;
/* don't propagate the hash_version unless unspecified */ if (bloom_settings.hash_version == -1) bloom_settings.hash_version = g->bloom_filter_settings->hash_version; bloom_settings.bits_per_entry = g->bloom_filter_settings->bits_per_entry; bloom_settings.num_hashes = g->bloom_filter_settings->num_hashes; bloom_settings.max_changed_paths = g->bloom_filter_settings->max_changed_paths; } }
bloom_settings.hash_version = bloom_settings.hash_version == 2 ? 2 : 1;
if (ctx.split) { for (struct commit_graph *chain = g; chain; chain = chain->base_graph) ctx.num_commit_graphs_before++;
if (ctx.num_commit_graphs_before) { ALLOC_ARRAY(ctx.commit_graph_filenames_before, ctx.num_commit_graphs_before); i = ctx.num_commit_graphs_before;
for (struct commit_graph *chain = g; chain; chain = chain->base_graph) ctx.commit_graph_filenames_before[--i] = xstrdup(chain->filename); }
if (ctx.opts) replace = ctx.opts->split_flags & COMMIT_GRAPH_SPLIT_REPLACE; }
ctx.approx_nr_objects = repo_approximate_object_count(r);
if (ctx.append && g) { for (i = 0; i < g->num_commits; i++) { struct object_id oid; oidread(&oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i), r->hash_algo); oid_array_append(&ctx.oids, &oid); } }
if (pack_indexes) { ctx.order_by_pack = 1; if ((res = fill_oids_from_packs(&ctx, pack_indexes))) goto cleanup; }
if (commits) { if ((res = fill_oids_from_commits(&ctx, commits))) goto cleanup; }
if (!pack_indexes && !commits) { ctx.order_by_pack = 1; fill_oids_from_all_packs(&ctx); }
close_reachable(&ctx);
copy_oids_to_commits(&ctx);
if (ctx.commits.nr >= GRAPH_EDGE_LAST_MASK) { error(_("too many commits to write graph")); res = -1; goto cleanup; }
if (!ctx.commits.nr && !replace) goto cleanup;
if (ctx.split) { split_graph_merge_strategy(&ctx, g);
if (!replace) merge_commit_graphs(&ctx, g); } else { ctx.num_commit_graphs_after = 1; }
ctx.trust_generation_numbers = validate_mixed_generation_chain(g);
compute_topological_levels(&ctx); if (ctx.write_generation_data) compute_generation_numbers(&ctx);
if (ctx.changed_paths) compute_bloom_filters(&ctx);
res = write_commit_graph_file(&ctx);
if (ctx.changed_paths) deinit_bloom_filters();
if (ctx.split) mark_commit_graphs(&ctx);
expire_commit_graphs(&ctx);
cleanup: free(ctx.graph_name); free(ctx.base_graph_name); free(ctx.commits.list); oid_array_clear(&ctx.oids); clear_topo_level_slab(&topo_levels);
if (ctx.r->objects->commit_graph) { struct commit_graph *g = ctx.r->objects->commit_graph;
while (g) { g->topo_levels = NULL; g = g->base_graph; } }
for (i = 0; i < ctx.num_commit_graphs_before; i++) free(ctx.commit_graph_filenames_before[i]); free(ctx.commit_graph_filenames_before);
for (i = 0; i < ctx.num_commit_graphs_after; i++) { free(ctx.commit_graph_filenames_after[i]); free(ctx.commit_graph_hash_after[i]); } free(ctx.commit_graph_filenames_after); free(ctx.commit_graph_hash_after);
return res;}
#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2static int verify_commit_graph_error;
__attribute__((format (printf, 1, 2)))static void graph_report(const char *fmt, ...){ va_list ap;
verify_commit_graph_error = 1; va_start(ap, fmt); vfprintf(stderr, fmt, ap); fprintf(stderr, "\n"); va_end(ap);}
static int commit_graph_checksum_valid(struct commit_graph *g){ return hashfile_checksum_valid(g->hash_algo, g->data, g->data_len);}
static int verify_one_commit_graph(struct commit_graph *g, struct progress *progress, uint64_t *seen){ struct repository *r = g->odb_source->odb->repo; uint32_t i, cur_fanout_pos = 0; struct object_id prev_oid, cur_oid; struct commit *seen_gen_zero = NULL; struct commit *seen_gen_non_zero = NULL;
if (!commit_graph_checksum_valid(g)) { graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt")); verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH; }
for (i = 0; i < g->num_commits; i++) { struct commit *graph_commit;
oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i), g->hash_algo);
if (i && oidcmp(&prev_oid, &cur_oid) >= 0) graph_report(_("commit-graph has incorrect OID order: %s then %s"), oid_to_hex(&prev_oid), oid_to_hex(&cur_oid));
oidcpy(&prev_oid, &cur_oid);
while (cur_oid.hash[0] > cur_fanout_pos) { uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
if (i != fanout_value) graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"), cur_fanout_pos, fanout_value, i); cur_fanout_pos++; }
graph_commit = lookup_commit(r, &cur_oid); if (!parse_commit_in_graph_one(g, graph_commit)) graph_report(_("failed to parse commit %s from commit-graph"), oid_to_hex(&cur_oid)); }
while (cur_fanout_pos < 256) { uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
if (g->num_commits != fanout_value) graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"), cur_fanout_pos, fanout_value, i);
cur_fanout_pos++; }
if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH) return verify_commit_graph_error;
for (i = 0; i < g->num_commits; i++) { struct commit *graph_commit, *odb_commit; struct commit_list *graph_parents, *odb_parents; timestamp_t max_generation = 0; timestamp_t generation;
display_progress(progress, ++(*seen)); oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_algo->rawsz, i), g->hash_algo);
graph_commit = lookup_commit(r, &cur_oid); odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r)); if (repo_parse_commit_internal(r, odb_commit, 0, 0)) { graph_report(_("failed to parse commit %s from object database for commit-graph"), oid_to_hex(&cur_oid)); continue; }
if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, get_commit_tree_oid(odb_commit))) graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"), oid_to_hex(&cur_oid), oid_to_hex(get_commit_tree_oid(graph_commit)), oid_to_hex(get_commit_tree_oid(odb_commit)));
graph_parents = graph_commit->parents; odb_parents = odb_commit->parents;
while (graph_parents) { if (!odb_parents) { graph_report(_("commit-graph parent list for commit %s is too long"), oid_to_hex(&cur_oid)); break; }
/* parse parent in case it is in a base graph */ parse_commit_in_graph_one(g, graph_parents->item);
if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid)) graph_report(_("commit-graph parent for %s is %s != %s"), oid_to_hex(&cur_oid), oid_to_hex(&graph_parents->item->object.oid), oid_to_hex(&odb_parents->item->object.oid));
generation = commit_graph_generation_from_graph(graph_parents->item); if (generation > max_generation) max_generation = generation;
graph_parents = graph_parents->next; odb_parents = odb_parents->next; }
if (odb_parents) graph_report(_("commit-graph parent list for commit %s terminates early"), oid_to_hex(&cur_oid));
if (commit_graph_generation_from_graph(graph_commit)) seen_gen_non_zero = graph_commit; else seen_gen_zero = graph_commit;
if (seen_gen_zero) continue;
/* * If we are using topological level and one of our parents has * generation GENERATION_NUMBER_V1_MAX, then our generation is * also GENERATION_NUMBER_V1_MAX. Decrement to avoid extra logic * in the following condition. */ if (!g->read_generation_data && max_generation == GENERATION_NUMBER_V1_MAX) max_generation--;
generation = commit_graph_generation(graph_commit); if (generation < max_generation + 1) graph_report(_("commit-graph generation for commit %s is %"PRItime" < %"PRItime), oid_to_hex(&cur_oid), generation, max_generation + 1);
if (graph_commit->date != odb_commit->date) graph_report(_("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime), oid_to_hex(&cur_oid), graph_commit->date, odb_commit->date); }
if (seen_gen_zero && seen_gen_non_zero) graph_report(_("commit-graph has both zero and non-zero " "generations (e.g., commits '%s' and '%s')"), oid_to_hex(&seen_gen_zero->object.oid), oid_to_hex(&seen_gen_non_zero->object.oid));
return verify_commit_graph_error;}
int verify_commit_graph(struct commit_graph *g, int flags){ struct progress *progress = NULL; int local_error = 0; uint64_t seen = 0;
if (!g) { graph_report("no commit-graph file loaded"); return 1; }
if (flags & COMMIT_GRAPH_WRITE_PROGRESS) { uint64_t total = g->num_commits; if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW)) total += g->num_commits_in_base;
progress = start_progress(g->odb_source->odb->repo, _("Verifying commits in commit graph"), total); }
for (; g; g = g->base_graph) { local_error |= verify_one_commit_graph(g, progress, &seen); if (flags & COMMIT_GRAPH_VERIFY_SHALLOW) break; }
stop_progress(&progress);
return local_error;}
void free_commit_graph(struct commit_graph *g){ while (g) { struct commit_graph *next = g->base_graph;
if (g->data) munmap((void *)g->data, g->data_len); free(g->filename); free(g->bloom_filter_settings); free(g);
g = next; }}
void disable_commit_graph(struct repository *r){ r->commit_graph_disabled = 1;}