Coverage Report

Created: 2026-03-21 06:46

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/git/fetch-pack.c
Line
Count
Source
1
#define USE_THE_REPOSITORY_VARIABLE
2
#define DISABLE_SIGN_COMPARE_WARNINGS
3
4
#include "git-compat-util.h"
5
#include "repository.h"
6
#include "config.h"
7
#include "date.h"
8
#include "environment.h"
9
#include "gettext.h"
10
#include "hex.h"
11
#include "lockfile.h"
12
#include "refs.h"
13
#include "pkt-line.h"
14
#include "commit.h"
15
#include "tag.h"
16
#include "pack.h"
17
#include "sideband.h"
18
#include "fetch-pack.h"
19
#include "remote.h"
20
#include "run-command.h"
21
#include "connect.h"
22
#include "trace2.h"
23
#include "version.h"
24
#include "oid-array.h"
25
#include "oidset.h"
26
#include "packfile.h"
27
#include "odb.h"
28
#include "path.h"
29
#include "connected.h"
30
#include "fetch-negotiator.h"
31
#include "fsck.h"
32
#include "shallow.h"
33
#include "commit-reach.h"
34
#include "commit-graph.h"
35
#include "sigchain.h"
36
#include "mergesort.h"
37
#include "prio-queue.h"
38
#include "promisor-remote.h"
39
40
static int transfer_unpack_limit = -1;
41
static int fetch_unpack_limit = -1;
42
static int unpack_limit = 100;
43
static int prefer_ofs_delta = 1;
44
static int no_done;
45
static int deepen_since_ok;
46
static int deepen_not_ok;
47
static int fetch_fsck_objects = -1;
48
static int transfer_fsck_objects = -1;
49
static int agent_supported;
50
static int server_supports_filtering;
51
static int advertise_sid;
52
static struct shallow_lock shallow_lock;
53
static const char *alternate_shallow_file;
54
static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
55
static struct strbuf fsck_msg_types = STRBUF_INIT;
56
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
57
58
/* Remember to update object flag allocation in object.h */
59
0
#define COMPLETE  (1U << 0)
60
0
#define ALTERNATE (1U << 1)
61
0
#define COMMON    (1U << 6)
62
0
#define REACH_SCRATCH (1U << 7)
63
64
/*
65
 * After sending this many "have"s if we do not get any new ACK , we
66
 * give up traversing our history.
67
 */
68
0
#define MAX_IN_VAIN 256
69
70
static int multi_ack, use_sideband;
71
/* Allow specifying sha1 if it is a ref tip. */
72
0
#define ALLOW_TIP_SHA1  01
73
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
74
0
#define ALLOW_REACHABLE_SHA1  02
75
static unsigned int allow_unadvertised_object_request;
76
77
__attribute__((format (printf, 2, 3)))
78
static inline void print_verbose(const struct fetch_pack_args *args,
79
         const char *fmt, ...)
80
0
{
81
0
  va_list params;
82
83
0
  if (!args->verbose)
84
0
    return;
85
86
0
  va_start(params, fmt);
87
0
  vfprintf(stderr, fmt, params);
88
0
  va_end(params);
89
0
  fputc('\n', stderr);
90
0
}
91
92
struct alternate_object_cache {
93
  struct object **items;
94
  size_t nr, alloc;
95
};
96
97
static void cache_one_alternate(const struct object_id *oid,
98
        void *vcache)
99
0
{
100
0
  struct alternate_object_cache *cache = vcache;
101
0
  struct object *obj = parse_object(the_repository, oid);
102
103
0
  if (!obj || (obj->flags & ALTERNATE))
104
0
    return;
105
106
0
  obj->flags |= ALTERNATE;
107
0
  ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
108
0
  cache->items[cache->nr++] = obj;
109
0
}
110
111
static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
112
              void (*cb)(struct fetch_negotiator *,
113
             struct object *))
114
0
{
115
0
  static int initialized;
116
0
  static struct alternate_object_cache cache;
117
0
  size_t i;
118
119
0
  if (!initialized) {
120
0
    odb_for_each_alternate_ref(the_repository->objects,
121
0
             cache_one_alternate, &cache);
122
0
    initialized = 1;
123
0
  }
124
125
0
  for (i = 0; i < cache.nr; i++)
126
0
    cb(negotiator, cache.items[i]);
127
0
}
128
129
static void die_in_commit_graph_only(const struct object_id *oid)
130
0
{
131
0
  die(_("You are attempting to fetch %s, which is in the commit graph file but not in the object database.\n"
132
0
        "This is probably due to repo corruption.\n"
133
0
        "If you are attempting to repair this repo corruption by refetching the missing object, use 'git fetch --refetch' with the missing object."),
134
0
        oid_to_hex(oid));
135
0
}
136
137
static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
138
                 int mark_tags_complete_and_check_obj_db)
139
0
{
140
0
  enum object_type type;
141
0
  struct object_info info = { .typep = &type };
142
0
  struct commit *commit;
143
144
0
  commit = lookup_commit_in_graph(the_repository, oid);
145
0
  if (commit) {
146
0
    if (mark_tags_complete_and_check_obj_db) {
147
0
      if (!odb_has_object(the_repository->objects, oid,
148
0
              HAS_OBJECT_RECHECK_PACKED))
149
0
        die_in_commit_graph_only(oid);
150
0
    }
151
0
    return commit;
152
0
  }
153
154
0
  while (1) {
155
0
    if (odb_read_object_info_extended(the_repository->objects, oid, &info,
156
0
              OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK))
157
0
      return NULL;
158
0
    if (type == OBJ_TAG) {
159
0
      struct tag *tag = (struct tag *)
160
0
        parse_object(the_repository, oid);
161
162
0
      if (!tag->tagged)
163
0
        return NULL;
164
0
      if (mark_tags_complete_and_check_obj_db)
165
0
        tag->object.flags |= COMPLETE;
166
0
      oid = &tag->tagged->oid;
167
0
    } else {
168
0
      break;
169
0
    }
170
0
  }
171
172
0
  if (type == OBJ_COMMIT) {
173
0
    struct commit *commit = lookup_commit(the_repository, oid);
174
0
    if (!commit || repo_parse_commit(the_repository, commit))
175
0
      return NULL;
176
0
    return commit;
177
0
  }
178
179
0
  return NULL;
180
0
}
181
182
static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
183
             const struct object_id *oid)
184
0
{
185
0
  struct commit *c = deref_without_lazy_fetch(oid, 0);
186
187
0
  if (c)
188
0
    negotiator->add_tip(negotiator, c);
189
0
  return 0;
190
0
}
191
192
static int rev_list_insert_ref_oid(const struct reference *ref, void *cb_data)
193
0
{
194
0
  return rev_list_insert_ref(cb_data, ref->oid);
195
0
}
196
197
enum ack_type {
198
  NAK = 0,
199
  ACK,
200
  ACK_continue,
201
  ACK_common,
202
  ACK_ready
203
};
204
205
static void consume_shallow_list(struct fetch_pack_args *args,
206
         struct packet_reader *reader)
207
0
{
208
0
  if (args->stateless_rpc && args->deepen) {
209
    /* If we sent a depth we will get back "duplicate"
210
     * shallow and unshallow commands every time there
211
     * is a block of have lines exchanged.
212
     */
213
0
    while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
214
0
      if (starts_with(reader->line, "shallow "))
215
0
        continue;
216
0
      if (starts_with(reader->line, "unshallow "))
217
0
        continue;
218
0
      die(_("git fetch-pack: expected shallow list"));
219
0
    }
220
0
    if (reader->status != PACKET_READ_FLUSH)
221
0
      die(_("git fetch-pack: expected a flush packet after shallow list"));
222
0
  }
223
0
}
224
225
static enum ack_type get_ack(struct packet_reader *reader,
226
           struct object_id *result_oid)
227
0
{
228
0
  int len;
229
0
  const char *arg;
230
231
0
  if (packet_reader_read(reader) != PACKET_READ_NORMAL)
232
0
    die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
233
0
  len = reader->pktlen;
234
235
0
  if (!strcmp(reader->line, "NAK"))
236
0
    return NAK;
237
0
  if (skip_prefix(reader->line, "ACK ", &arg)) {
238
0
    const char *p;
239
0
    if (!parse_oid_hex(arg, result_oid, &p)) {
240
0
      len -= p - reader->line;
241
0
      if (len < 1)
242
0
        return ACK;
243
0
      if (strstr(p, "continue"))
244
0
        return ACK_continue;
245
0
      if (strstr(p, "common"))
246
0
        return ACK_common;
247
0
      if (strstr(p, "ready"))
248
0
        return ACK_ready;
249
0
      return ACK;
250
0
    }
251
0
  }
252
0
  die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
253
0
}
254
255
static void send_request(struct fetch_pack_args *args,
256
       int fd, struct strbuf *buf)
257
0
{
258
0
  if (args->stateless_rpc) {
259
0
    send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
260
0
    packet_flush(fd);
261
0
  } else {
262
0
    if (write_in_full(fd, buf->buf, buf->len) < 0)
263
0
      die_errno(_("unable to write to remote"));
264
0
  }
265
0
}
266
267
static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
268
          struct object *obj)
269
0
{
270
0
  rev_list_insert_ref(negotiator, &obj->oid);
271
0
}
272
273
0
#define INITIAL_FLUSH 16
274
0
#define PIPESAFE_FLUSH 32
275
0
#define LARGE_FLUSH 16384
276
277
static int next_flush(int stateless_rpc, int count)
278
0
{
279
0
  if (stateless_rpc) {
280
0
    if (count < LARGE_FLUSH)
281
0
      count <<= 1;
282
0
    else
283
0
      count = count * 11 / 10;
284
0
  } else {
285
0
    if (count < PIPESAFE_FLUSH)
286
0
      count <<= 1;
287
0
    else
288
0
      count += PIPESAFE_FLUSH;
289
0
  }
290
0
  return count;
291
0
}
292
293
static void mark_tips(struct fetch_negotiator *negotiator,
294
          const struct oid_array *negotiation_tips)
295
0
{
296
0
  struct refs_for_each_ref_options opts = {
297
0
    .flags = REFS_FOR_EACH_INCLUDE_BROKEN,
298
0
  };
299
0
  int i;
300
301
0
  if (!negotiation_tips) {
302
0
    refs_for_each_ref_ext(get_main_ref_store(the_repository),
303
0
              rev_list_insert_ref_oid, negotiator, &opts);
304
0
    return;
305
0
  }
306
307
0
  for (i = 0; i < negotiation_tips->nr; i++)
308
0
    rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
309
0
  return;
310
0
}
311
312
static void send_filter(struct fetch_pack_args *args,
313
      struct strbuf *req_buf,
314
      int server_supports_filter)
315
0
{
316
0
  if (args->filter_options.choice) {
317
0
    const char *spec =
318
0
      expand_list_objects_filter_spec(&args->filter_options);
319
0
    if (server_supports_filter) {
320
0
      print_verbose(args, _("Server supports filter"));
321
0
      packet_buf_write(req_buf, "filter %s", spec);
322
0
      trace2_data_string("fetch", the_repository,
323
0
             "filter/effective", spec);
324
0
    } else {
325
0
      warning("filtering not recognized by server, ignoring");
326
0
      trace2_data_string("fetch", the_repository,
327
0
             "filter/unsupported", spec);
328
0
    }
329
0
  } else {
330
0
    trace2_data_string("fetch", the_repository,
331
0
           "filter/none", "");
332
0
  }
333
0
}
334
335
static int find_common(struct fetch_negotiator *negotiator,
336
           struct fetch_pack_args *args,
337
           int fd[2], struct object_id *result_oid,
338
           struct ref *refs)
339
0
{
340
0
  int fetching;
341
0
  int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
342
0
  int negotiation_round = 0, haves = 0;
343
0
  const struct object_id *oid;
344
0
  unsigned in_vain = 0;
345
0
  int got_continue = 0;
346
0
  int got_ready = 0;
347
0
  struct strbuf req_buf = STRBUF_INIT;
348
0
  size_t state_len = 0;
349
0
  struct packet_reader reader;
350
351
0
  if (args->stateless_rpc && multi_ack == 1)
352
0
    die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");
353
354
0
  packet_reader_init(&reader, fd[0], NULL, 0,
355
0
         PACKET_READ_CHOMP_NEWLINE |
356
0
         PACKET_READ_DIE_ON_ERR_PACKET);
357
358
0
  mark_tips(negotiator, args->negotiation_tips);
359
0
  for_each_cached_alternate(negotiator, insert_one_alternate_object);
360
361
0
  fetching = 0;
362
0
  for ( ; refs ; refs = refs->next) {
363
0
    struct object_id *remote = &refs->old_oid;
364
0
    const char *remote_hex;
365
0
    struct object *o;
366
367
0
    if (!args->refetch) {
368
      /*
369
      * If that object is complete (i.e. it is an ancestor of a
370
      * local ref), we tell them we have it but do not have to
371
      * tell them about its ancestors, which they already know
372
      * about.
373
      *
374
      * We use lookup_object here because we are only
375
      * interested in the case we *know* the object is
376
      * reachable and we have already scanned it.
377
      */
378
0
      if (((o = lookup_object(the_repository, remote)) != NULL) &&
379
0
          (o->flags & COMPLETE)) {
380
0
        continue;
381
0
      }
382
0
    }
383
384
0
    remote_hex = oid_to_hex(remote);
385
0
    if (!fetching) {
386
0
      struct strbuf c = STRBUF_INIT;
387
0
      if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
388
0
      if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
389
0
      if (no_done)            strbuf_addstr(&c, " no-done");
390
0
      if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
391
0
      if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
392
0
      if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
393
0
      if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
394
0
      if (args->no_progress)   strbuf_addstr(&c, " no-progress");
395
0
      if (args->include_tag)   strbuf_addstr(&c, " include-tag");
396
0
      if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
397
0
      if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
398
0
      if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
399
0
      if (agent_supported)    strbuf_addf(&c, " agent=%s",
400
0
                  git_user_agent_sanitized());
401
0
      if (advertise_sid)
402
0
        strbuf_addf(&c, " session-id=%s", trace2_session_id());
403
0
      if (args->filter_options.choice)
404
0
        strbuf_addstr(&c, " filter");
405
0
      packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
406
0
      strbuf_release(&c);
407
0
    } else
408
0
      packet_buf_write(&req_buf, "want %s\n", remote_hex);
409
0
    fetching++;
410
0
  }
411
412
0
  if (!fetching) {
413
0
    strbuf_release(&req_buf);
414
0
    packet_flush(fd[1]);
415
0
    return 1;
416
0
  }
417
418
0
  if (is_repository_shallow(the_repository))
419
0
    write_shallow_commits(&req_buf, 1, NULL);
420
0
  if (args->depth > 0)
421
0
    packet_buf_write(&req_buf, "deepen %d", args->depth);
422
0
  if (args->deepen_since) {
423
0
    timestamp_t max_age = approxidate(args->deepen_since);
424
0
    packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
425
0
  }
426
0
  if (args->deepen_not) {
427
0
    int i;
428
0
    for (i = 0; i < args->deepen_not->nr; i++) {
429
0
      struct string_list_item *s = args->deepen_not->items + i;
430
0
      packet_buf_write(&req_buf, "deepen-not %s", s->string);
431
0
    }
432
0
  }
433
0
  send_filter(args, &req_buf, server_supports_filtering);
434
0
  packet_buf_flush(&req_buf);
435
0
  state_len = req_buf.len;
436
437
0
  if (args->deepen) {
438
0
    const char *arg;
439
0
    struct object_id oid;
440
441
0
    send_request(args, fd[1], &req_buf);
442
0
    while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
443
0
      if (skip_prefix(reader.line, "shallow ", &arg)) {
444
0
        if (get_oid_hex(arg, &oid))
445
0
          die(_("invalid shallow line: %s"), reader.line);
446
0
        register_shallow(the_repository, &oid);
447
0
        continue;
448
0
      }
449
0
      if (skip_prefix(reader.line, "unshallow ", &arg)) {
450
0
        if (get_oid_hex(arg, &oid))
451
0
          die(_("invalid unshallow line: %s"), reader.line);
452
0
        if (!lookup_object(the_repository, &oid))
453
0
          die(_("object not found: %s"), reader.line);
454
        /* make sure that it is parsed as shallow */
455
0
        if (!parse_object(the_repository, &oid))
456
0
          die(_("error in object: %s"), reader.line);
457
0
        if (unregister_shallow(&oid))
458
0
          die(_("no shallow found: %s"), reader.line);
459
0
        continue;
460
0
      }
461
0
      die(_("expected shallow/unshallow, got %s"), reader.line);
462
0
    }
463
0
  } else if (!args->stateless_rpc)
464
0
    send_request(args, fd[1], &req_buf);
465
466
0
  if (!args->stateless_rpc) {
467
    /* If we aren't using the stateless-rpc interface
468
     * we don't need to retain the headers.
469
     */
470
0
    strbuf_setlen(&req_buf, 0);
471
0
    state_len = 0;
472
0
  }
473
474
0
  trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
475
0
  flushes = 0;
476
0
  retval = -1;
477
0
  while ((oid = negotiator->next(negotiator))) {
478
0
    packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
479
0
    print_verbose(args, "have %s", oid_to_hex(oid));
480
0
    in_vain++;
481
0
    haves++;
482
0
    if (flush_at <= ++count) {
483
0
      int ack;
484
485
0
      negotiation_round++;
486
0
      trace2_region_enter_printf("negotiation_v0_v1", "round",
487
0
               the_repository, "%d",
488
0
               negotiation_round);
489
0
      trace2_data_intmax("negotiation_v0_v1", the_repository,
490
0
             "haves_added", haves);
491
0
      trace2_data_intmax("negotiation_v0_v1", the_repository,
492
0
             "in_vain", in_vain);
493
0
      haves = 0;
494
0
      packet_buf_flush(&req_buf);
495
0
      send_request(args, fd[1], &req_buf);
496
0
      strbuf_setlen(&req_buf, state_len);
497
0
      flushes++;
498
0
      flush_at = next_flush(args->stateless_rpc, count);
499
500
      /*
501
       * We keep one window "ahead" of the other side, and
502
       * will wait for an ACK only on the next one
503
       */
504
0
      if (!args->stateless_rpc && count == INITIAL_FLUSH)
505
0
        continue;
506
507
0
      consume_shallow_list(args, &reader);
508
0
      do {
509
0
        ack = get_ack(&reader, result_oid);
510
0
        if (ack)
511
0
          print_verbose(args, _("got %s %d %s"), "ack",
512
0
                  ack, oid_to_hex(result_oid));
513
0
        switch (ack) {
514
0
        case ACK:
515
0
          trace2_region_leave_printf("negotiation_v0_v1", "round",
516
0
                   the_repository, "%d",
517
0
                   negotiation_round);
518
0
          flushes = 0;
519
0
          multi_ack = 0;
520
0
          retval = 0;
521
0
          goto done;
522
0
        case ACK_common:
523
0
        case ACK_ready:
524
0
        case ACK_continue: {
525
0
          struct commit *commit =
526
0
            lookup_commit(the_repository,
527
0
                    result_oid);
528
0
          int was_common;
529
530
0
          if (!commit)
531
0
            die(_("invalid commit %s"), oid_to_hex(result_oid));
532
0
          was_common = negotiator->ack(negotiator, commit);
533
0
          if (args->stateless_rpc
534
0
           && ack == ACK_common
535
0
           && !was_common) {
536
            /* We need to replay the have for this object
537
             * on the next RPC request so the peer knows
538
             * it is in common with us.
539
             */
540
0
            const char *hex = oid_to_hex(result_oid);
541
0
            packet_buf_write(&req_buf, "have %s\n", hex);
542
0
            state_len = req_buf.len;
543
0
            haves++;
544
            /*
545
             * Reset in_vain because an ack
546
             * for this commit has not been
547
             * seen.
548
             */
549
0
            in_vain = 0;
550
0
          } else if (!args->stateless_rpc
551
0
               || ack != ACK_common)
552
0
            in_vain = 0;
553
0
          retval = 0;
554
0
          got_continue = 1;
555
0
          if (ack == ACK_ready)
556
0
            got_ready = 1;
557
0
          break;
558
0
          }
559
0
        }
560
0
      } while (ack);
561
0
      flushes--;
562
0
      trace2_region_leave_printf("negotiation_v0_v1", "round",
563
0
               the_repository, "%d",
564
0
               negotiation_round);
565
0
      if (got_continue && MAX_IN_VAIN < in_vain) {
566
0
        print_verbose(args, _("giving up"));
567
0
        break; /* give up */
568
0
      }
569
0
      if (got_ready)
570
0
        break;
571
0
    }
572
0
  }
573
0
done:
574
0
  trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
575
0
  trace2_data_intmax("negotiation_v0_v1", the_repository, "total_rounds",
576
0
         negotiation_round);
577
0
  if (!got_ready || !no_done) {
578
0
    packet_buf_write(&req_buf, "done\n");
579
0
    send_request(args, fd[1], &req_buf);
580
0
  }
581
0
  print_verbose(args, _("done"));
582
0
  if (retval != 0) {
583
0
    multi_ack = 0;
584
0
    flushes++;
585
0
  }
586
0
  strbuf_release(&req_buf);
587
588
0
  if (!got_ready || !no_done)
589
0
    consume_shallow_list(args, &reader);
590
0
  while (flushes || multi_ack) {
591
0
    int ack = get_ack(&reader, result_oid);
592
0
    if (ack) {
593
0
      print_verbose(args, _("got %s (%d) %s"), "ack",
594
0
              ack, oid_to_hex(result_oid));
595
0
      if (ack == ACK)
596
0
        return 0;
597
0
      multi_ack = 1;
598
0
      continue;
599
0
    }
600
0
    flushes--;
601
0
  }
602
  /* it is no error to fetch into a completely empty repo */
603
0
  return count ? retval : 0;
604
0
}
605
606
static struct prio_queue complete = { compare_commits_by_commit_date };
607
608
static int mark_complete(const struct object_id *oid)
609
0
{
610
0
  struct commit *commit = deref_without_lazy_fetch(oid, 1);
611
612
0
  if (commit && !(commit->object.flags & COMPLETE)) {
613
0
    commit->object.flags |= COMPLETE;
614
0
    prio_queue_put(&complete, commit);
615
0
  }
616
0
  return 0;
617
0
}
618
619
static int mark_complete_oid(const struct reference *ref, void *cb_data UNUSED)
620
0
{
621
0
  return mark_complete(ref->oid);
622
0
}
623
624
static void mark_recent_complete_commits(struct fetch_pack_args *args,
625
           timestamp_t cutoff)
626
0
{
627
0
  while (complete.nr) {
628
0
    struct commit *item = prio_queue_peek(&complete);
629
0
    if (item->date < cutoff)
630
0
      break;
631
0
    print_verbose(args, _("Marking %s as complete"),
632
0
            oid_to_hex(&item->object.oid));
633
0
    pop_most_recent_commit(&complete, COMPLETE);
634
0
  }
635
0
}
636
637
static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
638
0
{
639
0
  for (; refs; refs = refs->next)
640
0
    oidset_insert(oids, &refs->old_oid);
641
0
}
642
643
static int is_unmatched_ref(const struct ref *ref)
644
0
{
645
0
  struct object_id oid;
646
0
  const char *p;
647
0
  return  ref->match_status == REF_NOT_MATCHED &&
648
0
    !parse_oid_hex(ref->name, &oid, &p) &&
649
0
    *p == '\0' &&
650
0
    oideq(&oid, &ref->old_oid);
651
0
}
652
653
static void filter_refs(struct fetch_pack_args *args,
654
      struct ref **refs,
655
      struct ref **sought, int nr_sought)
656
0
{
657
0
  struct ref *newlist = NULL;
658
0
  struct ref **newtail = &newlist;
659
0
  struct ref *unmatched = NULL;
660
0
  struct ref *ref, *next;
661
0
  struct oidset tip_oids = OIDSET_INIT;
662
0
  int i;
663
0
  int strict = !(allow_unadvertised_object_request &
664
0
           (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
665
666
0
  i = 0;
667
0
  for (ref = *refs; ref; ref = next) {
668
0
    int keep = 0;
669
0
    next = ref->next;
670
671
0
    if (starts_with(ref->name, "refs/") &&
672
0
        check_refname_format(ref->name, 0)) {
673
      /*
674
       * trash or a peeled value; do not even add it to
675
       * unmatched list
676
       */
677
0
      free_one_ref(ref);
678
0
      continue;
679
0
    } else {
680
0
      while (i < nr_sought) {
681
0
        int cmp = strcmp(ref->name, sought[i]->name);
682
0
        if (cmp < 0)
683
0
          break; /* definitely do not have it */
684
0
        else if (cmp == 0) {
685
0
          keep = 1; /* definitely have it */
686
0
          sought[i]->match_status = REF_MATCHED;
687
0
        }
688
0
        i++;
689
0
      }
690
691
0
      if (!keep && args->fetch_all &&
692
0
          (!args->deepen || !starts_with(ref->name, "refs/tags/")))
693
0
        keep = 1;
694
0
    }
695
696
0
    if (keep) {
697
0
      *newtail = ref;
698
0
      ref->next = NULL;
699
0
      newtail = &ref->next;
700
0
    } else {
701
0
      ref->next = unmatched;
702
0
      unmatched = ref;
703
0
    }
704
0
  }
705
706
0
  if (strict) {
707
0
    for (i = 0; i < nr_sought; i++) {
708
0
      ref = sought[i];
709
0
      if (!is_unmatched_ref(ref))
710
0
        continue;
711
712
0
      add_refs_to_oidset(&tip_oids, unmatched);
713
0
      add_refs_to_oidset(&tip_oids, newlist);
714
0
      break;
715
0
    }
716
0
  }
717
718
  /* Append unmatched requests to the list */
719
0
  for (i = 0; i < nr_sought; i++) {
720
0
    ref = sought[i];
721
0
    if (!is_unmatched_ref(ref))
722
0
      continue;
723
724
0
    if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
725
0
      ref->match_status = REF_MATCHED;
726
0
      *newtail = copy_ref(ref);
727
0
      newtail = &(*newtail)->next;
728
0
    } else {
729
0
      ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
730
0
    }
731
0
  }
732
733
0
  oidset_clear(&tip_oids);
734
0
  free_refs(unmatched);
735
736
0
  *refs = newlist;
737
0
}
738
739
static void mark_alternate_complete(struct fetch_negotiator *negotiator UNUSED,
740
            struct object *obj)
741
0
{
742
0
  mark_complete(&obj->oid);
743
0
}
744
745
/*
746
 * Mark recent commits available locally and reachable from a local ref as
747
 * COMPLETE.
748
 *
749
 * The cutoff time for recency is determined by this heuristic: it is the
750
 * earliest commit time of the objects in refs that are commits and that we know
751
 * the commit time of.
752
 */
753
static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
754
           struct fetch_pack_args *args,
755
           struct ref **refs)
756
0
{
757
0
  struct ref *ref;
758
0
  int old_save_commit_buffer = save_commit_buffer;
759
0
  timestamp_t cutoff = 0;
760
761
0
  if (args->refetch)
762
0
    return;
763
764
0
  save_commit_buffer = 0;
765
766
0
  trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
767
0
  for (ref = *refs; ref; ref = ref->next) {
768
0
    struct commit *commit;
769
770
0
    commit = lookup_commit_in_graph(the_repository, &ref->old_oid);
771
0
    if (!commit) {
772
0
      struct object *o;
773
774
0
      if (!odb_has_object(the_repository->objects, &ref->old_oid, 0))
775
0
        continue;
776
0
      o = parse_object(the_repository, &ref->old_oid);
777
0
      if (!o || o->type != OBJ_COMMIT)
778
0
        continue;
779
780
0
      commit = (struct commit *)o;
781
0
    }
782
783
    /*
784
     * We already have it -- which may mean that we were
785
     * in sync with the other side at some time after
786
     * that (it is OK if we guess wrong here).
787
     */
788
0
    if (!cutoff || cutoff < commit->date)
789
0
      cutoff = commit->date;
790
0
  }
791
0
  trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
792
793
  /*
794
   * This block marks all local refs as COMPLETE, and then recursively marks all
795
   * parents of those refs as COMPLETE.
796
   */
797
0
  trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
798
0
  if (!args->deepen) {
799
0
    struct refs_for_each_ref_options opts = {
800
0
      .flags = REFS_FOR_EACH_INCLUDE_BROKEN,
801
0
    };
802
803
0
    refs_for_each_ref_ext(get_main_ref_store(the_repository),
804
0
              mark_complete_oid, NULL, &opts);
805
0
    for_each_cached_alternate(NULL, mark_alternate_complete);
806
0
    if (cutoff)
807
0
      mark_recent_complete_commits(args, cutoff);
808
0
  }
809
0
  trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
810
811
  /*
812
   * Mark all complete remote refs as common refs.
813
   * Don't mark them common yet; the server has to be told so first.
814
   */
815
0
  trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
816
0
  for (ref = *refs; ref; ref = ref->next) {
817
0
    struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);
818
819
0
    if (!c || !(c->object.flags & COMPLETE))
820
0
      continue;
821
822
0
    negotiator->known_common(negotiator, c);
823
0
  }
824
0
  trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
825
826
0
  save_commit_buffer = old_save_commit_buffer;
827
0
}
828
829
/*
830
 * Returns 1 if every object pointed to by the given remote refs is available
831
 * locally and reachable from a local ref, and 0 otherwise.
832
 */
833
static int everything_local(struct fetch_pack_args *args,
834
          struct ref **refs)
835
0
{
836
0
  struct ref *ref;
837
0
  int retval;
838
839
0
  for (retval = 1, ref = *refs; ref ; ref = ref->next) {
840
0
    const struct object_id *remote = &ref->old_oid;
841
0
    struct object *o;
842
843
0
    o = lookup_object(the_repository, remote);
844
0
    if (!o || !(o->flags & COMPLETE)) {
845
0
      retval = 0;
846
0
      print_verbose(args, "want %s (%s)", oid_to_hex(remote),
847
0
              ref->name);
848
0
      continue;
849
0
    }
850
0
    print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
851
0
            ref->name);
852
0
  }
853
854
0
  return retval;
855
0
}
856
857
static int sideband_demux(int in UNUSED, int out, void *data)
858
0
{
859
0
  int *xd = data;
860
0
  int ret;
861
862
0
  ret = recv_sideband("fetch-pack", xd[0], out);
863
0
  close(out);
864
0
  return ret;
865
0
}
866
867
static void create_promisor_file(const char *keep_name,
868
         struct ref **sought, int nr_sought)
869
0
{
870
0
  struct strbuf promisor_name = STRBUF_INIT;
871
0
  int suffix_stripped;
872
873
0
  strbuf_addstr(&promisor_name, keep_name);
874
0
  suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
875
0
  if (!suffix_stripped)
876
0
    BUG("name of pack lockfile should end with .keep (was '%s')",
877
0
        keep_name);
878
0
  strbuf_addstr(&promisor_name, ".promisor");
879
880
0
  write_promisor_file(promisor_name.buf, sought, nr_sought);
881
882
0
  strbuf_release(&promisor_name);
883
0
}
884
885
static void parse_gitmodules_oids(int fd, struct oidset *gitmodules_oids)
886
0
{
887
0
  int len = the_hash_algo->hexsz + 1; /* hash + NL */
888
889
0
  do {
890
0
    char hex_hash[GIT_MAX_HEXSZ + 1];
891
0
    int read_len = read_in_full(fd, hex_hash, len);
892
0
    struct object_id oid;
893
0
    const char *end;
894
895
0
    if (!read_len)
896
0
      return;
897
0
    if (read_len != len)
898
0
      die("invalid length read %d", read_len);
899
0
    if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n')
900
0
      die("invalid hash");
901
0
    oidset_insert(gitmodules_oids, &oid);
902
0
  } while (1);
903
0
}
904
905
static void add_index_pack_keep_option(struct strvec *args)
906
0
{
907
0
  char hostname[HOST_NAME_MAX + 1];
908
909
0
  if (xgethostname(hostname, sizeof(hostname)))
910
0
    xsnprintf(hostname, sizeof(hostname), "localhost");
911
0
  strvec_pushf(args, "--keep=fetch-pack %"PRIuMAX " on %s",
912
0
         (uintmax_t)getpid(), hostname);
913
0
}
914
915
/*
916
 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
917
 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
918
 * stored there. (It must be freed by the caller.)
919
 */
920
static int get_pack(struct fetch_pack_args *args,
921
        int xd[2], struct string_list *pack_lockfiles,
922
        struct strvec *index_pack_args,
923
        struct ref **sought, int nr_sought,
924
        struct oidset *gitmodules_oids)
925
0
{
926
0
  struct async demux;
927
0
  int do_keep = args->keep_pack;
928
0
  const char *cmd_name;
929
0
  struct pack_header header;
930
0
  int pass_header = 0;
931
0
  struct child_process cmd = CHILD_PROCESS_INIT;
932
0
  int fsck_objects = 0;
933
0
  int ret;
934
935
0
  memset(&demux, 0, sizeof(demux));
936
0
  if (use_sideband) {
937
    /* xd[] is talking with upload-pack; subprocess reads from
938
     * xd[0], spits out band#2 to stderr, and feeds us band#1
939
     * through demux->out.
940
     */
941
0
    demux.proc = sideband_demux;
942
0
    demux.data = xd;
943
0
    demux.out = -1;
944
0
    demux.isolate_sigpipe = 1;
945
0
    if (start_async(&demux))
946
0
      die(_("fetch-pack: unable to fork off sideband demultiplexer"));
947
0
  }
948
0
  else
949
0
    demux.out = xd[0];
950
951
0
  if (!args->keep_pack && unpack_limit && !index_pack_args) {
952
953
0
    if (read_pack_header(demux.out, &header))
954
0
      die(_("protocol error: bad pack header"));
955
0
    pass_header = 1;
956
0
    if (ntohl(header.hdr_entries) < unpack_limit)
957
0
      do_keep = 0;
958
0
    else
959
0
      do_keep = 1;
960
0
  }
961
962
0
  if (alternate_shallow_file) {
963
0
    strvec_push(&cmd.args, "--shallow-file");
964
0
    strvec_push(&cmd.args, alternate_shallow_file);
965
0
  }
966
967
0
  fsck_objects = fetch_pack_fsck_objects();
968
969
0
  if (do_keep || args->from_promisor || index_pack_args || fsck_objects) {
970
0
    if (pack_lockfiles || fsck_objects)
971
0
      cmd.out = -1;
972
0
    cmd_name = "index-pack";
973
0
    strvec_push(&cmd.args, cmd_name);
974
0
    strvec_push(&cmd.args, "--stdin");
975
0
    if (!args->quiet && !args->no_progress)
976
0
      strvec_push(&cmd.args, "-v");
977
0
    if (args->use_thin_pack)
978
0
      strvec_push(&cmd.args, "--fix-thin");
979
0
    if ((do_keep || index_pack_args) && (args->lock_pack || unpack_limit))
980
0
      add_index_pack_keep_option(&cmd.args);
981
0
    if (!index_pack_args && args->check_self_contained_and_connected)
982
0
      strvec_push(&cmd.args, "--check-self-contained-and-connected");
983
0
    else
984
      /*
985
       * We cannot perform any connectivity checks because
986
       * not all packs have been downloaded; let the caller
987
       * have this responsibility.
988
       */
989
0
      args->check_self_contained_and_connected = 0;
990
991
0
    if (args->from_promisor)
992
      /*
993
       * create_promisor_file() may be called afterwards but
994
       * we still need index-pack to know that this is a
995
       * promisor pack. For example, if transfer.fsckobjects
996
       * is true, index-pack needs to know that .gitmodules
997
       * is a promisor object (so that it won't complain if
998
       * it is missing).
999
       */
1000
0
      strvec_push(&cmd.args, "--promisor");
1001
0
  }
1002
0
  else {
1003
0
    cmd_name = "unpack-objects";
1004
0
    strvec_push(&cmd.args, cmd_name);
1005
0
    if (args->quiet || args->no_progress)
1006
0
      strvec_push(&cmd.args, "-q");
1007
0
    args->check_self_contained_and_connected = 0;
1008
0
  }
1009
1010
0
  if (pass_header)
1011
0
    strvec_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
1012
0
           ntohl(header.hdr_version),
1013
0
         ntohl(header.hdr_entries));
1014
0
  if (fsck_objects) {
1015
0
    if (args->from_promisor || index_pack_args)
1016
      /*
1017
       * We cannot use --strict in index-pack because it
1018
       * checks both broken objects and links, but we only
1019
       * want to check for broken objects.
1020
       */
1021
0
      strvec_push(&cmd.args, "--fsck-objects");
1022
0
    else
1023
0
      strvec_pushf(&cmd.args, "--strict%s",
1024
0
             fsck_msg_types.buf);
1025
0
  }
1026
1027
0
  if (index_pack_args) {
1028
0
    int i;
1029
1030
0
    for (i = 0; i < cmd.args.nr; i++)
1031
0
      strvec_push(index_pack_args, cmd.args.v[i]);
1032
0
  }
1033
1034
0
  sigchain_push(SIGPIPE, SIG_IGN);
1035
1036
0
  cmd.in = demux.out;
1037
0
  cmd.git_cmd = 1;
1038
0
  if (start_command(&cmd))
1039
0
    die(_("fetch-pack: unable to fork off %s"), cmd_name);
1040
0
  if (do_keep && (pack_lockfiles || fsck_objects)) {
1041
0
    int is_well_formed;
1042
0
    char *pack_lockfile = index_pack_lockfile(the_repository,
1043
0
                cmd.out,
1044
0
                &is_well_formed);
1045
1046
0
    if (!is_well_formed)
1047
0
      die(_("fetch-pack: invalid index-pack output"));
1048
0
    if (pack_lockfiles && pack_lockfile)
1049
0
      string_list_append_nodup(pack_lockfiles, pack_lockfile);
1050
0
    else
1051
0
      free(pack_lockfile);
1052
0
    parse_gitmodules_oids(cmd.out, gitmodules_oids);
1053
0
    close(cmd.out);
1054
0
  }
1055
1056
0
  if (!use_sideband)
1057
    /* Closed by start_command() */
1058
0
    xd[0] = -1;
1059
1060
0
  ret = finish_command(&cmd);
1061
0
  if (!ret || (args->check_self_contained_and_connected && ret == 1))
1062
0
    args->self_contained_and_connected =
1063
0
      args->check_self_contained_and_connected &&
1064
0
      ret == 0;
1065
0
  else
1066
0
    die(_("%s failed"), cmd_name);
1067
0
  if (use_sideband && finish_async(&demux))
1068
0
    die(_("error in sideband demultiplexer"));
1069
1070
0
  sigchain_pop(SIGPIPE);
1071
1072
  /*
1073
   * Now that index-pack has succeeded, write the promisor file using the
1074
   * obtained .keep filename if necessary
1075
   */
1076
0
  if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
1077
0
    create_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
1078
1079
0
  return 0;
1080
0
}
1081
1082
static int ref_compare_name(const struct ref *a, const struct ref *b)
1083
0
{
1084
0
  return strcmp(a->name, b->name);
1085
0
}
1086
1087
0
DEFINE_LIST_SORT(static, sort_ref_list, struct ref, next);
1088
0
1089
0
static int cmp_ref_by_name(const void *a_, const void *b_)
1090
0
{
1091
0
  const struct ref *a = *((const struct ref **)a_);
1092
0
  const struct ref *b = *((const struct ref **)b_);
1093
0
  return strcmp(a->name, b->name);
1094
0
}
1095
1096
static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1097
         int fd[2],
1098
         const struct ref *orig_ref,
1099
         struct ref **sought, int nr_sought,
1100
         struct shallow_info *si,
1101
         struct string_list *pack_lockfiles)
1102
0
{
1103
0
  struct repository *r = the_repository;
1104
0
  struct ref *ref = copy_ref_list(orig_ref);
1105
0
  struct object_id oid;
1106
0
  const char *agent_feature;
1107
0
  size_t agent_len;
1108
0
  struct fetch_negotiator negotiator_alloc;
1109
0
  struct fetch_negotiator *negotiator;
1110
1111
0
  negotiator = &negotiator_alloc;
1112
0
  if (args->refetch) {
1113
0
    fetch_negotiator_init_noop(negotiator);
1114
0
  } else {
1115
0
    fetch_negotiator_init(r, negotiator);
1116
0
  }
1117
1118
0
  sort_ref_list(&ref, ref_compare_name);
1119
0
  QSORT(sought, nr_sought, cmp_ref_by_name);
1120
1121
0
  if ((agent_feature = server_feature_value("agent", &agent_len))) {
1122
0
    agent_supported = 1;
1123
0
    if (agent_len)
1124
0
      print_verbose(args, _("Server version is %.*s"),
1125
0
              (int)agent_len, agent_feature);
1126
0
  }
1127
1128
0
  if (!server_supports("session-id"))
1129
0
    advertise_sid = 0;
1130
1131
0
  if (server_supports("shallow"))
1132
0
    print_verbose(args, _("Server supports %s"), "shallow");
1133
0
  else if (args->depth > 0 || is_repository_shallow(r))
1134
0
    die(_("Server does not support shallow clients"));
1135
0
  if (args->depth > 0 || args->deepen_since || args->deepen_not)
1136
0
    args->deepen = 1;
1137
0
  if (server_supports("multi_ack_detailed")) {
1138
0
    print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
1139
0
    multi_ack = 2;
1140
0
    if (server_supports("no-done")) {
1141
0
      print_verbose(args, _("Server supports %s"), "no-done");
1142
0
      if (args->stateless_rpc)
1143
0
        no_done = 1;
1144
0
    }
1145
0
  }
1146
0
  else if (server_supports("multi_ack")) {
1147
0
    print_verbose(args, _("Server supports %s"), "multi_ack");
1148
0
    multi_ack = 1;
1149
0
  }
1150
0
  if (server_supports("side-band-64k")) {
1151
0
    print_verbose(args, _("Server supports %s"), "side-band-64k");
1152
0
    use_sideband = 2;
1153
0
  }
1154
0
  else if (server_supports("side-band")) {
1155
0
    print_verbose(args, _("Server supports %s"), "side-band");
1156
0
    use_sideband = 1;
1157
0
  }
1158
0
  if (server_supports("allow-tip-sha1-in-want")) {
1159
0
    print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
1160
0
    allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1161
0
  }
1162
0
  if (server_supports("allow-reachable-sha1-in-want")) {
1163
0
    print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
1164
0
    allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1165
0
  }
1166
0
  if (server_supports("thin-pack"))
1167
0
    print_verbose(args, _("Server supports %s"), "thin-pack");
1168
0
  else
1169
0
    args->use_thin_pack = 0;
1170
0
  if (server_supports("no-progress"))
1171
0
    print_verbose(args, _("Server supports %s"), "no-progress");
1172
0
  else
1173
0
    args->no_progress = 0;
1174
0
  if (server_supports("include-tag"))
1175
0
    print_verbose(args, _("Server supports %s"), "include-tag");
1176
0
  else
1177
0
    args->include_tag = 0;
1178
0
  if (server_supports("ofs-delta"))
1179
0
    print_verbose(args, _("Server supports %s"), "ofs-delta");
1180
0
  else
1181
0
    prefer_ofs_delta = 0;
1182
1183
0
  if (server_supports("filter")) {
1184
0
    server_supports_filtering = 1;
1185
0
    print_verbose(args, _("Server supports %s"), "filter");
1186
0
  } else if (args->filter_options.choice) {
1187
0
    warning("filtering not recognized by server, ignoring");
1188
0
  }
1189
1190
0
  if (server_supports("deepen-since")) {
1191
0
    print_verbose(args, _("Server supports %s"), "deepen-since");
1192
0
    deepen_since_ok = 1;
1193
0
  } else if (args->deepen_since)
1194
0
    die(_("Server does not support --shallow-since"));
1195
0
  if (server_supports("deepen-not")) {
1196
0
    print_verbose(args, _("Server supports %s"), "deepen-not");
1197
0
    deepen_not_ok = 1;
1198
0
  } else if (args->deepen_not)
1199
0
    die(_("Server does not support --shallow-exclude"));
1200
0
  if (server_supports("deepen-relative"))
1201
0
    print_verbose(args, _("Server supports %s"), "deepen-relative");
1202
0
  else if (args->deepen_relative)
1203
0
    die(_("Server does not support --deepen"));
1204
0
  if (!server_supports_hash(the_hash_algo->name, NULL))
1205
0
    die(_("Server does not support this repository's object format"));
1206
1207
0
  mark_complete_and_common_ref(negotiator, args, &ref);
1208
0
  filter_refs(args, &ref, sought, nr_sought);
1209
0
  if (!args->refetch && everything_local(args, &ref)) {
1210
0
    packet_flush(fd[1]);
1211
0
    goto all_done;
1212
0
  }
1213
0
  if (find_common(negotiator, args, fd, &oid, ref) < 0)
1214
0
    if (!args->keep_pack)
1215
      /* When cloning, it is not unusual to have
1216
       * no common commit.
1217
       */
1218
0
      warning(_("no common commits"));
1219
1220
0
  if (args->stateless_rpc)
1221
0
    packet_flush(fd[1]);
1222
0
  if (args->deepen)
1223
0
    setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1224
0
          NULL);
1225
0
  else if (si->nr_ours || si->nr_theirs) {
1226
0
    if (args->reject_shallow_remote)
1227
0
      die(_("source repository is shallow, reject to clone."));
1228
0
    alternate_shallow_file = setup_temporary_shallow(si->shallow);
1229
0
  } else
1230
0
    alternate_shallow_file = NULL;
1231
0
  if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
1232
0
         &fsck_options.gitmodules_found))
1233
0
    die(_("git fetch-pack: fetch failed."));
1234
0
  if (fsck_finish(&fsck_options))
1235
0
    die("fsck failed");
1236
1237
0
 all_done:
1238
0
  if (negotiator)
1239
0
    negotiator->release(negotiator);
1240
0
  return ref;
1241
0
}
1242
1243
static void add_shallow_requests(struct strbuf *req_buf,
1244
         const struct fetch_pack_args *args)
1245
0
{
1246
0
  if (is_repository_shallow(the_repository))
1247
0
    write_shallow_commits(req_buf, 1, NULL);
1248
0
  if (args->depth > 0)
1249
0
    packet_buf_write(req_buf, "deepen %d", args->depth);
1250
0
  if (args->deepen_since) {
1251
0
    timestamp_t max_age = approxidate(args->deepen_since);
1252
0
    packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1253
0
  }
1254
0
  if (args->deepen_not) {
1255
0
    int i;
1256
0
    for (i = 0; i < args->deepen_not->nr; i++) {
1257
0
      struct string_list_item *s = args->deepen_not->items + i;
1258
0
      packet_buf_write(req_buf, "deepen-not %s", s->string);
1259
0
    }
1260
0
  }
1261
0
  if (args->deepen_relative)
1262
0
    packet_buf_write(req_buf, "deepen-relative\n");
1263
0
}
1264
1265
static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1266
0
{
1267
0
  int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1268
1269
0
  for ( ; wants ; wants = wants->next) {
1270
0
    const struct object_id *remote = &wants->old_oid;
1271
0
    struct object *o;
1272
1273
    /*
1274
     * If that object is complete (i.e. it is an ancestor of a
1275
     * local ref), we tell them we have it but do not have to
1276
     * tell them about its ancestors, which they already know
1277
     * about.
1278
     *
1279
     * We use lookup_object here because we are only
1280
     * interested in the case we *know* the object is
1281
     * reachable and we have already scanned it.
1282
     */
1283
0
    if (((o = lookup_object(the_repository, remote)) != NULL) &&
1284
0
        (o->flags & COMPLETE)) {
1285
0
      continue;
1286
0
    }
1287
1288
0
    if (!use_ref_in_want || wants->exact_oid)
1289
0
      packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1290
0
    else
1291
0
      packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1292
0
  }
1293
0
}
1294
1295
static void add_common(struct strbuf *req_buf, struct oidset *common)
1296
0
{
1297
0
  struct oidset_iter iter;
1298
0
  const struct object_id *oid;
1299
0
  oidset_iter_init(common, &iter);
1300
1301
0
  while ((oid = oidset_iter_next(&iter))) {
1302
0
    packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1303
0
  }
1304
0
}
1305
1306
static int add_haves(struct fetch_negotiator *negotiator,
1307
         struct strbuf *req_buf,
1308
         int *haves_to_send)
1309
0
{
1310
0
  int haves_added = 0;
1311
0
  const struct object_id *oid;
1312
1313
0
  while ((oid = negotiator->next(negotiator))) {
1314
0
    packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1315
0
    if (++haves_added >= *haves_to_send)
1316
0
      break;
1317
0
  }
1318
1319
  /* Increase haves to send on next round */
1320
0
  *haves_to_send = next_flush(1, *haves_to_send);
1321
1322
0
  return haves_added;
1323
0
}
1324
1325
static void write_fetch_command_and_capabilities(struct strbuf *req_buf,
1326
             const struct string_list *server_options)
1327
0
{
1328
0
  const char *hash_name;
1329
1330
0
  ensure_server_supports_v2("fetch");
1331
0
  packet_buf_write(req_buf, "command=fetch");
1332
0
  if (server_supports_v2("agent"))
1333
0
    packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized());
1334
0
  if (advertise_sid && server_supports_v2("session-id"))
1335
0
    packet_buf_write(req_buf, "session-id=%s", trace2_session_id());
1336
0
  if (server_options && server_options->nr) {
1337
0
    int i;
1338
0
    ensure_server_supports_v2("server-option");
1339
0
    for (i = 0; i < server_options->nr; i++)
1340
0
      packet_buf_write(req_buf, "server-option=%s",
1341
0
           server_options->items[i].string);
1342
0
  }
1343
1344
0
  if (server_feature_v2("object-format", &hash_name)) {
1345
0
    int hash_algo = hash_algo_by_name(hash_name);
1346
0
    if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
1347
0
      die(_("mismatched algorithms: client %s; server %s"),
1348
0
          the_hash_algo->name, hash_name);
1349
0
    packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name);
1350
0
  } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1_LEGACY) {
1351
0
    die(_("the server does not support algorithm '%s'"),
1352
0
        the_hash_algo->name);
1353
0
  }
1354
0
  packet_buf_delim(req_buf);
1355
0
}
1356
1357
static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1358
            struct fetch_pack_args *args,
1359
            const struct ref *wants, struct oidset *common,
1360
            int *haves_to_send, int *in_vain,
1361
            int sideband_all, int seen_ack)
1362
0
{
1363
0
  int haves_added;
1364
0
  int done_sent = 0;
1365
0
  struct strbuf req_buf = STRBUF_INIT;
1366
1367
0
  write_fetch_command_and_capabilities(&req_buf, args->server_options);
1368
1369
0
  if (args->use_thin_pack)
1370
0
    packet_buf_write(&req_buf, "thin-pack");
1371
0
  if (args->no_progress)
1372
0
    packet_buf_write(&req_buf, "no-progress");
1373
0
  if (args->include_tag)
1374
0
    packet_buf_write(&req_buf, "include-tag");
1375
0
  if (prefer_ofs_delta)
1376
0
    packet_buf_write(&req_buf, "ofs-delta");
1377
0
  if (sideband_all)
1378
0
    packet_buf_write(&req_buf, "sideband-all");
1379
1380
  /* Add shallow-info and deepen request */
1381
0
  if (server_supports_feature("fetch", "shallow", 0))
1382
0
    add_shallow_requests(&req_buf, args);
1383
0
  else if (is_repository_shallow(the_repository) || args->deepen)
1384
0
    die(_("Server does not support shallow requests"));
1385
1386
  /* Add filter */
1387
0
  send_filter(args, &req_buf,
1388
0
        server_supports_feature("fetch", "filter", 0));
1389
1390
0
  if (server_supports_feature("fetch", "packfile-uris", 0)) {
1391
0
    int i;
1392
0
    struct strbuf to_send = STRBUF_INIT;
1393
1394
0
    for (i = 0; i < uri_protocols.nr; i++) {
1395
0
      const char *s = uri_protocols.items[i].string;
1396
1397
0
      if (!strcmp(s, "https") || !strcmp(s, "http")) {
1398
0
        if (to_send.len)
1399
0
          strbuf_addch(&to_send, ',');
1400
0
        strbuf_addstr(&to_send, s);
1401
0
      }
1402
0
    }
1403
0
    if (to_send.len) {
1404
0
      packet_buf_write(&req_buf, "packfile-uris %s",
1405
0
           to_send.buf);
1406
0
      strbuf_release(&to_send);
1407
0
    }
1408
0
  }
1409
1410
  /* add wants */
1411
0
  add_wants(wants, &req_buf);
1412
1413
  /* Add all of the common commits we've found in previous rounds */
1414
0
  add_common(&req_buf, common);
1415
1416
0
  haves_added = add_haves(negotiator, &req_buf, haves_to_send);
1417
0
  *in_vain += haves_added;
1418
0
  trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added);
1419
0
  trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain);
1420
0
  if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
1421
    /* Send Done */
1422
0
    packet_buf_write(&req_buf, "done\n");
1423
0
    done_sent = 1;
1424
0
  }
1425
1426
  /* Send request */
1427
0
  packet_buf_flush(&req_buf);
1428
0
  if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1429
0
    die_errno(_("unable to write request to remote"));
1430
1431
0
  strbuf_release(&req_buf);
1432
0
  return done_sent;
1433
0
}
1434
1435
/*
1436
 * Processes a section header in a server's response and checks if it matches
1437
 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1438
 * not consumed); if 0, the line will be consumed and the function will die if
1439
 * the section header doesn't match what was expected.
1440
 */
1441
static int process_section_header(struct packet_reader *reader,
1442
          const char *section, int peek)
1443
0
{
1444
0
  int ret = 0;
1445
1446
0
  if (packet_reader_peek(reader) == PACKET_READ_NORMAL &&
1447
0
      !strcmp(reader->line, section))
1448
0
    ret = 1;
1449
1450
0
  if (!peek) {
1451
0
    if (!ret) {
1452
0
      if (reader->line)
1453
0
        die(_("expected '%s', received '%s'"),
1454
0
            section, reader->line);
1455
0
      else
1456
0
        die(_("expected '%s'"), section);
1457
0
    }
1458
0
    packet_reader_read(reader);
1459
0
  }
1460
1461
0
  return ret;
1462
0
}
1463
1464
static int process_ack(struct fetch_negotiator *negotiator,
1465
           struct packet_reader *reader,
1466
           struct object_id *common_oid,
1467
           int *received_ready)
1468
0
{
1469
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1470
0
    const char *arg;
1471
1472
0
    if (!strcmp(reader->line, "NAK"))
1473
0
      continue;
1474
1475
0
    if (skip_prefix(reader->line, "ACK ", &arg)) {
1476
0
      if (!get_oid_hex(arg, common_oid)) {
1477
0
        struct commit *commit;
1478
0
        commit = lookup_commit(the_repository, common_oid);
1479
0
        if (negotiator)
1480
0
          negotiator->ack(negotiator, commit);
1481
0
      }
1482
0
      return 1;
1483
0
    }
1484
1485
0
    if (!strcmp(reader->line, "ready")) {
1486
0
      *received_ready = 1;
1487
0
      continue;
1488
0
    }
1489
1490
0
    die(_("unexpected acknowledgment line: '%s'"), reader->line);
1491
0
  }
1492
1493
0
  if (reader->status != PACKET_READ_FLUSH &&
1494
0
      reader->status != PACKET_READ_DELIM)
1495
0
    die(_("error processing acks: %d"), reader->status);
1496
1497
  /*
1498
   * If an "acknowledgments" section is sent, a packfile is sent if and
1499
   * only if "ready" was sent in this section. The other sections
1500
   * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1501
   * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1502
   * otherwise.
1503
   */
1504
0
  if (*received_ready && reader->status != PACKET_READ_DELIM)
1505
    /*
1506
     * TRANSLATORS: The parameter will be 'ready', a protocol
1507
     * keyword.
1508
     */
1509
0
    die(_("expected packfile to be sent after '%s'"), "ready");
1510
0
  if (!*received_ready && reader->status != PACKET_READ_FLUSH)
1511
    /*
1512
     * TRANSLATORS: The parameter will be 'ready', a protocol
1513
     * keyword.
1514
     */
1515
0
    die(_("expected no other sections to be sent after no '%s'"), "ready");
1516
1517
0
  return 0;
1518
0
}
1519
1520
static void receive_shallow_info(struct fetch_pack_args *args,
1521
         struct packet_reader *reader,
1522
         struct oid_array *shallows,
1523
         struct shallow_info *si)
1524
0
{
1525
0
  int unshallow_received = 0;
1526
1527
0
  process_section_header(reader, "shallow-info", 0);
1528
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1529
0
    const char *arg;
1530
0
    struct object_id oid;
1531
1532
0
    if (skip_prefix(reader->line, "shallow ", &arg)) {
1533
0
      if (get_oid_hex(arg, &oid))
1534
0
        die(_("invalid shallow line: %s"), reader->line);
1535
0
      oid_array_append(shallows, &oid);
1536
0
      continue;
1537
0
    }
1538
0
    if (skip_prefix(reader->line, "unshallow ", &arg)) {
1539
0
      if (get_oid_hex(arg, &oid))
1540
0
        die(_("invalid unshallow line: %s"), reader->line);
1541
0
      if (!lookup_object(the_repository, &oid))
1542
0
        die(_("object not found: %s"), reader->line);
1543
      /* make sure that it is parsed as shallow */
1544
0
      if (!parse_object(the_repository, &oid))
1545
0
        die(_("error in object: %s"), reader->line);
1546
0
      if (unregister_shallow(&oid))
1547
0
        die(_("no shallow found: %s"), reader->line);
1548
0
      unshallow_received = 1;
1549
0
      continue;
1550
0
    }
1551
0
    die(_("expected shallow/unshallow, got %s"), reader->line);
1552
0
  }
1553
1554
0
  if (reader->status != PACKET_READ_FLUSH &&
1555
0
      reader->status != PACKET_READ_DELIM)
1556
0
    die(_("error processing shallow info: %d"), reader->status);
1557
1558
0
  if (args->deepen || unshallow_received) {
1559
    /*
1560
     * Treat these as shallow lines caused by our depth settings.
1561
     * In v0, these lines cannot cause refs to be rejected; do the
1562
     * same.
1563
     */
1564
0
    int i;
1565
1566
0
    for (i = 0; i < shallows->nr; i++)
1567
0
      register_shallow(the_repository, &shallows->oid[i]);
1568
0
    setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1569
0
          NULL);
1570
0
    args->deepen = 1;
1571
0
  } else if (shallows->nr) {
1572
    /*
1573
     * Treat these as shallow lines caused by the remote being
1574
     * shallow. In v0, remote refs that reach these objects are
1575
     * rejected (unless --update-shallow is set); do the same.
1576
     */
1577
0
    prepare_shallow_info(si, shallows);
1578
0
    if (si->nr_ours || si->nr_theirs) {
1579
0
      if (args->reject_shallow_remote)
1580
0
        die(_("source repository is shallow, reject to clone."));
1581
0
      alternate_shallow_file =
1582
0
        setup_temporary_shallow(si->shallow);
1583
0
    } else
1584
0
      alternate_shallow_file = NULL;
1585
0
  } else {
1586
0
    alternate_shallow_file = NULL;
1587
0
  }
1588
0
}
1589
1590
static int cmp_name_ref(const void *name, const void *ref)
1591
0
{
1592
0
  return strcmp(name, (*(struct ref **)ref)->name);
1593
0
}
1594
1595
static void receive_wanted_refs(struct packet_reader *reader,
1596
        struct ref **sought, int nr_sought)
1597
0
{
1598
0
  process_section_header(reader, "wanted-refs", 0);
1599
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1600
0
    struct object_id oid;
1601
0
    const char *end;
1602
0
    struct ref **found;
1603
1604
0
    if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1605
0
      die(_("expected wanted-ref, got '%s'"), reader->line);
1606
1607
0
    found = bsearch(end, sought, nr_sought, sizeof(*sought),
1608
0
        cmp_name_ref);
1609
0
    if (!found)
1610
0
      die(_("unexpected wanted-ref: '%s'"), reader->line);
1611
0
    oidcpy(&(*found)->old_oid, &oid);
1612
0
  }
1613
1614
0
  if (reader->status != PACKET_READ_DELIM)
1615
0
    die(_("error processing wanted refs: %d"), reader->status);
1616
0
}
1617
1618
static void receive_packfile_uris(struct packet_reader *reader,
1619
          struct string_list *uris)
1620
0
{
1621
0
  process_section_header(reader, "packfile-uris", 0);
1622
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1623
0
    if (reader->pktlen < the_hash_algo->hexsz ||
1624
0
        reader->line[the_hash_algo->hexsz] != ' ')
1625
0
      die("expected '<hash> <uri>', got: %s", reader->line);
1626
1627
0
    string_list_append(uris, reader->line);
1628
0
  }
1629
0
  if (reader->status != PACKET_READ_DELIM)
1630
0
    die("expected DELIM");
1631
0
}
1632
1633
enum fetch_state {
1634
  FETCH_CHECK_LOCAL = 0,
1635
  FETCH_SEND_REQUEST,
1636
  FETCH_PROCESS_ACKS,
1637
  FETCH_GET_PACK,
1638
  FETCH_DONE,
1639
};
1640
1641
static void do_check_stateless_delimiter(int stateless_rpc,
1642
           struct packet_reader *reader)
1643
0
{
1644
0
  check_stateless_delimiter(stateless_rpc, reader,
1645
0
          _("git fetch-pack: expected response end packet"));
1646
0
}
1647
1648
static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1649
            int fd[2],
1650
            const struct ref *orig_ref,
1651
            struct ref **sought, int nr_sought,
1652
            struct oid_array *shallows,
1653
            struct shallow_info *si,
1654
            struct string_list *pack_lockfiles)
1655
0
{
1656
0
  struct repository *r = the_repository;
1657
0
  struct ref *ref = copy_ref_list(orig_ref);
1658
0
  enum fetch_state state = FETCH_CHECK_LOCAL;
1659
0
  struct oidset common = OIDSET_INIT;
1660
0
  struct packet_reader reader;
1661
0
  int in_vain = 0, negotiation_started = 0;
1662
0
  int negotiation_round = 0;
1663
0
  int haves_to_send = INITIAL_FLUSH;
1664
0
  struct fetch_negotiator negotiator_alloc;
1665
0
  struct fetch_negotiator *negotiator;
1666
0
  int seen_ack = 0;
1667
0
  struct object_id common_oid;
1668
0
  int received_ready = 0;
1669
0
  struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1670
0
  int i;
1671
0
  struct strvec index_pack_args = STRVEC_INIT;
1672
0
  const char *promisor_remote_config;
1673
1674
0
  if (server_feature_v2("promisor-remote", &promisor_remote_config))
1675
0
    promisor_remote_reply(promisor_remote_config, NULL);
1676
1677
0
  if (args->filter_options.choice == LOFC_AUTO) {
1678
0
    struct strbuf errbuf = STRBUF_INIT;
1679
0
    char *constructed_filter = promisor_remote_construct_filter(r);
1680
1681
0
    list_objects_filter_release(&args->filter_options);
1682
    /* Disallow 'auto' as a result of the resolution of this 'auto' filter below */
1683
0
    args->filter_options.allow_auto_filter = 0;
1684
1685
0
    if (constructed_filter &&
1686
0
        gently_parse_list_objects_filter(&args->filter_options,
1687
0
                 constructed_filter,
1688
0
                 &errbuf))
1689
0
      die(_("couldn't resolve 'auto' filter '%s': %s"),
1690
0
          constructed_filter, errbuf.buf);
1691
1692
0
    free(constructed_filter);
1693
0
    strbuf_release(&errbuf);
1694
0
  }
1695
1696
0
  negotiator = &negotiator_alloc;
1697
0
  if (args->refetch)
1698
0
    fetch_negotiator_init_noop(negotiator);
1699
0
  else
1700
0
    fetch_negotiator_init(r, negotiator);
1701
1702
0
  packet_reader_init(&reader, fd[0], NULL, 0,
1703
0
         PACKET_READ_CHOMP_NEWLINE |
1704
0
         PACKET_READ_DIE_ON_ERR_PACKET);
1705
0
  if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1706
0
      server_supports_feature("fetch", "sideband-all", 0)) {
1707
0
    reader.use_sideband = 1;
1708
0
    reader.me = "fetch-pack";
1709
0
  }
1710
1711
0
  while (state != FETCH_DONE) {
1712
0
    switch (state) {
1713
0
    case FETCH_CHECK_LOCAL:
1714
0
      sort_ref_list(&ref, ref_compare_name);
1715
0
      QSORT(sought, nr_sought, cmp_ref_by_name);
1716
1717
      /* v2 supports these by default */
1718
0
      allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1719
0
      use_sideband = 2;
1720
0
      if (args->depth > 0 || args->deepen_since || args->deepen_not)
1721
0
        args->deepen = 1;
1722
1723
      /* Filter 'ref' by 'sought' and those that aren't local */
1724
0
      mark_complete_and_common_ref(negotiator, args, &ref);
1725
0
      filter_refs(args, &ref, sought, nr_sought);
1726
0
      if (!args->refetch && everything_local(args, &ref))
1727
0
        state = FETCH_DONE;
1728
0
      else
1729
0
        state = FETCH_SEND_REQUEST;
1730
1731
0
      mark_tips(negotiator, args->negotiation_tips);
1732
0
      for_each_cached_alternate(negotiator,
1733
0
              insert_one_alternate_object);
1734
0
      break;
1735
0
    case FETCH_SEND_REQUEST:
1736
0
      if (!negotiation_started) {
1737
0
        negotiation_started = 1;
1738
0
        trace2_region_enter("fetch-pack",
1739
0
                "negotiation_v2",
1740
0
                the_repository);
1741
0
      }
1742
0
      negotiation_round++;
1743
0
      trace2_region_enter_printf("negotiation_v2", "round",
1744
0
               the_repository, "%d",
1745
0
               negotiation_round);
1746
0
      if (send_fetch_request(negotiator, fd[1], args, ref,
1747
0
                 &common,
1748
0
                 &haves_to_send, &in_vain,
1749
0
                 reader.use_sideband,
1750
0
                 seen_ack)) {
1751
0
        trace2_region_leave_printf("negotiation_v2", "round",
1752
0
                 the_repository, "%d",
1753
0
                 negotiation_round);
1754
0
        state = FETCH_GET_PACK;
1755
0
      }
1756
0
      else
1757
0
        state = FETCH_PROCESS_ACKS;
1758
0
      break;
1759
0
    case FETCH_PROCESS_ACKS:
1760
      /* Process ACKs/NAKs */
1761
0
      process_section_header(&reader, "acknowledgments", 0);
1762
0
      while (process_ack(negotiator, &reader, &common_oid,
1763
0
             &received_ready)) {
1764
0
        in_vain = 0;
1765
0
        seen_ack = 1;
1766
0
        oidset_insert(&common, &common_oid);
1767
0
      }
1768
0
      trace2_region_leave_printf("negotiation_v2", "round",
1769
0
               the_repository, "%d",
1770
0
               negotiation_round);
1771
0
      if (received_ready) {
1772
        /*
1773
         * Don't check for response delimiter; get_pack() will
1774
         * read the rest of this response.
1775
         */
1776
0
        state = FETCH_GET_PACK;
1777
0
      } else {
1778
0
        do_check_stateless_delimiter(args->stateless_rpc, &reader);
1779
0
        state = FETCH_SEND_REQUEST;
1780
0
      }
1781
0
      break;
1782
0
    case FETCH_GET_PACK:
1783
0
      trace2_region_leave("fetch-pack",
1784
0
              "negotiation_v2",
1785
0
              the_repository);
1786
0
      trace2_data_intmax("negotiation_v2", the_repository,
1787
0
             "total_rounds", negotiation_round);
1788
      /* Check for shallow-info section */
1789
0
      if (process_section_header(&reader, "shallow-info", 1))
1790
0
        receive_shallow_info(args, &reader, shallows, si);
1791
1792
0
      if (process_section_header(&reader, "wanted-refs", 1))
1793
0
        receive_wanted_refs(&reader, sought, nr_sought);
1794
1795
      /* get the pack(s) */
1796
0
      if (git_env_bool("GIT_TRACE_REDACT", 1))
1797
0
        reader.options |= PACKET_READ_REDACT_URI_PATH;
1798
0
      if (process_section_header(&reader, "packfile-uris", 1))
1799
0
        receive_packfile_uris(&reader, &packfile_uris);
1800
      /* We don't expect more URIs. Reset to avoid expensive URI check. */
1801
0
      reader.options &= ~PACKET_READ_REDACT_URI_PATH;
1802
1803
0
      process_section_header(&reader, "packfile", 0);
1804
1805
      /*
1806
       * this is the final request we'll make of the server;
1807
       * do a half-duplex shutdown to indicate that they can
1808
       * hang up as soon as the pack is sent.
1809
       */
1810
0
      close(fd[1]);
1811
0
      fd[1] = -1;
1812
1813
0
      if (get_pack(args, fd, pack_lockfiles,
1814
0
             packfile_uris.nr ? &index_pack_args : NULL,
1815
0
             sought, nr_sought, &fsck_options.gitmodules_found))
1816
0
        die(_("git fetch-pack: fetch failed."));
1817
0
      do_check_stateless_delimiter(args->stateless_rpc, &reader);
1818
1819
0
      state = FETCH_DONE;
1820
0
      break;
1821
0
    case FETCH_DONE:
1822
0
      continue;
1823
0
    }
1824
0
  }
1825
1826
0
  for (i = 0; i < packfile_uris.nr; i++) {
1827
0
    int j;
1828
0
    struct child_process cmd = CHILD_PROCESS_INIT;
1829
0
    char packname[GIT_MAX_HEXSZ + 1];
1830
0
    const char *uri = packfile_uris.items[i].string +
1831
0
      the_hash_algo->hexsz + 1;
1832
1833
0
    strvec_push(&cmd.args, "http-fetch");
1834
0
    strvec_pushf(&cmd.args, "--packfile=%.*s",
1835
0
           (int) the_hash_algo->hexsz,
1836
0
           packfile_uris.items[i].string);
1837
0
    for (j = 0; j < index_pack_args.nr; j++)
1838
0
      strvec_pushf(&cmd.args, "--index-pack-arg=%s",
1839
0
             index_pack_args.v[j]);
1840
0
    strvec_push(&cmd.args, uri);
1841
0
    cmd.git_cmd = 1;
1842
0
    cmd.no_stdin = 1;
1843
0
    cmd.out = -1;
1844
0
    if (start_command(&cmd))
1845
0
      die("fetch-pack: unable to spawn http-fetch");
1846
1847
0
    if (read_in_full(cmd.out, packname, 5) < 0 ||
1848
0
        memcmp(packname, "keep\t", 5))
1849
0
      die("fetch-pack: expected keep then TAB at start of http-fetch output");
1850
1851
0
    if (read_in_full(cmd.out, packname,
1852
0
         the_hash_algo->hexsz + 1) < 0 ||
1853
0
        packname[the_hash_algo->hexsz] != '\n')
1854
0
      die("fetch-pack: expected hash then LF at end of http-fetch output");
1855
1856
0
    packname[the_hash_algo->hexsz] = '\0';
1857
1858
0
    parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
1859
1860
0
    close(cmd.out);
1861
1862
0
    if (finish_command(&cmd))
1863
0
      die("fetch-pack: unable to finish http-fetch");
1864
1865
0
    if (memcmp(packfile_uris.items[i].string, packname,
1866
0
         the_hash_algo->hexsz))
1867
0
      die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1868
0
          uri, (int) the_hash_algo->hexsz,
1869
0
          packfile_uris.items[i].string);
1870
1871
0
    string_list_append_nodup(pack_lockfiles,
1872
0
           xstrfmt("%s/pack/pack-%s.keep",
1873
0
             repo_get_object_directory(the_repository),
1874
0
             packname));
1875
0
  }
1876
0
  string_list_clear(&packfile_uris, 0);
1877
0
  strvec_clear(&index_pack_args);
1878
1879
0
  if (fsck_finish(&fsck_options))
1880
0
    die("fsck failed");
1881
1882
0
  if (negotiator)
1883
0
    negotiator->release(negotiator);
1884
1885
0
  oidset_clear(&common);
1886
0
  return ref;
1887
0
}
1888
1889
int fetch_pack_fsck_config(const char *var, const char *value,
1890
         struct strbuf *msg_types)
1891
0
{
1892
0
  const char *msg_id;
1893
1894
0
  if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1895
0
    char *path ;
1896
1897
0
    if (git_config_pathname(&path, var, value))
1898
0
      return -1;
1899
0
    if (path)
1900
0
      strbuf_addf(msg_types, "%cskiplist=%s",
1901
0
            msg_types->len ? ',' : '=', path);
1902
0
    free(path);
1903
0
    return 0;
1904
0
  }
1905
1906
0
  if (skip_prefix(var, "fetch.fsck.", &msg_id)) {
1907
0
    if (!value)
1908
0
      return config_error_nonbool(var);
1909
0
    if (is_valid_msg_type(msg_id, value))
1910
0
      strbuf_addf(msg_types, "%c%s=%s",
1911
0
        msg_types->len ? ',' : '=', msg_id, value);
1912
0
    else
1913
0
      warning("Skipping unknown msg id '%s'", msg_id);
1914
0
    return 0;
1915
0
  }
1916
1917
0
  return 1;
1918
0
}
1919
1920
static int fetch_pack_config_cb(const char *var, const char *value,
1921
        const struct config_context *ctx, void *cb)
1922
0
{
1923
0
  int ret = fetch_pack_fsck_config(var, value, &fsck_msg_types);
1924
0
  if (ret > 0)
1925
0
    return git_default_config(var, value, ctx, cb);
1926
1927
0
  return ret;
1928
0
}
1929
1930
static void fetch_pack_config(void)
1931
0
{
1932
0
  repo_config_get_int(the_repository, "fetch.unpacklimit", &fetch_unpack_limit);
1933
0
  repo_config_get_int(the_repository, "transfer.unpacklimit", &transfer_unpack_limit);
1934
0
  repo_config_get_bool(the_repository, "repack.usedeltabaseoffset", &prefer_ofs_delta);
1935
0
  repo_config_get_bool(the_repository, "fetch.fsckobjects", &fetch_fsck_objects);
1936
0
  repo_config_get_bool(the_repository, "transfer.fsckobjects", &transfer_fsck_objects);
1937
0
  repo_config_get_bool(the_repository, "transfer.advertisesid", &advertise_sid);
1938
0
  if (!uri_protocols.nr) {
1939
0
    char *str;
1940
1941
0
    if (!repo_config_get_string(the_repository, "fetch.uriprotocols", &str) && str) {
1942
0
      string_list_split(&uri_protocols, str, ",", -1);
1943
0
      free(str);
1944
0
    }
1945
0
  }
1946
1947
0
  repo_config(the_repository, fetch_pack_config_cb, NULL);
1948
0
}
1949
1950
static void fetch_pack_setup(void)
1951
0
{
1952
0
  static int did_setup;
1953
0
  if (did_setup)
1954
0
    return;
1955
0
  fetch_pack_config();
1956
0
  if (0 <= fetch_unpack_limit)
1957
0
    unpack_limit = fetch_unpack_limit;
1958
0
  else if (0 <= transfer_unpack_limit)
1959
0
    unpack_limit = transfer_unpack_limit;
1960
0
  did_setup = 1;
1961
0
}
1962
1963
static int remove_duplicates_in_refs(struct ref **ref, int nr)
1964
0
{
1965
0
  struct string_list names = STRING_LIST_INIT_NODUP;
1966
0
  int src, dst;
1967
1968
0
  for (src = dst = 0; src < nr; src++) {
1969
0
    struct string_list_item *item;
1970
0
    item = string_list_insert(&names, ref[src]->name);
1971
0
    if (item->util)
1972
0
      continue; /* already have it */
1973
0
    item->util = ref[src];
1974
0
    if (src != dst)
1975
0
      ref[dst] = ref[src];
1976
0
    dst++;
1977
0
  }
1978
0
  for (src = dst; src < nr; src++)
1979
0
    ref[src] = NULL;
1980
0
  string_list_clear(&names, 0);
1981
0
  return dst;
1982
0
}
1983
1984
static void update_shallow(struct fetch_pack_args *args,
1985
         struct ref **sought, int nr_sought,
1986
         struct shallow_info *si)
1987
0
{
1988
0
  struct oid_array ref = OID_ARRAY_INIT;
1989
0
  int *status;
1990
0
  int i;
1991
1992
0
  if (args->deepen && alternate_shallow_file) {
1993
0
    if (*alternate_shallow_file == '\0') { /* --unshallow */
1994
0
      unlink_or_warn(git_path_shallow(the_repository));
1995
0
      rollback_shallow_file(the_repository, &shallow_lock);
1996
0
    } else
1997
0
      commit_shallow_file(the_repository, &shallow_lock);
1998
0
    alternate_shallow_file = NULL;
1999
0
    return;
2000
0
  }
2001
2002
0
  if (!si->shallow || !si->shallow->nr)
2003
0
    return;
2004
2005
0
  if (args->cloning) {
2006
    /*
2007
     * remote is shallow, but this is a clone, there are
2008
     * no objects in repo to worry about. Accept any
2009
     * shallow points that exist in the pack (iow in repo
2010
     * after get_pack() and odb_reprepare())
2011
     */
2012
0
    struct oid_array extra = OID_ARRAY_INIT;
2013
0
    struct object_id *oid = si->shallow->oid;
2014
0
    for (i = 0; i < si->shallow->nr; i++)
2015
0
      if (odb_has_object(the_repository->objects, &oid[i],
2016
0
             HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))
2017
0
        oid_array_append(&extra, &oid[i]);
2018
0
    if (extra.nr) {
2019
0
      setup_alternate_shallow(&shallow_lock,
2020
0
            &alternate_shallow_file,
2021
0
            &extra);
2022
0
      commit_shallow_file(the_repository, &shallow_lock);
2023
0
      alternate_shallow_file = NULL;
2024
0
    }
2025
0
    oid_array_clear(&extra);
2026
0
    return;
2027
0
  }
2028
2029
0
  if (!si->nr_ours && !si->nr_theirs)
2030
0
    return;
2031
2032
0
  remove_nonexistent_theirs_shallow(si);
2033
0
  if (!si->nr_ours && !si->nr_theirs)
2034
0
    return;
2035
0
  for (i = 0; i < nr_sought; i++)
2036
0
    oid_array_append(&ref, &sought[i]->old_oid);
2037
0
  si->ref = &ref;
2038
2039
0
  if (args->update_shallow) {
2040
    /*
2041
     * remote is also shallow, .git/shallow may be updated
2042
     * so all refs can be accepted. Make sure we only add
2043
     * shallow roots that are actually reachable from new
2044
     * refs.
2045
     */
2046
0
    struct oid_array extra = OID_ARRAY_INIT;
2047
0
    struct object_id *oid = si->shallow->oid;
2048
0
    assign_shallow_commits_to_refs(si, NULL, NULL);
2049
0
    if (!si->nr_ours && !si->nr_theirs) {
2050
0
      oid_array_clear(&ref);
2051
0
      return;
2052
0
    }
2053
0
    for (i = 0; i < si->nr_ours; i++)
2054
0
      oid_array_append(&extra, &oid[si->ours[i]]);
2055
0
    for (i = 0; i < si->nr_theirs; i++)
2056
0
      oid_array_append(&extra, &oid[si->theirs[i]]);
2057
0
    setup_alternate_shallow(&shallow_lock,
2058
0
          &alternate_shallow_file,
2059
0
          &extra);
2060
0
    commit_shallow_file(the_repository, &shallow_lock);
2061
0
    oid_array_clear(&extra);
2062
0
    oid_array_clear(&ref);
2063
0
    alternate_shallow_file = NULL;
2064
0
    return;
2065
0
  }
2066
2067
  /*
2068
   * remote is also shallow, check what ref is safe to update
2069
   * without updating .git/shallow
2070
   */
2071
0
  CALLOC_ARRAY(status, nr_sought);
2072
0
  assign_shallow_commits_to_refs(si, NULL, status);
2073
0
  if (si->nr_ours || si->nr_theirs) {
2074
0
    for (i = 0; i < nr_sought; i++)
2075
0
      if (status[i])
2076
0
        sought[i]->status = REF_STATUS_REJECT_SHALLOW;
2077
0
  }
2078
0
  free(status);
2079
0
  oid_array_clear(&ref);
2080
0
}
2081
2082
static const struct object_id *iterate_ref_map(void *cb_data)
2083
0
{
2084
0
  struct ref **rm = cb_data;
2085
0
  struct ref *ref = *rm;
2086
2087
0
  if (!ref)
2088
0
    return NULL;
2089
0
  *rm = ref->next;
2090
0
  return &ref->old_oid;
2091
0
}
2092
2093
int fetch_pack_fsck_objects(void)
2094
0
{
2095
0
  fetch_pack_setup();
2096
0
  if (fetch_fsck_objects >= 0)
2097
0
    return fetch_fsck_objects;
2098
0
  if (transfer_fsck_objects >= 0)
2099
0
    return transfer_fsck_objects;
2100
0
  return 0;
2101
0
}
2102
2103
struct ref *fetch_pack(struct fetch_pack_args *args,
2104
           int fd[],
2105
           const struct ref *ref,
2106
           struct ref **sought, int nr_sought,
2107
           struct oid_array *shallow,
2108
           struct string_list *pack_lockfiles,
2109
           enum protocol_version version)
2110
0
{
2111
0
  struct ref *ref_cpy;
2112
0
  struct shallow_info si;
2113
0
  struct oid_array shallows_scratch = OID_ARRAY_INIT;
2114
2115
0
  fetch_pack_setup();
2116
0
  if (nr_sought)
2117
0
    nr_sought = remove_duplicates_in_refs(sought, nr_sought);
2118
2119
0
  if (version != protocol_v2 && !ref) {
2120
0
    packet_flush(fd[1]);
2121
0
    die(_("no matching remote head"));
2122
0
  }
2123
0
  if (version == protocol_v2) {
2124
0
    if (shallow->nr)
2125
0
      BUG("Protocol V2 does not provide shallows at this point in the fetch");
2126
0
    memset(&si, 0, sizeof(si));
2127
0
    ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
2128
0
             &shallows_scratch, &si,
2129
0
             pack_lockfiles);
2130
0
  } else {
2131
0
    prepare_shallow_info(&si, shallow);
2132
0
    ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
2133
0
          &si, pack_lockfiles);
2134
0
  }
2135
0
  odb_reprepare(the_repository->objects);
2136
2137
0
  if (!args->cloning && args->deepen) {
2138
0
    struct check_connected_options opt = CHECK_CONNECTED_INIT;
2139
0
    struct ref *iterator = ref_cpy;
2140
0
    opt.shallow_file = alternate_shallow_file;
2141
0
    if (args->deepen)
2142
0
      opt.is_deepening_fetch = 1;
2143
0
    if (check_connected(iterate_ref_map, &iterator, &opt)) {
2144
0
      error(_("remote did not send all necessary objects"));
2145
0
      free_refs(ref_cpy);
2146
0
      ref_cpy = NULL;
2147
0
      rollback_shallow_file(the_repository, &shallow_lock);
2148
0
      goto cleanup;
2149
0
    }
2150
0
    args->connectivity_checked = 1;
2151
0
  }
2152
2153
0
  update_shallow(args, sought, nr_sought, &si);
2154
0
cleanup:
2155
0
  clear_shallow_info(&si);
2156
0
  oid_array_clear(&shallows_scratch);
2157
0
  return ref_cpy;
2158
0
}
2159
2160
static int add_to_object_array(const struct object_id *oid, void *data)
2161
0
{
2162
0
  struct object_array *a = data;
2163
2164
0
  add_object_array(lookup_object(the_repository, oid), "", a);
2165
0
  return 0;
2166
0
}
2167
2168
static void clear_common_flag(struct oidset *s)
2169
0
{
2170
0
  struct oidset_iter iter;
2171
0
  const struct object_id *oid;
2172
0
  oidset_iter_init(s, &iter);
2173
2174
0
  while ((oid = oidset_iter_next(&iter))) {
2175
0
    struct object *obj = lookup_object(the_repository, oid);
2176
0
    obj->flags &= ~COMMON;
2177
0
  }
2178
0
}
2179
2180
void negotiate_using_fetch(const struct oid_array *negotiation_tips,
2181
         const struct string_list *server_options,
2182
         int stateless_rpc,
2183
         int fd[],
2184
         struct oidset *acked_commits)
2185
0
{
2186
0
  struct fetch_negotiator negotiator;
2187
0
  struct packet_reader reader;
2188
0
  struct object_array nt_object_array = OBJECT_ARRAY_INIT;
2189
0
  struct strbuf req_buf = STRBUF_INIT;
2190
0
  int haves_to_send = INITIAL_FLUSH;
2191
0
  int in_vain = 0;
2192
0
  int seen_ack = 0;
2193
0
  int last_iteration = 0;
2194
0
  int negotiation_round = 0;
2195
0
  timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
2196
2197
0
  fetch_negotiator_init(the_repository, &negotiator);
2198
0
  mark_tips(&negotiator, negotiation_tips);
2199
2200
0
  packet_reader_init(&reader, fd[0], NULL, 0,
2201
0
         PACKET_READ_CHOMP_NEWLINE |
2202
0
         PACKET_READ_DIE_ON_ERR_PACKET);
2203
2204
0
  oid_array_for_each((struct oid_array *) negotiation_tips,
2205
0
         add_to_object_array,
2206
0
         &nt_object_array);
2207
2208
0
  trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository);
2209
0
  while (!last_iteration) {
2210
0
    int haves_added;
2211
0
    struct object_id common_oid;
2212
0
    int received_ready = 0;
2213
2214
0
    negotiation_round++;
2215
2216
0
    trace2_region_enter_printf("negotiate_using_fetch", "round",
2217
0
             the_repository, "%d",
2218
0
             negotiation_round);
2219
0
    strbuf_reset(&req_buf);
2220
0
    write_fetch_command_and_capabilities(&req_buf, server_options);
2221
2222
0
    packet_buf_write(&req_buf, "wait-for-done");
2223
2224
0
    haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
2225
0
    in_vain += haves_added;
2226
0
    if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
2227
0
      last_iteration = 1;
2228
2229
0
    trace2_data_intmax("negotiate_using_fetch", the_repository,
2230
0
           "haves_added", haves_added);
2231
0
    trace2_data_intmax("negotiate_using_fetch", the_repository,
2232
0
           "in_vain", in_vain);
2233
2234
    /* Send request */
2235
0
    packet_buf_flush(&req_buf);
2236
0
    if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
2237
0
      die_errno(_("unable to write request to remote"));
2238
2239
    /* Process ACKs/NAKs */
2240
0
    process_section_header(&reader, "acknowledgments", 0);
2241
0
    while (process_ack(&negotiator, &reader, &common_oid,
2242
0
           &received_ready)) {
2243
0
      struct commit *commit = lookup_commit(the_repository,
2244
0
                    &common_oid);
2245
0
      if (commit) {
2246
0
        timestamp_t generation;
2247
2248
0
        parse_commit_or_die(commit);
2249
0
        commit->object.flags |= COMMON;
2250
0
        generation = commit_graph_generation(commit);
2251
0
        if (generation < min_generation)
2252
0
          min_generation = generation;
2253
0
      }
2254
0
      in_vain = 0;
2255
0
      seen_ack = 1;
2256
0
      oidset_insert(acked_commits, &common_oid);
2257
0
    }
2258
0
    if (received_ready)
2259
0
      die(_("unexpected 'ready' from remote"));
2260
0
    else
2261
0
      do_check_stateless_delimiter(stateless_rpc, &reader);
2262
0
    if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
2263
0
             REACH_SCRATCH, 0,
2264
0
             min_generation))
2265
0
      last_iteration = 1;
2266
0
    trace2_region_leave_printf("negotiation", "round",
2267
0
             the_repository, "%d",
2268
0
             negotiation_round);
2269
0
  }
2270
0
  trace2_region_leave("fetch-pack", "negotiate_using_fetch", the_repository);
2271
0
  trace2_data_intmax("negotiate_using_fetch", the_repository,
2272
0
         "total_rounds", negotiation_round);
2273
2274
0
  clear_common_flag(acked_commits);
2275
0
  object_array_clear(&nt_object_array);
2276
0
  negotiator.release(&negotiator);
2277
0
  strbuf_release(&req_buf);
2278
0
}
2279
2280
int report_unmatched_refs(struct ref **sought, int nr_sought)
2281
0
{
2282
0
  int i, ret = 0;
2283
2284
0
  for (i = 0; i < nr_sought; i++) {
2285
0
    if (!sought[i])
2286
0
      continue;
2287
0
    switch (sought[i]->match_status) {
2288
0
    case REF_MATCHED:
2289
0
      continue;
2290
0
    case REF_NOT_MATCHED:
2291
0
      error(_("no such remote ref %s"), sought[i]->name);
2292
0
      break;
2293
0
    case REF_UNADVERTISED_NOT_ALLOWED:
2294
0
      error(_("Server does not allow request for unadvertised object %s"),
2295
0
            sought[i]->name);
2296
0
      break;
2297
0
    }
2298
0
    ret = 1;
2299
0
  }
2300
0
  return ret;
2301
0
}