Coverage Report

Created: 2026-02-26 06:44

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/git/fetch-pack.c
Line
Count
Source
1
#define USE_THE_REPOSITORY_VARIABLE
2
#define DISABLE_SIGN_COMPARE_WARNINGS
3
4
#include "git-compat-util.h"
5
#include "repository.h"
6
#include "config.h"
7
#include "date.h"
8
#include "environment.h"
9
#include "gettext.h"
10
#include "hex.h"
11
#include "lockfile.h"
12
#include "refs.h"
13
#include "pkt-line.h"
14
#include "commit.h"
15
#include "tag.h"
16
#include "pack.h"
17
#include "sideband.h"
18
#include "fetch-pack.h"
19
#include "remote.h"
20
#include "run-command.h"
21
#include "connect.h"
22
#include "trace2.h"
23
#include "version.h"
24
#include "oid-array.h"
25
#include "oidset.h"
26
#include "packfile.h"
27
#include "odb.h"
28
#include "path.h"
29
#include "connected.h"
30
#include "fetch-negotiator.h"
31
#include "fsck.h"
32
#include "shallow.h"
33
#include "commit-reach.h"
34
#include "commit-graph.h"
35
#include "sigchain.h"
36
#include "mergesort.h"
37
#include "prio-queue.h"
38
#include "promisor-remote.h"
39
40
static int transfer_unpack_limit = -1;
41
static int fetch_unpack_limit = -1;
42
static int unpack_limit = 100;
43
static int prefer_ofs_delta = 1;
44
static int no_done;
45
static int deepen_since_ok;
46
static int deepen_not_ok;
47
static int fetch_fsck_objects = -1;
48
static int transfer_fsck_objects = -1;
49
static int agent_supported;
50
static int server_supports_filtering;
51
static int advertise_sid;
52
static struct shallow_lock shallow_lock;
53
static const char *alternate_shallow_file;
54
static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
55
static struct strbuf fsck_msg_types = STRBUF_INIT;
56
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
57
58
/* Remember to update object flag allocation in object.h */
59
0
#define COMPLETE  (1U << 0)
60
0
#define ALTERNATE (1U << 1)
61
0
#define COMMON    (1U << 6)
62
0
#define REACH_SCRATCH (1U << 7)
63
64
/*
65
 * After sending this many "have"s if we do not get any new ACK , we
66
 * give up traversing our history.
67
 */
68
0
#define MAX_IN_VAIN 256
69
70
static int multi_ack, use_sideband;
71
/* Allow specifying sha1 if it is a ref tip. */
72
0
#define ALLOW_TIP_SHA1  01
73
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
74
0
#define ALLOW_REACHABLE_SHA1  02
75
static unsigned int allow_unadvertised_object_request;
76
77
__attribute__((format (printf, 2, 3)))
78
static inline void print_verbose(const struct fetch_pack_args *args,
79
         const char *fmt, ...)
80
0
{
81
0
  va_list params;
82
83
0
  if (!args->verbose)
84
0
    return;
85
86
0
  va_start(params, fmt);
87
0
  vfprintf(stderr, fmt, params);
88
0
  va_end(params);
89
0
  fputc('\n', stderr);
90
0
}
91
92
struct alternate_object_cache {
93
  struct object **items;
94
  size_t nr, alloc;
95
};
96
97
static void cache_one_alternate(const struct object_id *oid,
98
        void *vcache)
99
0
{
100
0
  struct alternate_object_cache *cache = vcache;
101
0
  struct object *obj = parse_object(the_repository, oid);
102
103
0
  if (!obj || (obj->flags & ALTERNATE))
104
0
    return;
105
106
0
  obj->flags |= ALTERNATE;
107
0
  ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
108
0
  cache->items[cache->nr++] = obj;
109
0
}
110
111
static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
112
              void (*cb)(struct fetch_negotiator *,
113
             struct object *))
114
0
{
115
0
  static int initialized;
116
0
  static struct alternate_object_cache cache;
117
0
  size_t i;
118
119
0
  if (!initialized) {
120
0
    odb_for_each_alternate_ref(the_repository->objects,
121
0
             cache_one_alternate, &cache);
122
0
    initialized = 1;
123
0
  }
124
125
0
  for (i = 0; i < cache.nr; i++)
126
0
    cb(negotiator, cache.items[i]);
127
0
}
128
129
static void die_in_commit_graph_only(const struct object_id *oid)
130
0
{
131
0
  die(_("You are attempting to fetch %s, which is in the commit graph file but not in the object database.\n"
132
0
        "This is probably due to repo corruption.\n"
133
0
        "If you are attempting to repair this repo corruption by refetching the missing object, use 'git fetch --refetch' with the missing object."),
134
0
        oid_to_hex(oid));
135
0
}
136
137
static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
138
                 int mark_tags_complete_and_check_obj_db)
139
0
{
140
0
  enum object_type type;
141
0
  struct object_info info = { .typep = &type };
142
0
  struct commit *commit;
143
144
0
  commit = lookup_commit_in_graph(the_repository, oid);
145
0
  if (commit) {
146
0
    if (mark_tags_complete_and_check_obj_db) {
147
0
      if (!odb_has_object(the_repository->objects, oid,
148
0
              HAS_OBJECT_RECHECK_PACKED))
149
0
        die_in_commit_graph_only(oid);
150
0
    }
151
0
    return commit;
152
0
  }
153
154
0
  while (1) {
155
0
    if (odb_read_object_info_extended(the_repository->objects, oid, &info,
156
0
              OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK))
157
0
      return NULL;
158
0
    if (type == OBJ_TAG) {
159
0
      struct tag *tag = (struct tag *)
160
0
        parse_object(the_repository, oid);
161
162
0
      if (!tag->tagged)
163
0
        return NULL;
164
0
      if (mark_tags_complete_and_check_obj_db)
165
0
        tag->object.flags |= COMPLETE;
166
0
      oid = &tag->tagged->oid;
167
0
    } else {
168
0
      break;
169
0
    }
170
0
  }
171
172
0
  if (type == OBJ_COMMIT) {
173
0
    struct commit *commit = lookup_commit(the_repository, oid);
174
0
    if (!commit || repo_parse_commit(the_repository, commit))
175
0
      return NULL;
176
0
    return commit;
177
0
  }
178
179
0
  return NULL;
180
0
}
181
182
static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
183
             const struct object_id *oid)
184
0
{
185
0
  struct commit *c = deref_without_lazy_fetch(oid, 0);
186
187
0
  if (c)
188
0
    negotiator->add_tip(negotiator, c);
189
0
  return 0;
190
0
}
191
192
static int rev_list_insert_ref_oid(const struct reference *ref, void *cb_data)
193
0
{
194
0
  return rev_list_insert_ref(cb_data, ref->oid);
195
0
}
196
197
enum ack_type {
198
  NAK = 0,
199
  ACK,
200
  ACK_continue,
201
  ACK_common,
202
  ACK_ready
203
};
204
205
static void consume_shallow_list(struct fetch_pack_args *args,
206
         struct packet_reader *reader)
207
0
{
208
0
  if (args->stateless_rpc && args->deepen) {
209
    /* If we sent a depth we will get back "duplicate"
210
     * shallow and unshallow commands every time there
211
     * is a block of have lines exchanged.
212
     */
213
0
    while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
214
0
      if (starts_with(reader->line, "shallow "))
215
0
        continue;
216
0
      if (starts_with(reader->line, "unshallow "))
217
0
        continue;
218
0
      die(_("git fetch-pack: expected shallow list"));
219
0
    }
220
0
    if (reader->status != PACKET_READ_FLUSH)
221
0
      die(_("git fetch-pack: expected a flush packet after shallow list"));
222
0
  }
223
0
}
224
225
static enum ack_type get_ack(struct packet_reader *reader,
226
           struct object_id *result_oid)
227
0
{
228
0
  int len;
229
0
  const char *arg;
230
231
0
  if (packet_reader_read(reader) != PACKET_READ_NORMAL)
232
0
    die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
233
0
  len = reader->pktlen;
234
235
0
  if (!strcmp(reader->line, "NAK"))
236
0
    return NAK;
237
0
  if (skip_prefix(reader->line, "ACK ", &arg)) {
238
0
    const char *p;
239
0
    if (!parse_oid_hex(arg, result_oid, &p)) {
240
0
      len -= p - reader->line;
241
0
      if (len < 1)
242
0
        return ACK;
243
0
      if (strstr(p, "continue"))
244
0
        return ACK_continue;
245
0
      if (strstr(p, "common"))
246
0
        return ACK_common;
247
0
      if (strstr(p, "ready"))
248
0
        return ACK_ready;
249
0
      return ACK;
250
0
    }
251
0
  }
252
0
  die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
253
0
}
254
255
static void send_request(struct fetch_pack_args *args,
256
       int fd, struct strbuf *buf)
257
0
{
258
0
  if (args->stateless_rpc) {
259
0
    send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
260
0
    packet_flush(fd);
261
0
  } else {
262
0
    if (write_in_full(fd, buf->buf, buf->len) < 0)
263
0
      die_errno(_("unable to write to remote"));
264
0
  }
265
0
}
266
267
static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
268
          struct object *obj)
269
0
{
270
0
  rev_list_insert_ref(negotiator, &obj->oid);
271
0
}
272
273
0
#define INITIAL_FLUSH 16
274
0
#define PIPESAFE_FLUSH 32
275
0
#define LARGE_FLUSH 16384
276
277
static int next_flush(int stateless_rpc, int count)
278
0
{
279
0
  if (stateless_rpc) {
280
0
    if (count < LARGE_FLUSH)
281
0
      count <<= 1;
282
0
    else
283
0
      count = count * 11 / 10;
284
0
  } else {
285
0
    if (count < PIPESAFE_FLUSH)
286
0
      count <<= 1;
287
0
    else
288
0
      count += PIPESAFE_FLUSH;
289
0
  }
290
0
  return count;
291
0
}
292
293
static void mark_tips(struct fetch_negotiator *negotiator,
294
          const struct oid_array *negotiation_tips)
295
0
{
296
0
  int i;
297
298
0
  if (!negotiation_tips) {
299
0
    refs_for_each_rawref(get_main_ref_store(the_repository),
300
0
             rev_list_insert_ref_oid, negotiator);
301
0
    return;
302
0
  }
303
304
0
  for (i = 0; i < negotiation_tips->nr; i++)
305
0
    rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
306
0
  return;
307
0
}
308
309
static void send_filter(struct fetch_pack_args *args,
310
      struct strbuf *req_buf,
311
      int server_supports_filter)
312
0
{
313
0
  if (args->filter_options.choice) {
314
0
    const char *spec =
315
0
      expand_list_objects_filter_spec(&args->filter_options);
316
0
    if (server_supports_filter) {
317
0
      print_verbose(args, _("Server supports filter"));
318
0
      packet_buf_write(req_buf, "filter %s", spec);
319
0
      trace2_data_string("fetch", the_repository,
320
0
             "filter/effective", spec);
321
0
    } else {
322
0
      warning("filtering not recognized by server, ignoring");
323
0
      trace2_data_string("fetch", the_repository,
324
0
             "filter/unsupported", spec);
325
0
    }
326
0
  } else {
327
0
    trace2_data_string("fetch", the_repository,
328
0
           "filter/none", "");
329
0
  }
330
0
}
331
332
static int find_common(struct fetch_negotiator *negotiator,
333
           struct fetch_pack_args *args,
334
           int fd[2], struct object_id *result_oid,
335
           struct ref *refs)
336
0
{
337
0
  int fetching;
338
0
  int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
339
0
  int negotiation_round = 0, haves = 0;
340
0
  const struct object_id *oid;
341
0
  unsigned in_vain = 0;
342
0
  int got_continue = 0;
343
0
  int got_ready = 0;
344
0
  struct strbuf req_buf = STRBUF_INIT;
345
0
  size_t state_len = 0;
346
0
  struct packet_reader reader;
347
348
0
  if (args->stateless_rpc && multi_ack == 1)
349
0
    die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");
350
351
0
  packet_reader_init(&reader, fd[0], NULL, 0,
352
0
         PACKET_READ_CHOMP_NEWLINE |
353
0
         PACKET_READ_DIE_ON_ERR_PACKET);
354
355
0
  mark_tips(negotiator, args->negotiation_tips);
356
0
  for_each_cached_alternate(negotiator, insert_one_alternate_object);
357
358
0
  fetching = 0;
359
0
  for ( ; refs ; refs = refs->next) {
360
0
    struct object_id *remote = &refs->old_oid;
361
0
    const char *remote_hex;
362
0
    struct object *o;
363
364
0
    if (!args->refetch) {
365
      /*
366
      * If that object is complete (i.e. it is an ancestor of a
367
      * local ref), we tell them we have it but do not have to
368
      * tell them about its ancestors, which they already know
369
      * about.
370
      *
371
      * We use lookup_object here because we are only
372
      * interested in the case we *know* the object is
373
      * reachable and we have already scanned it.
374
      */
375
0
      if (((o = lookup_object(the_repository, remote)) != NULL) &&
376
0
          (o->flags & COMPLETE)) {
377
0
        continue;
378
0
      }
379
0
    }
380
381
0
    remote_hex = oid_to_hex(remote);
382
0
    if (!fetching) {
383
0
      struct strbuf c = STRBUF_INIT;
384
0
      if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
385
0
      if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
386
0
      if (no_done)            strbuf_addstr(&c, " no-done");
387
0
      if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
388
0
      if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
389
0
      if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
390
0
      if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
391
0
      if (args->no_progress)   strbuf_addstr(&c, " no-progress");
392
0
      if (args->include_tag)   strbuf_addstr(&c, " include-tag");
393
0
      if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
394
0
      if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
395
0
      if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
396
0
      if (agent_supported)    strbuf_addf(&c, " agent=%s",
397
0
                  git_user_agent_sanitized());
398
0
      if (advertise_sid)
399
0
        strbuf_addf(&c, " session-id=%s", trace2_session_id());
400
0
      if (args->filter_options.choice)
401
0
        strbuf_addstr(&c, " filter");
402
0
      packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
403
0
      strbuf_release(&c);
404
0
    } else
405
0
      packet_buf_write(&req_buf, "want %s\n", remote_hex);
406
0
    fetching++;
407
0
  }
408
409
0
  if (!fetching) {
410
0
    strbuf_release(&req_buf);
411
0
    packet_flush(fd[1]);
412
0
    return 1;
413
0
  }
414
415
0
  if (is_repository_shallow(the_repository))
416
0
    write_shallow_commits(&req_buf, 1, NULL);
417
0
  if (args->depth > 0)
418
0
    packet_buf_write(&req_buf, "deepen %d", args->depth);
419
0
  if (args->deepen_since) {
420
0
    timestamp_t max_age = approxidate(args->deepen_since);
421
0
    packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
422
0
  }
423
0
  if (args->deepen_not) {
424
0
    int i;
425
0
    for (i = 0; i < args->deepen_not->nr; i++) {
426
0
      struct string_list_item *s = args->deepen_not->items + i;
427
0
      packet_buf_write(&req_buf, "deepen-not %s", s->string);
428
0
    }
429
0
  }
430
0
  send_filter(args, &req_buf, server_supports_filtering);
431
0
  packet_buf_flush(&req_buf);
432
0
  state_len = req_buf.len;
433
434
0
  if (args->deepen) {
435
0
    const char *arg;
436
0
    struct object_id oid;
437
438
0
    send_request(args, fd[1], &req_buf);
439
0
    while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
440
0
      if (skip_prefix(reader.line, "shallow ", &arg)) {
441
0
        if (get_oid_hex(arg, &oid))
442
0
          die(_("invalid shallow line: %s"), reader.line);
443
0
        register_shallow(the_repository, &oid);
444
0
        continue;
445
0
      }
446
0
      if (skip_prefix(reader.line, "unshallow ", &arg)) {
447
0
        if (get_oid_hex(arg, &oid))
448
0
          die(_("invalid unshallow line: %s"), reader.line);
449
0
        if (!lookup_object(the_repository, &oid))
450
0
          die(_("object not found: %s"), reader.line);
451
        /* make sure that it is parsed as shallow */
452
0
        if (!parse_object(the_repository, &oid))
453
0
          die(_("error in object: %s"), reader.line);
454
0
        if (unregister_shallow(&oid))
455
0
          die(_("no shallow found: %s"), reader.line);
456
0
        continue;
457
0
      }
458
0
      die(_("expected shallow/unshallow, got %s"), reader.line);
459
0
    }
460
0
  } else if (!args->stateless_rpc)
461
0
    send_request(args, fd[1], &req_buf);
462
463
0
  if (!args->stateless_rpc) {
464
    /* If we aren't using the stateless-rpc interface
465
     * we don't need to retain the headers.
466
     */
467
0
    strbuf_setlen(&req_buf, 0);
468
0
    state_len = 0;
469
0
  }
470
471
0
  trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
472
0
  flushes = 0;
473
0
  retval = -1;
474
0
  while ((oid = negotiator->next(negotiator))) {
475
0
    packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
476
0
    print_verbose(args, "have %s", oid_to_hex(oid));
477
0
    in_vain++;
478
0
    haves++;
479
0
    if (flush_at <= ++count) {
480
0
      int ack;
481
482
0
      negotiation_round++;
483
0
      trace2_region_enter_printf("negotiation_v0_v1", "round",
484
0
               the_repository, "%d",
485
0
               negotiation_round);
486
0
      trace2_data_intmax("negotiation_v0_v1", the_repository,
487
0
             "haves_added", haves);
488
0
      trace2_data_intmax("negotiation_v0_v1", the_repository,
489
0
             "in_vain", in_vain);
490
0
      haves = 0;
491
0
      packet_buf_flush(&req_buf);
492
0
      send_request(args, fd[1], &req_buf);
493
0
      strbuf_setlen(&req_buf, state_len);
494
0
      flushes++;
495
0
      flush_at = next_flush(args->stateless_rpc, count);
496
497
      /*
498
       * We keep one window "ahead" of the other side, and
499
       * will wait for an ACK only on the next one
500
       */
501
0
      if (!args->stateless_rpc && count == INITIAL_FLUSH)
502
0
        continue;
503
504
0
      consume_shallow_list(args, &reader);
505
0
      do {
506
0
        ack = get_ack(&reader, result_oid);
507
0
        if (ack)
508
0
          print_verbose(args, _("got %s %d %s"), "ack",
509
0
                  ack, oid_to_hex(result_oid));
510
0
        switch (ack) {
511
0
        case ACK:
512
0
          trace2_region_leave_printf("negotiation_v0_v1", "round",
513
0
                   the_repository, "%d",
514
0
                   negotiation_round);
515
0
          flushes = 0;
516
0
          multi_ack = 0;
517
0
          retval = 0;
518
0
          goto done;
519
0
        case ACK_common:
520
0
        case ACK_ready:
521
0
        case ACK_continue: {
522
0
          struct commit *commit =
523
0
            lookup_commit(the_repository,
524
0
                    result_oid);
525
0
          int was_common;
526
527
0
          if (!commit)
528
0
            die(_("invalid commit %s"), oid_to_hex(result_oid));
529
0
          was_common = negotiator->ack(negotiator, commit);
530
0
          if (args->stateless_rpc
531
0
           && ack == ACK_common
532
0
           && !was_common) {
533
            /* We need to replay the have for this object
534
             * on the next RPC request so the peer knows
535
             * it is in common with us.
536
             */
537
0
            const char *hex = oid_to_hex(result_oid);
538
0
            packet_buf_write(&req_buf, "have %s\n", hex);
539
0
            state_len = req_buf.len;
540
0
            haves++;
541
            /*
542
             * Reset in_vain because an ack
543
             * for this commit has not been
544
             * seen.
545
             */
546
0
            in_vain = 0;
547
0
          } else if (!args->stateless_rpc
548
0
               || ack != ACK_common)
549
0
            in_vain = 0;
550
0
          retval = 0;
551
0
          got_continue = 1;
552
0
          if (ack == ACK_ready)
553
0
            got_ready = 1;
554
0
          break;
555
0
          }
556
0
        }
557
0
      } while (ack);
558
0
      flushes--;
559
0
      trace2_region_leave_printf("negotiation_v0_v1", "round",
560
0
               the_repository, "%d",
561
0
               negotiation_round);
562
0
      if (got_continue && MAX_IN_VAIN < in_vain) {
563
0
        print_verbose(args, _("giving up"));
564
0
        break; /* give up */
565
0
      }
566
0
      if (got_ready)
567
0
        break;
568
0
    }
569
0
  }
570
0
done:
571
0
  trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
572
0
  trace2_data_intmax("negotiation_v0_v1", the_repository, "total_rounds",
573
0
         negotiation_round);
574
0
  if (!got_ready || !no_done) {
575
0
    packet_buf_write(&req_buf, "done\n");
576
0
    send_request(args, fd[1], &req_buf);
577
0
  }
578
0
  print_verbose(args, _("done"));
579
0
  if (retval != 0) {
580
0
    multi_ack = 0;
581
0
    flushes++;
582
0
  }
583
0
  strbuf_release(&req_buf);
584
585
0
  if (!got_ready || !no_done)
586
0
    consume_shallow_list(args, &reader);
587
0
  while (flushes || multi_ack) {
588
0
    int ack = get_ack(&reader, result_oid);
589
0
    if (ack) {
590
0
      print_verbose(args, _("got %s (%d) %s"), "ack",
591
0
              ack, oid_to_hex(result_oid));
592
0
      if (ack == ACK)
593
0
        return 0;
594
0
      multi_ack = 1;
595
0
      continue;
596
0
    }
597
0
    flushes--;
598
0
  }
599
  /* it is no error to fetch into a completely empty repo */
600
0
  return count ? retval : 0;
601
0
}
602
603
static struct prio_queue complete = { compare_commits_by_commit_date };
604
605
static int mark_complete(const struct object_id *oid)
606
0
{
607
0
  struct commit *commit = deref_without_lazy_fetch(oid, 1);
608
609
0
  if (commit && !(commit->object.flags & COMPLETE)) {
610
0
    commit->object.flags |= COMPLETE;
611
0
    prio_queue_put(&complete, commit);
612
0
  }
613
0
  return 0;
614
0
}
615
616
static int mark_complete_oid(const struct reference *ref, void *cb_data UNUSED)
617
0
{
618
0
  return mark_complete(ref->oid);
619
0
}
620
621
static void mark_recent_complete_commits(struct fetch_pack_args *args,
622
           timestamp_t cutoff)
623
0
{
624
0
  while (complete.nr) {
625
0
    struct commit *item = prio_queue_peek(&complete);
626
0
    if (item->date < cutoff)
627
0
      break;
628
0
    print_verbose(args, _("Marking %s as complete"),
629
0
            oid_to_hex(&item->object.oid));
630
0
    pop_most_recent_commit(&complete, COMPLETE);
631
0
  }
632
0
}
633
634
static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
635
0
{
636
0
  for (; refs; refs = refs->next)
637
0
    oidset_insert(oids, &refs->old_oid);
638
0
}
639
640
static int is_unmatched_ref(const struct ref *ref)
641
0
{
642
0
  struct object_id oid;
643
0
  const char *p;
644
0
  return  ref->match_status == REF_NOT_MATCHED &&
645
0
    !parse_oid_hex(ref->name, &oid, &p) &&
646
0
    *p == '\0' &&
647
0
    oideq(&oid, &ref->old_oid);
648
0
}
649
650
static void filter_refs(struct fetch_pack_args *args,
651
      struct ref **refs,
652
      struct ref **sought, int nr_sought)
653
0
{
654
0
  struct ref *newlist = NULL;
655
0
  struct ref **newtail = &newlist;
656
0
  struct ref *unmatched = NULL;
657
0
  struct ref *ref, *next;
658
0
  struct oidset tip_oids = OIDSET_INIT;
659
0
  int i;
660
0
  int strict = !(allow_unadvertised_object_request &
661
0
           (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
662
663
0
  i = 0;
664
0
  for (ref = *refs; ref; ref = next) {
665
0
    int keep = 0;
666
0
    next = ref->next;
667
668
0
    if (starts_with(ref->name, "refs/") &&
669
0
        check_refname_format(ref->name, 0)) {
670
      /*
671
       * trash or a peeled value; do not even add it to
672
       * unmatched list
673
       */
674
0
      free_one_ref(ref);
675
0
      continue;
676
0
    } else {
677
0
      while (i < nr_sought) {
678
0
        int cmp = strcmp(ref->name, sought[i]->name);
679
0
        if (cmp < 0)
680
0
          break; /* definitely do not have it */
681
0
        else if (cmp == 0) {
682
0
          keep = 1; /* definitely have it */
683
0
          sought[i]->match_status = REF_MATCHED;
684
0
        }
685
0
        i++;
686
0
      }
687
688
0
      if (!keep && args->fetch_all &&
689
0
          (!args->deepen || !starts_with(ref->name, "refs/tags/")))
690
0
        keep = 1;
691
0
    }
692
693
0
    if (keep) {
694
0
      *newtail = ref;
695
0
      ref->next = NULL;
696
0
      newtail = &ref->next;
697
0
    } else {
698
0
      ref->next = unmatched;
699
0
      unmatched = ref;
700
0
    }
701
0
  }
702
703
0
  if (strict) {
704
0
    for (i = 0; i < nr_sought; i++) {
705
0
      ref = sought[i];
706
0
      if (!is_unmatched_ref(ref))
707
0
        continue;
708
709
0
      add_refs_to_oidset(&tip_oids, unmatched);
710
0
      add_refs_to_oidset(&tip_oids, newlist);
711
0
      break;
712
0
    }
713
0
  }
714
715
  /* Append unmatched requests to the list */
716
0
  for (i = 0; i < nr_sought; i++) {
717
0
    ref = sought[i];
718
0
    if (!is_unmatched_ref(ref))
719
0
      continue;
720
721
0
    if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
722
0
      ref->match_status = REF_MATCHED;
723
0
      *newtail = copy_ref(ref);
724
0
      newtail = &(*newtail)->next;
725
0
    } else {
726
0
      ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
727
0
    }
728
0
  }
729
730
0
  oidset_clear(&tip_oids);
731
0
  free_refs(unmatched);
732
733
0
  *refs = newlist;
734
0
}
735
736
static void mark_alternate_complete(struct fetch_negotiator *negotiator UNUSED,
737
            struct object *obj)
738
0
{
739
0
  mark_complete(&obj->oid);
740
0
}
741
742
/*
743
 * Mark recent commits available locally and reachable from a local ref as
744
 * COMPLETE.
745
 *
746
 * The cutoff time for recency is determined by this heuristic: it is the
747
 * earliest commit time of the objects in refs that are commits and that we know
748
 * the commit time of.
749
 */
750
static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
751
           struct fetch_pack_args *args,
752
           struct ref **refs)
753
0
{
754
0
  struct ref *ref;
755
0
  int old_save_commit_buffer = save_commit_buffer;
756
0
  timestamp_t cutoff = 0;
757
758
0
  if (args->refetch)
759
0
    return;
760
761
0
  save_commit_buffer = 0;
762
763
0
  trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
764
0
  for (ref = *refs; ref; ref = ref->next) {
765
0
    struct commit *commit;
766
767
0
    commit = lookup_commit_in_graph(the_repository, &ref->old_oid);
768
0
    if (!commit) {
769
0
      struct object *o;
770
771
0
      if (!odb_has_object(the_repository->objects, &ref->old_oid, 0))
772
0
        continue;
773
0
      o = parse_object(the_repository, &ref->old_oid);
774
0
      if (!o || o->type != OBJ_COMMIT)
775
0
        continue;
776
777
0
      commit = (struct commit *)o;
778
0
    }
779
780
    /*
781
     * We already have it -- which may mean that we were
782
     * in sync with the other side at some time after
783
     * that (it is OK if we guess wrong here).
784
     */
785
0
    if (!cutoff || cutoff < commit->date)
786
0
      cutoff = commit->date;
787
0
  }
788
0
  trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
789
790
  /*
791
   * This block marks all local refs as COMPLETE, and then recursively marks all
792
   * parents of those refs as COMPLETE.
793
   */
794
0
  trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
795
0
  if (!args->deepen) {
796
0
    refs_for_each_rawref(get_main_ref_store(the_repository),
797
0
             mark_complete_oid, NULL);
798
0
    for_each_cached_alternate(NULL, mark_alternate_complete);
799
0
    if (cutoff)
800
0
      mark_recent_complete_commits(args, cutoff);
801
0
  }
802
0
  trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
803
804
  /*
805
   * Mark all complete remote refs as common refs.
806
   * Don't mark them common yet; the server has to be told so first.
807
   */
808
0
  trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
809
0
  for (ref = *refs; ref; ref = ref->next) {
810
0
    struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);
811
812
0
    if (!c || !(c->object.flags & COMPLETE))
813
0
      continue;
814
815
0
    negotiator->known_common(negotiator, c);
816
0
  }
817
0
  trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
818
819
0
  save_commit_buffer = old_save_commit_buffer;
820
0
}
821
822
/*
823
 * Returns 1 if every object pointed to by the given remote refs is available
824
 * locally and reachable from a local ref, and 0 otherwise.
825
 */
826
static int everything_local(struct fetch_pack_args *args,
827
          struct ref **refs)
828
0
{
829
0
  struct ref *ref;
830
0
  int retval;
831
832
0
  for (retval = 1, ref = *refs; ref ; ref = ref->next) {
833
0
    const struct object_id *remote = &ref->old_oid;
834
0
    struct object *o;
835
836
0
    o = lookup_object(the_repository, remote);
837
0
    if (!o || !(o->flags & COMPLETE)) {
838
0
      retval = 0;
839
0
      print_verbose(args, "want %s (%s)", oid_to_hex(remote),
840
0
              ref->name);
841
0
      continue;
842
0
    }
843
0
    print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
844
0
            ref->name);
845
0
  }
846
847
0
  return retval;
848
0
}
849
850
static int sideband_demux(int in UNUSED, int out, void *data)
851
0
{
852
0
  int *xd = data;
853
0
  int ret;
854
855
0
  ret = recv_sideband("fetch-pack", xd[0], out);
856
0
  close(out);
857
0
  return ret;
858
0
}
859
860
static void create_promisor_file(const char *keep_name,
861
         struct ref **sought, int nr_sought)
862
0
{
863
0
  struct strbuf promisor_name = STRBUF_INIT;
864
0
  int suffix_stripped;
865
866
0
  strbuf_addstr(&promisor_name, keep_name);
867
0
  suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
868
0
  if (!suffix_stripped)
869
0
    BUG("name of pack lockfile should end with .keep (was '%s')",
870
0
        keep_name);
871
0
  strbuf_addstr(&promisor_name, ".promisor");
872
873
0
  write_promisor_file(promisor_name.buf, sought, nr_sought);
874
875
0
  strbuf_release(&promisor_name);
876
0
}
877
878
static void parse_gitmodules_oids(int fd, struct oidset *gitmodules_oids)
879
0
{
880
0
  int len = the_hash_algo->hexsz + 1; /* hash + NL */
881
882
0
  do {
883
0
    char hex_hash[GIT_MAX_HEXSZ + 1];
884
0
    int read_len = read_in_full(fd, hex_hash, len);
885
0
    struct object_id oid;
886
0
    const char *end;
887
888
0
    if (!read_len)
889
0
      return;
890
0
    if (read_len != len)
891
0
      die("invalid length read %d", read_len);
892
0
    if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n')
893
0
      die("invalid hash");
894
0
    oidset_insert(gitmodules_oids, &oid);
895
0
  } while (1);
896
0
}
897
898
static void add_index_pack_keep_option(struct strvec *args)
899
0
{
900
0
  char hostname[HOST_NAME_MAX + 1];
901
902
0
  if (xgethostname(hostname, sizeof(hostname)))
903
0
    xsnprintf(hostname, sizeof(hostname), "localhost");
904
0
  strvec_pushf(args, "--keep=fetch-pack %"PRIuMAX " on %s",
905
0
         (uintmax_t)getpid(), hostname);
906
0
}
907
908
/*
909
 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
910
 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
911
 * stored there. (It must be freed by the caller.)
912
 */
913
static int get_pack(struct fetch_pack_args *args,
914
        int xd[2], struct string_list *pack_lockfiles,
915
        struct strvec *index_pack_args,
916
        struct ref **sought, int nr_sought,
917
        struct oidset *gitmodules_oids)
918
0
{
919
0
  struct async demux;
920
0
  int do_keep = args->keep_pack;
921
0
  const char *cmd_name;
922
0
  struct pack_header header;
923
0
  int pass_header = 0;
924
0
  struct child_process cmd = CHILD_PROCESS_INIT;
925
0
  int fsck_objects = 0;
926
0
  int ret;
927
928
0
  memset(&demux, 0, sizeof(demux));
929
0
  if (use_sideband) {
930
    /* xd[] is talking with upload-pack; subprocess reads from
931
     * xd[0], spits out band#2 to stderr, and feeds us band#1
932
     * through demux->out.
933
     */
934
0
    demux.proc = sideband_demux;
935
0
    demux.data = xd;
936
0
    demux.out = -1;
937
0
    demux.isolate_sigpipe = 1;
938
0
    if (start_async(&demux))
939
0
      die(_("fetch-pack: unable to fork off sideband demultiplexer"));
940
0
  }
941
0
  else
942
0
    demux.out = xd[0];
943
944
0
  if (!args->keep_pack && unpack_limit && !index_pack_args) {
945
946
0
    if (read_pack_header(demux.out, &header))
947
0
      die(_("protocol error: bad pack header"));
948
0
    pass_header = 1;
949
0
    if (ntohl(header.hdr_entries) < unpack_limit)
950
0
      do_keep = 0;
951
0
    else
952
0
      do_keep = 1;
953
0
  }
954
955
0
  if (alternate_shallow_file) {
956
0
    strvec_push(&cmd.args, "--shallow-file");
957
0
    strvec_push(&cmd.args, alternate_shallow_file);
958
0
  }
959
960
0
  fsck_objects = fetch_pack_fsck_objects();
961
962
0
  if (do_keep || args->from_promisor || index_pack_args || fsck_objects) {
963
0
    if (pack_lockfiles || fsck_objects)
964
0
      cmd.out = -1;
965
0
    cmd_name = "index-pack";
966
0
    strvec_push(&cmd.args, cmd_name);
967
0
    strvec_push(&cmd.args, "--stdin");
968
0
    if (!args->quiet && !args->no_progress)
969
0
      strvec_push(&cmd.args, "-v");
970
0
    if (args->use_thin_pack)
971
0
      strvec_push(&cmd.args, "--fix-thin");
972
0
    if ((do_keep || index_pack_args) && (args->lock_pack || unpack_limit))
973
0
      add_index_pack_keep_option(&cmd.args);
974
0
    if (!index_pack_args && args->check_self_contained_and_connected)
975
0
      strvec_push(&cmd.args, "--check-self-contained-and-connected");
976
0
    else
977
      /*
978
       * We cannot perform any connectivity checks because
979
       * not all packs have been downloaded; let the caller
980
       * have this responsibility.
981
       */
982
0
      args->check_self_contained_and_connected = 0;
983
984
0
    if (args->from_promisor)
985
      /*
986
       * create_promisor_file() may be called afterwards but
987
       * we still need index-pack to know that this is a
988
       * promisor pack. For example, if transfer.fsckobjects
989
       * is true, index-pack needs to know that .gitmodules
990
       * is a promisor object (so that it won't complain if
991
       * it is missing).
992
       */
993
0
      strvec_push(&cmd.args, "--promisor");
994
0
  }
995
0
  else {
996
0
    cmd_name = "unpack-objects";
997
0
    strvec_push(&cmd.args, cmd_name);
998
0
    if (args->quiet || args->no_progress)
999
0
      strvec_push(&cmd.args, "-q");
1000
0
    args->check_self_contained_and_connected = 0;
1001
0
  }
1002
1003
0
  if (pass_header)
1004
0
    strvec_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
1005
0
           ntohl(header.hdr_version),
1006
0
         ntohl(header.hdr_entries));
1007
0
  if (fsck_objects) {
1008
0
    if (args->from_promisor || index_pack_args)
1009
      /*
1010
       * We cannot use --strict in index-pack because it
1011
       * checks both broken objects and links, but we only
1012
       * want to check for broken objects.
1013
       */
1014
0
      strvec_push(&cmd.args, "--fsck-objects");
1015
0
    else
1016
0
      strvec_pushf(&cmd.args, "--strict%s",
1017
0
             fsck_msg_types.buf);
1018
0
  }
1019
1020
0
  if (index_pack_args) {
1021
0
    int i;
1022
1023
0
    for (i = 0; i < cmd.args.nr; i++)
1024
0
      strvec_push(index_pack_args, cmd.args.v[i]);
1025
0
  }
1026
1027
0
  sigchain_push(SIGPIPE, SIG_IGN);
1028
1029
0
  cmd.in = demux.out;
1030
0
  cmd.git_cmd = 1;
1031
0
  if (start_command(&cmd))
1032
0
    die(_("fetch-pack: unable to fork off %s"), cmd_name);
1033
0
  if (do_keep && (pack_lockfiles || fsck_objects)) {
1034
0
    int is_well_formed;
1035
0
    char *pack_lockfile = index_pack_lockfile(the_repository,
1036
0
                cmd.out,
1037
0
                &is_well_formed);
1038
1039
0
    if (!is_well_formed)
1040
0
      die(_("fetch-pack: invalid index-pack output"));
1041
0
    if (pack_lockfiles && pack_lockfile)
1042
0
      string_list_append_nodup(pack_lockfiles, pack_lockfile);
1043
0
    else
1044
0
      free(pack_lockfile);
1045
0
    parse_gitmodules_oids(cmd.out, gitmodules_oids);
1046
0
    close(cmd.out);
1047
0
  }
1048
1049
0
  if (!use_sideband)
1050
    /* Closed by start_command() */
1051
0
    xd[0] = -1;
1052
1053
0
  ret = finish_command(&cmd);
1054
0
  if (!ret || (args->check_self_contained_and_connected && ret == 1))
1055
0
    args->self_contained_and_connected =
1056
0
      args->check_self_contained_and_connected &&
1057
0
      ret == 0;
1058
0
  else
1059
0
    die(_("%s failed"), cmd_name);
1060
0
  if (use_sideband && finish_async(&demux))
1061
0
    die(_("error in sideband demultiplexer"));
1062
1063
0
  sigchain_pop(SIGPIPE);
1064
1065
  /*
1066
   * Now that index-pack has succeeded, write the promisor file using the
1067
   * obtained .keep filename if necessary
1068
   */
1069
0
  if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
1070
0
    create_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
1071
1072
0
  return 0;
1073
0
}
1074
1075
static int ref_compare_name(const struct ref *a, const struct ref *b)
1076
0
{
1077
0
  return strcmp(a->name, b->name);
1078
0
}
1079
1080
0
DEFINE_LIST_SORT(static, sort_ref_list, struct ref, next);
1081
0
1082
0
static int cmp_ref_by_name(const void *a_, const void *b_)
1083
0
{
1084
0
  const struct ref *a = *((const struct ref **)a_);
1085
0
  const struct ref *b = *((const struct ref **)b_);
1086
0
  return strcmp(a->name, b->name);
1087
0
}
1088
1089
static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1090
         int fd[2],
1091
         const struct ref *orig_ref,
1092
         struct ref **sought, int nr_sought,
1093
         struct shallow_info *si,
1094
         struct string_list *pack_lockfiles)
1095
0
{
1096
0
  struct repository *r = the_repository;
1097
0
  struct ref *ref = copy_ref_list(orig_ref);
1098
0
  struct object_id oid;
1099
0
  const char *agent_feature;
1100
0
  size_t agent_len;
1101
0
  struct fetch_negotiator negotiator_alloc;
1102
0
  struct fetch_negotiator *negotiator;
1103
1104
0
  negotiator = &negotiator_alloc;
1105
0
  if (args->refetch) {
1106
0
    fetch_negotiator_init_noop(negotiator);
1107
0
  } else {
1108
0
    fetch_negotiator_init(r, negotiator);
1109
0
  }
1110
1111
0
  sort_ref_list(&ref, ref_compare_name);
1112
0
  QSORT(sought, nr_sought, cmp_ref_by_name);
1113
1114
0
  if ((agent_feature = server_feature_value("agent", &agent_len))) {
1115
0
    agent_supported = 1;
1116
0
    if (agent_len)
1117
0
      print_verbose(args, _("Server version is %.*s"),
1118
0
              (int)agent_len, agent_feature);
1119
0
  }
1120
1121
0
  if (!server_supports("session-id"))
1122
0
    advertise_sid = 0;
1123
1124
0
  if (server_supports("shallow"))
1125
0
    print_verbose(args, _("Server supports %s"), "shallow");
1126
0
  else if (args->depth > 0 || is_repository_shallow(r))
1127
0
    die(_("Server does not support shallow clients"));
1128
0
  if (args->depth > 0 || args->deepen_since || args->deepen_not)
1129
0
    args->deepen = 1;
1130
0
  if (server_supports("multi_ack_detailed")) {
1131
0
    print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
1132
0
    multi_ack = 2;
1133
0
    if (server_supports("no-done")) {
1134
0
      print_verbose(args, _("Server supports %s"), "no-done");
1135
0
      if (args->stateless_rpc)
1136
0
        no_done = 1;
1137
0
    }
1138
0
  }
1139
0
  else if (server_supports("multi_ack")) {
1140
0
    print_verbose(args, _("Server supports %s"), "multi_ack");
1141
0
    multi_ack = 1;
1142
0
  }
1143
0
  if (server_supports("side-band-64k")) {
1144
0
    print_verbose(args, _("Server supports %s"), "side-band-64k");
1145
0
    use_sideband = 2;
1146
0
  }
1147
0
  else if (server_supports("side-band")) {
1148
0
    print_verbose(args, _("Server supports %s"), "side-band");
1149
0
    use_sideband = 1;
1150
0
  }
1151
0
  if (server_supports("allow-tip-sha1-in-want")) {
1152
0
    print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
1153
0
    allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1154
0
  }
1155
0
  if (server_supports("allow-reachable-sha1-in-want")) {
1156
0
    print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
1157
0
    allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1158
0
  }
1159
0
  if (server_supports("thin-pack"))
1160
0
    print_verbose(args, _("Server supports %s"), "thin-pack");
1161
0
  else
1162
0
    args->use_thin_pack = 0;
1163
0
  if (server_supports("no-progress"))
1164
0
    print_verbose(args, _("Server supports %s"), "no-progress");
1165
0
  else
1166
0
    args->no_progress = 0;
1167
0
  if (server_supports("include-tag"))
1168
0
    print_verbose(args, _("Server supports %s"), "include-tag");
1169
0
  else
1170
0
    args->include_tag = 0;
1171
0
  if (server_supports("ofs-delta"))
1172
0
    print_verbose(args, _("Server supports %s"), "ofs-delta");
1173
0
  else
1174
0
    prefer_ofs_delta = 0;
1175
1176
0
  if (server_supports("filter")) {
1177
0
    server_supports_filtering = 1;
1178
0
    print_verbose(args, _("Server supports %s"), "filter");
1179
0
  } else if (args->filter_options.choice) {
1180
0
    warning("filtering not recognized by server, ignoring");
1181
0
  }
1182
1183
0
  if (server_supports("deepen-since")) {
1184
0
    print_verbose(args, _("Server supports %s"), "deepen-since");
1185
0
    deepen_since_ok = 1;
1186
0
  } else if (args->deepen_since)
1187
0
    die(_("Server does not support --shallow-since"));
1188
0
  if (server_supports("deepen-not")) {
1189
0
    print_verbose(args, _("Server supports %s"), "deepen-not");
1190
0
    deepen_not_ok = 1;
1191
0
  } else if (args->deepen_not)
1192
0
    die(_("Server does not support --shallow-exclude"));
1193
0
  if (server_supports("deepen-relative"))
1194
0
    print_verbose(args, _("Server supports %s"), "deepen-relative");
1195
0
  else if (args->deepen_relative)
1196
0
    die(_("Server does not support --deepen"));
1197
0
  if (!server_supports_hash(the_hash_algo->name, NULL))
1198
0
    die(_("Server does not support this repository's object format"));
1199
1200
0
  mark_complete_and_common_ref(negotiator, args, &ref);
1201
0
  filter_refs(args, &ref, sought, nr_sought);
1202
0
  if (!args->refetch && everything_local(args, &ref)) {
1203
0
    packet_flush(fd[1]);
1204
0
    goto all_done;
1205
0
  }
1206
0
  if (find_common(negotiator, args, fd, &oid, ref) < 0)
1207
0
    if (!args->keep_pack)
1208
      /* When cloning, it is not unusual to have
1209
       * no common commit.
1210
       */
1211
0
      warning(_("no common commits"));
1212
1213
0
  if (args->stateless_rpc)
1214
0
    packet_flush(fd[1]);
1215
0
  if (args->deepen)
1216
0
    setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1217
0
          NULL);
1218
0
  else if (si->nr_ours || si->nr_theirs) {
1219
0
    if (args->reject_shallow_remote)
1220
0
      die(_("source repository is shallow, reject to clone."));
1221
0
    alternate_shallow_file = setup_temporary_shallow(si->shallow);
1222
0
  } else
1223
0
    alternate_shallow_file = NULL;
1224
0
  if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
1225
0
         &fsck_options.gitmodules_found))
1226
0
    die(_("git fetch-pack: fetch failed."));
1227
0
  if (fsck_finish(&fsck_options))
1228
0
    die("fsck failed");
1229
1230
0
 all_done:
1231
0
  if (negotiator)
1232
0
    negotiator->release(negotiator);
1233
0
  return ref;
1234
0
}
1235
1236
static void add_shallow_requests(struct strbuf *req_buf,
1237
         const struct fetch_pack_args *args)
1238
0
{
1239
0
  if (is_repository_shallow(the_repository))
1240
0
    write_shallow_commits(req_buf, 1, NULL);
1241
0
  if (args->depth > 0)
1242
0
    packet_buf_write(req_buf, "deepen %d", args->depth);
1243
0
  if (args->deepen_since) {
1244
0
    timestamp_t max_age = approxidate(args->deepen_since);
1245
0
    packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1246
0
  }
1247
0
  if (args->deepen_not) {
1248
0
    int i;
1249
0
    for (i = 0; i < args->deepen_not->nr; i++) {
1250
0
      struct string_list_item *s = args->deepen_not->items + i;
1251
0
      packet_buf_write(req_buf, "deepen-not %s", s->string);
1252
0
    }
1253
0
  }
1254
0
  if (args->deepen_relative)
1255
0
    packet_buf_write(req_buf, "deepen-relative\n");
1256
0
}
1257
1258
static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1259
0
{
1260
0
  int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1261
1262
0
  for ( ; wants ; wants = wants->next) {
1263
0
    const struct object_id *remote = &wants->old_oid;
1264
0
    struct object *o;
1265
1266
    /*
1267
     * If that object is complete (i.e. it is an ancestor of a
1268
     * local ref), we tell them we have it but do not have to
1269
     * tell them about its ancestors, which they already know
1270
     * about.
1271
     *
1272
     * We use lookup_object here because we are only
1273
     * interested in the case we *know* the object is
1274
     * reachable and we have already scanned it.
1275
     */
1276
0
    if (((o = lookup_object(the_repository, remote)) != NULL) &&
1277
0
        (o->flags & COMPLETE)) {
1278
0
      continue;
1279
0
    }
1280
1281
0
    if (!use_ref_in_want || wants->exact_oid)
1282
0
      packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1283
0
    else
1284
0
      packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1285
0
  }
1286
0
}
1287
1288
static void add_common(struct strbuf *req_buf, struct oidset *common)
1289
0
{
1290
0
  struct oidset_iter iter;
1291
0
  const struct object_id *oid;
1292
0
  oidset_iter_init(common, &iter);
1293
1294
0
  while ((oid = oidset_iter_next(&iter))) {
1295
0
    packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1296
0
  }
1297
0
}
1298
1299
static int add_haves(struct fetch_negotiator *negotiator,
1300
         struct strbuf *req_buf,
1301
         int *haves_to_send)
1302
0
{
1303
0
  int haves_added = 0;
1304
0
  const struct object_id *oid;
1305
1306
0
  while ((oid = negotiator->next(negotiator))) {
1307
0
    packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1308
0
    if (++haves_added >= *haves_to_send)
1309
0
      break;
1310
0
  }
1311
1312
  /* Increase haves to send on next round */
1313
0
  *haves_to_send = next_flush(1, *haves_to_send);
1314
1315
0
  return haves_added;
1316
0
}
1317
1318
static void write_fetch_command_and_capabilities(struct strbuf *req_buf,
1319
             const struct string_list *server_options)
1320
0
{
1321
0
  const char *hash_name;
1322
1323
0
  ensure_server_supports_v2("fetch");
1324
0
  packet_buf_write(req_buf, "command=fetch");
1325
0
  if (server_supports_v2("agent"))
1326
0
    packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized());
1327
0
  if (advertise_sid && server_supports_v2("session-id"))
1328
0
    packet_buf_write(req_buf, "session-id=%s", trace2_session_id());
1329
0
  if (server_options && server_options->nr) {
1330
0
    int i;
1331
0
    ensure_server_supports_v2("server-option");
1332
0
    for (i = 0; i < server_options->nr; i++)
1333
0
      packet_buf_write(req_buf, "server-option=%s",
1334
0
           server_options->items[i].string);
1335
0
  }
1336
1337
0
  if (server_feature_v2("object-format", &hash_name)) {
1338
0
    int hash_algo = hash_algo_by_name(hash_name);
1339
0
    if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
1340
0
      die(_("mismatched algorithms: client %s; server %s"),
1341
0
          the_hash_algo->name, hash_name);
1342
0
    packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name);
1343
0
  } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1_LEGACY) {
1344
0
    die(_("the server does not support algorithm '%s'"),
1345
0
        the_hash_algo->name);
1346
0
  }
1347
0
  packet_buf_delim(req_buf);
1348
0
}
1349
1350
static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1351
            struct fetch_pack_args *args,
1352
            const struct ref *wants, struct oidset *common,
1353
            int *haves_to_send, int *in_vain,
1354
            int sideband_all, int seen_ack)
1355
0
{
1356
0
  int haves_added;
1357
0
  int done_sent = 0;
1358
0
  struct strbuf req_buf = STRBUF_INIT;
1359
1360
0
  write_fetch_command_and_capabilities(&req_buf, args->server_options);
1361
1362
0
  if (args->use_thin_pack)
1363
0
    packet_buf_write(&req_buf, "thin-pack");
1364
0
  if (args->no_progress)
1365
0
    packet_buf_write(&req_buf, "no-progress");
1366
0
  if (args->include_tag)
1367
0
    packet_buf_write(&req_buf, "include-tag");
1368
0
  if (prefer_ofs_delta)
1369
0
    packet_buf_write(&req_buf, "ofs-delta");
1370
0
  if (sideband_all)
1371
0
    packet_buf_write(&req_buf, "sideband-all");
1372
1373
  /* Add shallow-info and deepen request */
1374
0
  if (server_supports_feature("fetch", "shallow", 0))
1375
0
    add_shallow_requests(&req_buf, args);
1376
0
  else if (is_repository_shallow(the_repository) || args->deepen)
1377
0
    die(_("Server does not support shallow requests"));
1378
1379
  /* Add filter */
1380
0
  send_filter(args, &req_buf,
1381
0
        server_supports_feature("fetch", "filter", 0));
1382
1383
0
  if (server_supports_feature("fetch", "packfile-uris", 0)) {
1384
0
    int i;
1385
0
    struct strbuf to_send = STRBUF_INIT;
1386
1387
0
    for (i = 0; i < uri_protocols.nr; i++) {
1388
0
      const char *s = uri_protocols.items[i].string;
1389
1390
0
      if (!strcmp(s, "https") || !strcmp(s, "http")) {
1391
0
        if (to_send.len)
1392
0
          strbuf_addch(&to_send, ',');
1393
0
        strbuf_addstr(&to_send, s);
1394
0
      }
1395
0
    }
1396
0
    if (to_send.len) {
1397
0
      packet_buf_write(&req_buf, "packfile-uris %s",
1398
0
           to_send.buf);
1399
0
      strbuf_release(&to_send);
1400
0
    }
1401
0
  }
1402
1403
  /* add wants */
1404
0
  add_wants(wants, &req_buf);
1405
1406
  /* Add all of the common commits we've found in previous rounds */
1407
0
  add_common(&req_buf, common);
1408
1409
0
  haves_added = add_haves(negotiator, &req_buf, haves_to_send);
1410
0
  *in_vain += haves_added;
1411
0
  trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added);
1412
0
  trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain);
1413
0
  if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
1414
    /* Send Done */
1415
0
    packet_buf_write(&req_buf, "done\n");
1416
0
    done_sent = 1;
1417
0
  }
1418
1419
  /* Send request */
1420
0
  packet_buf_flush(&req_buf);
1421
0
  if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1422
0
    die_errno(_("unable to write request to remote"));
1423
1424
0
  strbuf_release(&req_buf);
1425
0
  return done_sent;
1426
0
}
1427
1428
/*
1429
 * Processes a section header in a server's response and checks if it matches
1430
 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1431
 * not consumed); if 0, the line will be consumed and the function will die if
1432
 * the section header doesn't match what was expected.
1433
 */
1434
static int process_section_header(struct packet_reader *reader,
1435
          const char *section, int peek)
1436
0
{
1437
0
  int ret = 0;
1438
1439
0
  if (packet_reader_peek(reader) == PACKET_READ_NORMAL &&
1440
0
      !strcmp(reader->line, section))
1441
0
    ret = 1;
1442
1443
0
  if (!peek) {
1444
0
    if (!ret) {
1445
0
      if (reader->line)
1446
0
        die(_("expected '%s', received '%s'"),
1447
0
            section, reader->line);
1448
0
      else
1449
0
        die(_("expected '%s'"), section);
1450
0
    }
1451
0
    packet_reader_read(reader);
1452
0
  }
1453
1454
0
  return ret;
1455
0
}
1456
1457
static int process_ack(struct fetch_negotiator *negotiator,
1458
           struct packet_reader *reader,
1459
           struct object_id *common_oid,
1460
           int *received_ready)
1461
0
{
1462
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1463
0
    const char *arg;
1464
1465
0
    if (!strcmp(reader->line, "NAK"))
1466
0
      continue;
1467
1468
0
    if (skip_prefix(reader->line, "ACK ", &arg)) {
1469
0
      if (!get_oid_hex(arg, common_oid)) {
1470
0
        struct commit *commit;
1471
0
        commit = lookup_commit(the_repository, common_oid);
1472
0
        if (negotiator)
1473
0
          negotiator->ack(negotiator, commit);
1474
0
      }
1475
0
      return 1;
1476
0
    }
1477
1478
0
    if (!strcmp(reader->line, "ready")) {
1479
0
      *received_ready = 1;
1480
0
      continue;
1481
0
    }
1482
1483
0
    die(_("unexpected acknowledgment line: '%s'"), reader->line);
1484
0
  }
1485
1486
0
  if (reader->status != PACKET_READ_FLUSH &&
1487
0
      reader->status != PACKET_READ_DELIM)
1488
0
    die(_("error processing acks: %d"), reader->status);
1489
1490
  /*
1491
   * If an "acknowledgments" section is sent, a packfile is sent if and
1492
   * only if "ready" was sent in this section. The other sections
1493
   * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1494
   * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1495
   * otherwise.
1496
   */
1497
0
  if (*received_ready && reader->status != PACKET_READ_DELIM)
1498
    /*
1499
     * TRANSLATORS: The parameter will be 'ready', a protocol
1500
     * keyword.
1501
     */
1502
0
    die(_("expected packfile to be sent after '%s'"), "ready");
1503
0
  if (!*received_ready && reader->status != PACKET_READ_FLUSH)
1504
    /*
1505
     * TRANSLATORS: The parameter will be 'ready', a protocol
1506
     * keyword.
1507
     */
1508
0
    die(_("expected no other sections to be sent after no '%s'"), "ready");
1509
1510
0
  return 0;
1511
0
}
1512
1513
static void receive_shallow_info(struct fetch_pack_args *args,
1514
         struct packet_reader *reader,
1515
         struct oid_array *shallows,
1516
         struct shallow_info *si)
1517
0
{
1518
0
  int unshallow_received = 0;
1519
1520
0
  process_section_header(reader, "shallow-info", 0);
1521
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1522
0
    const char *arg;
1523
0
    struct object_id oid;
1524
1525
0
    if (skip_prefix(reader->line, "shallow ", &arg)) {
1526
0
      if (get_oid_hex(arg, &oid))
1527
0
        die(_("invalid shallow line: %s"), reader->line);
1528
0
      oid_array_append(shallows, &oid);
1529
0
      continue;
1530
0
    }
1531
0
    if (skip_prefix(reader->line, "unshallow ", &arg)) {
1532
0
      if (get_oid_hex(arg, &oid))
1533
0
        die(_("invalid unshallow line: %s"), reader->line);
1534
0
      if (!lookup_object(the_repository, &oid))
1535
0
        die(_("object not found: %s"), reader->line);
1536
      /* make sure that it is parsed as shallow */
1537
0
      if (!parse_object(the_repository, &oid))
1538
0
        die(_("error in object: %s"), reader->line);
1539
0
      if (unregister_shallow(&oid))
1540
0
        die(_("no shallow found: %s"), reader->line);
1541
0
      unshallow_received = 1;
1542
0
      continue;
1543
0
    }
1544
0
    die(_("expected shallow/unshallow, got %s"), reader->line);
1545
0
  }
1546
1547
0
  if (reader->status != PACKET_READ_FLUSH &&
1548
0
      reader->status != PACKET_READ_DELIM)
1549
0
    die(_("error processing shallow info: %d"), reader->status);
1550
1551
0
  if (args->deepen || unshallow_received) {
1552
    /*
1553
     * Treat these as shallow lines caused by our depth settings.
1554
     * In v0, these lines cannot cause refs to be rejected; do the
1555
     * same.
1556
     */
1557
0
    int i;
1558
1559
0
    for (i = 0; i < shallows->nr; i++)
1560
0
      register_shallow(the_repository, &shallows->oid[i]);
1561
0
    setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1562
0
          NULL);
1563
0
    args->deepen = 1;
1564
0
  } else if (shallows->nr) {
1565
    /*
1566
     * Treat these as shallow lines caused by the remote being
1567
     * shallow. In v0, remote refs that reach these objects are
1568
     * rejected (unless --update-shallow is set); do the same.
1569
     */
1570
0
    prepare_shallow_info(si, shallows);
1571
0
    if (si->nr_ours || si->nr_theirs) {
1572
0
      if (args->reject_shallow_remote)
1573
0
        die(_("source repository is shallow, reject to clone."));
1574
0
      alternate_shallow_file =
1575
0
        setup_temporary_shallow(si->shallow);
1576
0
    } else
1577
0
      alternate_shallow_file = NULL;
1578
0
  } else {
1579
0
    alternate_shallow_file = NULL;
1580
0
  }
1581
0
}
1582
1583
static int cmp_name_ref(const void *name, const void *ref)
1584
0
{
1585
0
  return strcmp(name, (*(struct ref **)ref)->name);
1586
0
}
1587
1588
static void receive_wanted_refs(struct packet_reader *reader,
1589
        struct ref **sought, int nr_sought)
1590
0
{
1591
0
  process_section_header(reader, "wanted-refs", 0);
1592
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1593
0
    struct object_id oid;
1594
0
    const char *end;
1595
0
    struct ref **found;
1596
1597
0
    if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1598
0
      die(_("expected wanted-ref, got '%s'"), reader->line);
1599
1600
0
    found = bsearch(end, sought, nr_sought, sizeof(*sought),
1601
0
        cmp_name_ref);
1602
0
    if (!found)
1603
0
      die(_("unexpected wanted-ref: '%s'"), reader->line);
1604
0
    oidcpy(&(*found)->old_oid, &oid);
1605
0
  }
1606
1607
0
  if (reader->status != PACKET_READ_DELIM)
1608
0
    die(_("error processing wanted refs: %d"), reader->status);
1609
0
}
1610
1611
static void receive_packfile_uris(struct packet_reader *reader,
1612
          struct string_list *uris)
1613
0
{
1614
0
  process_section_header(reader, "packfile-uris", 0);
1615
0
  while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1616
0
    if (reader->pktlen < the_hash_algo->hexsz ||
1617
0
        reader->line[the_hash_algo->hexsz] != ' ')
1618
0
      die("expected '<hash> <uri>', got: %s", reader->line);
1619
1620
0
    string_list_append(uris, reader->line);
1621
0
  }
1622
0
  if (reader->status != PACKET_READ_DELIM)
1623
0
    die("expected DELIM");
1624
0
}
1625
1626
enum fetch_state {
1627
  FETCH_CHECK_LOCAL = 0,
1628
  FETCH_SEND_REQUEST,
1629
  FETCH_PROCESS_ACKS,
1630
  FETCH_GET_PACK,
1631
  FETCH_DONE,
1632
};
1633
1634
static void do_check_stateless_delimiter(int stateless_rpc,
1635
           struct packet_reader *reader)
1636
0
{
1637
0
  check_stateless_delimiter(stateless_rpc, reader,
1638
0
          _("git fetch-pack: expected response end packet"));
1639
0
}
1640
1641
static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1642
            int fd[2],
1643
            const struct ref *orig_ref,
1644
            struct ref **sought, int nr_sought,
1645
            struct oid_array *shallows,
1646
            struct shallow_info *si,
1647
            struct string_list *pack_lockfiles)
1648
0
{
1649
0
  struct repository *r = the_repository;
1650
0
  struct ref *ref = copy_ref_list(orig_ref);
1651
0
  enum fetch_state state = FETCH_CHECK_LOCAL;
1652
0
  struct oidset common = OIDSET_INIT;
1653
0
  struct packet_reader reader;
1654
0
  int in_vain = 0, negotiation_started = 0;
1655
0
  int negotiation_round = 0;
1656
0
  int haves_to_send = INITIAL_FLUSH;
1657
0
  struct fetch_negotiator negotiator_alloc;
1658
0
  struct fetch_negotiator *negotiator;
1659
0
  int seen_ack = 0;
1660
0
  struct object_id common_oid;
1661
0
  int received_ready = 0;
1662
0
  struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1663
0
  int i;
1664
0
  struct strvec index_pack_args = STRVEC_INIT;
1665
0
  const char *promisor_remote_config;
1666
1667
0
  if (server_feature_v2("promisor-remote", &promisor_remote_config))
1668
0
    promisor_remote_reply(promisor_remote_config, NULL);
1669
1670
0
  if (args->filter_options.choice == LOFC_AUTO) {
1671
0
    struct strbuf errbuf = STRBUF_INIT;
1672
0
    char *constructed_filter = promisor_remote_construct_filter(r);
1673
1674
0
    list_objects_filter_release(&args->filter_options);
1675
    /* Disallow 'auto' as a result of the resolution of this 'auto' filter below */
1676
0
    args->filter_options.allow_auto_filter = 0;
1677
1678
0
    if (constructed_filter &&
1679
0
        gently_parse_list_objects_filter(&args->filter_options,
1680
0
                 constructed_filter,
1681
0
                 &errbuf))
1682
0
      die(_("couldn't resolve 'auto' filter '%s': %s"),
1683
0
          constructed_filter, errbuf.buf);
1684
1685
0
    free(constructed_filter);
1686
0
    strbuf_release(&errbuf);
1687
0
  }
1688
1689
0
  negotiator = &negotiator_alloc;
1690
0
  if (args->refetch)
1691
0
    fetch_negotiator_init_noop(negotiator);
1692
0
  else
1693
0
    fetch_negotiator_init(r, negotiator);
1694
1695
0
  packet_reader_init(&reader, fd[0], NULL, 0,
1696
0
         PACKET_READ_CHOMP_NEWLINE |
1697
0
         PACKET_READ_DIE_ON_ERR_PACKET);
1698
0
  if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1699
0
      server_supports_feature("fetch", "sideband-all", 0)) {
1700
0
    reader.use_sideband = 1;
1701
0
    reader.me = "fetch-pack";
1702
0
  }
1703
1704
0
  while (state != FETCH_DONE) {
1705
0
    switch (state) {
1706
0
    case FETCH_CHECK_LOCAL:
1707
0
      sort_ref_list(&ref, ref_compare_name);
1708
0
      QSORT(sought, nr_sought, cmp_ref_by_name);
1709
1710
      /* v2 supports these by default */
1711
0
      allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1712
0
      use_sideband = 2;
1713
0
      if (args->depth > 0 || args->deepen_since || args->deepen_not)
1714
0
        args->deepen = 1;
1715
1716
      /* Filter 'ref' by 'sought' and those that aren't local */
1717
0
      mark_complete_and_common_ref(negotiator, args, &ref);
1718
0
      filter_refs(args, &ref, sought, nr_sought);
1719
0
      if (!args->refetch && everything_local(args, &ref))
1720
0
        state = FETCH_DONE;
1721
0
      else
1722
0
        state = FETCH_SEND_REQUEST;
1723
1724
0
      mark_tips(negotiator, args->negotiation_tips);
1725
0
      for_each_cached_alternate(negotiator,
1726
0
              insert_one_alternate_object);
1727
0
      break;
1728
0
    case FETCH_SEND_REQUEST:
1729
0
      if (!negotiation_started) {
1730
0
        negotiation_started = 1;
1731
0
        trace2_region_enter("fetch-pack",
1732
0
                "negotiation_v2",
1733
0
                the_repository);
1734
0
      }
1735
0
      negotiation_round++;
1736
0
      trace2_region_enter_printf("negotiation_v2", "round",
1737
0
               the_repository, "%d",
1738
0
               negotiation_round);
1739
0
      if (send_fetch_request(negotiator, fd[1], args, ref,
1740
0
                 &common,
1741
0
                 &haves_to_send, &in_vain,
1742
0
                 reader.use_sideband,
1743
0
                 seen_ack)) {
1744
0
        trace2_region_leave_printf("negotiation_v2", "round",
1745
0
                 the_repository, "%d",
1746
0
                 negotiation_round);
1747
0
        state = FETCH_GET_PACK;
1748
0
      }
1749
0
      else
1750
0
        state = FETCH_PROCESS_ACKS;
1751
0
      break;
1752
0
    case FETCH_PROCESS_ACKS:
1753
      /* Process ACKs/NAKs */
1754
0
      process_section_header(&reader, "acknowledgments", 0);
1755
0
      while (process_ack(negotiator, &reader, &common_oid,
1756
0
             &received_ready)) {
1757
0
        in_vain = 0;
1758
0
        seen_ack = 1;
1759
0
        oidset_insert(&common, &common_oid);
1760
0
      }
1761
0
      trace2_region_leave_printf("negotiation_v2", "round",
1762
0
               the_repository, "%d",
1763
0
               negotiation_round);
1764
0
      if (received_ready) {
1765
        /*
1766
         * Don't check for response delimiter; get_pack() will
1767
         * read the rest of this response.
1768
         */
1769
0
        state = FETCH_GET_PACK;
1770
0
      } else {
1771
0
        do_check_stateless_delimiter(args->stateless_rpc, &reader);
1772
0
        state = FETCH_SEND_REQUEST;
1773
0
      }
1774
0
      break;
1775
0
    case FETCH_GET_PACK:
1776
0
      trace2_region_leave("fetch-pack",
1777
0
              "negotiation_v2",
1778
0
              the_repository);
1779
0
      trace2_data_intmax("negotiation_v2", the_repository,
1780
0
             "total_rounds", negotiation_round);
1781
      /* Check for shallow-info section */
1782
0
      if (process_section_header(&reader, "shallow-info", 1))
1783
0
        receive_shallow_info(args, &reader, shallows, si);
1784
1785
0
      if (process_section_header(&reader, "wanted-refs", 1))
1786
0
        receive_wanted_refs(&reader, sought, nr_sought);
1787
1788
      /* get the pack(s) */
1789
0
      if (git_env_bool("GIT_TRACE_REDACT", 1))
1790
0
        reader.options |= PACKET_READ_REDACT_URI_PATH;
1791
0
      if (process_section_header(&reader, "packfile-uris", 1))
1792
0
        receive_packfile_uris(&reader, &packfile_uris);
1793
      /* We don't expect more URIs. Reset to avoid expensive URI check. */
1794
0
      reader.options &= ~PACKET_READ_REDACT_URI_PATH;
1795
1796
0
      process_section_header(&reader, "packfile", 0);
1797
1798
      /*
1799
       * this is the final request we'll make of the server;
1800
       * do a half-duplex shutdown to indicate that they can
1801
       * hang up as soon as the pack is sent.
1802
       */
1803
0
      close(fd[1]);
1804
0
      fd[1] = -1;
1805
1806
0
      if (get_pack(args, fd, pack_lockfiles,
1807
0
             packfile_uris.nr ? &index_pack_args : NULL,
1808
0
             sought, nr_sought, &fsck_options.gitmodules_found))
1809
0
        die(_("git fetch-pack: fetch failed."));
1810
0
      do_check_stateless_delimiter(args->stateless_rpc, &reader);
1811
1812
0
      state = FETCH_DONE;
1813
0
      break;
1814
0
    case FETCH_DONE:
1815
0
      continue;
1816
0
    }
1817
0
  }
1818
1819
0
  for (i = 0; i < packfile_uris.nr; i++) {
1820
0
    int j;
1821
0
    struct child_process cmd = CHILD_PROCESS_INIT;
1822
0
    char packname[GIT_MAX_HEXSZ + 1];
1823
0
    const char *uri = packfile_uris.items[i].string +
1824
0
      the_hash_algo->hexsz + 1;
1825
1826
0
    strvec_push(&cmd.args, "http-fetch");
1827
0
    strvec_pushf(&cmd.args, "--packfile=%.*s",
1828
0
           (int) the_hash_algo->hexsz,
1829
0
           packfile_uris.items[i].string);
1830
0
    for (j = 0; j < index_pack_args.nr; j++)
1831
0
      strvec_pushf(&cmd.args, "--index-pack-arg=%s",
1832
0
             index_pack_args.v[j]);
1833
0
    strvec_push(&cmd.args, uri);
1834
0
    cmd.git_cmd = 1;
1835
0
    cmd.no_stdin = 1;
1836
0
    cmd.out = -1;
1837
0
    if (start_command(&cmd))
1838
0
      die("fetch-pack: unable to spawn http-fetch");
1839
1840
0
    if (read_in_full(cmd.out, packname, 5) < 0 ||
1841
0
        memcmp(packname, "keep\t", 5))
1842
0
      die("fetch-pack: expected keep then TAB at start of http-fetch output");
1843
1844
0
    if (read_in_full(cmd.out, packname,
1845
0
         the_hash_algo->hexsz + 1) < 0 ||
1846
0
        packname[the_hash_algo->hexsz] != '\n')
1847
0
      die("fetch-pack: expected hash then LF at end of http-fetch output");
1848
1849
0
    packname[the_hash_algo->hexsz] = '\0';
1850
1851
0
    parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
1852
1853
0
    close(cmd.out);
1854
1855
0
    if (finish_command(&cmd))
1856
0
      die("fetch-pack: unable to finish http-fetch");
1857
1858
0
    if (memcmp(packfile_uris.items[i].string, packname,
1859
0
         the_hash_algo->hexsz))
1860
0
      die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1861
0
          uri, (int) the_hash_algo->hexsz,
1862
0
          packfile_uris.items[i].string);
1863
1864
0
    string_list_append_nodup(pack_lockfiles,
1865
0
           xstrfmt("%s/pack/pack-%s.keep",
1866
0
             repo_get_object_directory(the_repository),
1867
0
             packname));
1868
0
  }
1869
0
  string_list_clear(&packfile_uris, 0);
1870
0
  strvec_clear(&index_pack_args);
1871
1872
0
  if (fsck_finish(&fsck_options))
1873
0
    die("fsck failed");
1874
1875
0
  if (negotiator)
1876
0
    negotiator->release(negotiator);
1877
1878
0
  oidset_clear(&common);
1879
0
  return ref;
1880
0
}
1881
1882
int fetch_pack_fsck_config(const char *var, const char *value,
1883
         struct strbuf *msg_types)
1884
0
{
1885
0
  const char *msg_id;
1886
1887
0
  if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1888
0
    char *path ;
1889
1890
0
    if (git_config_pathname(&path, var, value))
1891
0
      return -1;
1892
0
    if (path)
1893
0
      strbuf_addf(msg_types, "%cskiplist=%s",
1894
0
            msg_types->len ? ',' : '=', path);
1895
0
    free(path);
1896
0
    return 0;
1897
0
  }
1898
1899
0
  if (skip_prefix(var, "fetch.fsck.", &msg_id)) {
1900
0
    if (!value)
1901
0
      return config_error_nonbool(var);
1902
0
    if (is_valid_msg_type(msg_id, value))
1903
0
      strbuf_addf(msg_types, "%c%s=%s",
1904
0
        msg_types->len ? ',' : '=', msg_id, value);
1905
0
    else
1906
0
      warning("Skipping unknown msg id '%s'", msg_id);
1907
0
    return 0;
1908
0
  }
1909
1910
0
  return 1;
1911
0
}
1912
1913
static int fetch_pack_config_cb(const char *var, const char *value,
1914
        const struct config_context *ctx, void *cb)
1915
0
{
1916
0
  int ret = fetch_pack_fsck_config(var, value, &fsck_msg_types);
1917
0
  if (ret > 0)
1918
0
    return git_default_config(var, value, ctx, cb);
1919
1920
0
  return ret;
1921
0
}
1922
1923
static void fetch_pack_config(void)
1924
0
{
1925
0
  repo_config_get_int(the_repository, "fetch.unpacklimit", &fetch_unpack_limit);
1926
0
  repo_config_get_int(the_repository, "transfer.unpacklimit", &transfer_unpack_limit);
1927
0
  repo_config_get_bool(the_repository, "repack.usedeltabaseoffset", &prefer_ofs_delta);
1928
0
  repo_config_get_bool(the_repository, "fetch.fsckobjects", &fetch_fsck_objects);
1929
0
  repo_config_get_bool(the_repository, "transfer.fsckobjects", &transfer_fsck_objects);
1930
0
  repo_config_get_bool(the_repository, "transfer.advertisesid", &advertise_sid);
1931
0
  if (!uri_protocols.nr) {
1932
0
    char *str;
1933
1934
0
    if (!repo_config_get_string(the_repository, "fetch.uriprotocols", &str) && str) {
1935
0
      string_list_split(&uri_protocols, str, ",", -1);
1936
0
      free(str);
1937
0
    }
1938
0
  }
1939
1940
0
  repo_config(the_repository, fetch_pack_config_cb, NULL);
1941
0
}
1942
1943
static void fetch_pack_setup(void)
1944
0
{
1945
0
  static int did_setup;
1946
0
  if (did_setup)
1947
0
    return;
1948
0
  fetch_pack_config();
1949
0
  if (0 <= fetch_unpack_limit)
1950
0
    unpack_limit = fetch_unpack_limit;
1951
0
  else if (0 <= transfer_unpack_limit)
1952
0
    unpack_limit = transfer_unpack_limit;
1953
0
  did_setup = 1;
1954
0
}
1955
1956
static int remove_duplicates_in_refs(struct ref **ref, int nr)
1957
0
{
1958
0
  struct string_list names = STRING_LIST_INIT_NODUP;
1959
0
  int src, dst;
1960
1961
0
  for (src = dst = 0; src < nr; src++) {
1962
0
    struct string_list_item *item;
1963
0
    item = string_list_insert(&names, ref[src]->name);
1964
0
    if (item->util)
1965
0
      continue; /* already have it */
1966
0
    item->util = ref[src];
1967
0
    if (src != dst)
1968
0
      ref[dst] = ref[src];
1969
0
    dst++;
1970
0
  }
1971
0
  for (src = dst; src < nr; src++)
1972
0
    ref[src] = NULL;
1973
0
  string_list_clear(&names, 0);
1974
0
  return dst;
1975
0
}
1976
1977
static void update_shallow(struct fetch_pack_args *args,
1978
         struct ref **sought, int nr_sought,
1979
         struct shallow_info *si)
1980
0
{
1981
0
  struct oid_array ref = OID_ARRAY_INIT;
1982
0
  int *status;
1983
0
  int i;
1984
1985
0
  if (args->deepen && alternate_shallow_file) {
1986
0
    if (*alternate_shallow_file == '\0') { /* --unshallow */
1987
0
      unlink_or_warn(git_path_shallow(the_repository));
1988
0
      rollback_shallow_file(the_repository, &shallow_lock);
1989
0
    } else
1990
0
      commit_shallow_file(the_repository, &shallow_lock);
1991
0
    alternate_shallow_file = NULL;
1992
0
    return;
1993
0
  }
1994
1995
0
  if (!si->shallow || !si->shallow->nr)
1996
0
    return;
1997
1998
0
  if (args->cloning) {
1999
    /*
2000
     * remote is shallow, but this is a clone, there are
2001
     * no objects in repo to worry about. Accept any
2002
     * shallow points that exist in the pack (iow in repo
2003
     * after get_pack() and odb_reprepare())
2004
     */
2005
0
    struct oid_array extra = OID_ARRAY_INIT;
2006
0
    struct object_id *oid = si->shallow->oid;
2007
0
    for (i = 0; i < si->shallow->nr; i++)
2008
0
      if (odb_has_object(the_repository->objects, &oid[i],
2009
0
             HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))
2010
0
        oid_array_append(&extra, &oid[i]);
2011
0
    if (extra.nr) {
2012
0
      setup_alternate_shallow(&shallow_lock,
2013
0
            &alternate_shallow_file,
2014
0
            &extra);
2015
0
      commit_shallow_file(the_repository, &shallow_lock);
2016
0
      alternate_shallow_file = NULL;
2017
0
    }
2018
0
    oid_array_clear(&extra);
2019
0
    return;
2020
0
  }
2021
2022
0
  if (!si->nr_ours && !si->nr_theirs)
2023
0
    return;
2024
2025
0
  remove_nonexistent_theirs_shallow(si);
2026
0
  if (!si->nr_ours && !si->nr_theirs)
2027
0
    return;
2028
0
  for (i = 0; i < nr_sought; i++)
2029
0
    oid_array_append(&ref, &sought[i]->old_oid);
2030
0
  si->ref = &ref;
2031
2032
0
  if (args->update_shallow) {
2033
    /*
2034
     * remote is also shallow, .git/shallow may be updated
2035
     * so all refs can be accepted. Make sure we only add
2036
     * shallow roots that are actually reachable from new
2037
     * refs.
2038
     */
2039
0
    struct oid_array extra = OID_ARRAY_INIT;
2040
0
    struct object_id *oid = si->shallow->oid;
2041
0
    assign_shallow_commits_to_refs(si, NULL, NULL);
2042
0
    if (!si->nr_ours && !si->nr_theirs) {
2043
0
      oid_array_clear(&ref);
2044
0
      return;
2045
0
    }
2046
0
    for (i = 0; i < si->nr_ours; i++)
2047
0
      oid_array_append(&extra, &oid[si->ours[i]]);
2048
0
    for (i = 0; i < si->nr_theirs; i++)
2049
0
      oid_array_append(&extra, &oid[si->theirs[i]]);
2050
0
    setup_alternate_shallow(&shallow_lock,
2051
0
          &alternate_shallow_file,
2052
0
          &extra);
2053
0
    commit_shallow_file(the_repository, &shallow_lock);
2054
0
    oid_array_clear(&extra);
2055
0
    oid_array_clear(&ref);
2056
0
    alternate_shallow_file = NULL;
2057
0
    return;
2058
0
  }
2059
2060
  /*
2061
   * remote is also shallow, check what ref is safe to update
2062
   * without updating .git/shallow
2063
   */
2064
0
  CALLOC_ARRAY(status, nr_sought);
2065
0
  assign_shallow_commits_to_refs(si, NULL, status);
2066
0
  if (si->nr_ours || si->nr_theirs) {
2067
0
    for (i = 0; i < nr_sought; i++)
2068
0
      if (status[i])
2069
0
        sought[i]->status = REF_STATUS_REJECT_SHALLOW;
2070
0
  }
2071
0
  free(status);
2072
0
  oid_array_clear(&ref);
2073
0
}
2074
2075
static const struct object_id *iterate_ref_map(void *cb_data)
2076
0
{
2077
0
  struct ref **rm = cb_data;
2078
0
  struct ref *ref = *rm;
2079
2080
0
  if (!ref)
2081
0
    return NULL;
2082
0
  *rm = ref->next;
2083
0
  return &ref->old_oid;
2084
0
}
2085
2086
int fetch_pack_fsck_objects(void)
2087
0
{
2088
0
  fetch_pack_setup();
2089
0
  if (fetch_fsck_objects >= 0)
2090
0
    return fetch_fsck_objects;
2091
0
  if (transfer_fsck_objects >= 0)
2092
0
    return transfer_fsck_objects;
2093
0
  return 0;
2094
0
}
2095
2096
struct ref *fetch_pack(struct fetch_pack_args *args,
2097
           int fd[],
2098
           const struct ref *ref,
2099
           struct ref **sought, int nr_sought,
2100
           struct oid_array *shallow,
2101
           struct string_list *pack_lockfiles,
2102
           enum protocol_version version)
2103
0
{
2104
0
  struct ref *ref_cpy;
2105
0
  struct shallow_info si;
2106
0
  struct oid_array shallows_scratch = OID_ARRAY_INIT;
2107
2108
0
  fetch_pack_setup();
2109
0
  if (nr_sought)
2110
0
    nr_sought = remove_duplicates_in_refs(sought, nr_sought);
2111
2112
0
  if (version != protocol_v2 && !ref) {
2113
0
    packet_flush(fd[1]);
2114
0
    die(_("no matching remote head"));
2115
0
  }
2116
0
  if (version == protocol_v2) {
2117
0
    if (shallow->nr)
2118
0
      BUG("Protocol V2 does not provide shallows at this point in the fetch");
2119
0
    memset(&si, 0, sizeof(si));
2120
0
    ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
2121
0
             &shallows_scratch, &si,
2122
0
             pack_lockfiles);
2123
0
  } else {
2124
0
    prepare_shallow_info(&si, shallow);
2125
0
    ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
2126
0
          &si, pack_lockfiles);
2127
0
  }
2128
0
  odb_reprepare(the_repository->objects);
2129
2130
0
  if (!args->cloning && args->deepen) {
2131
0
    struct check_connected_options opt = CHECK_CONNECTED_INIT;
2132
0
    struct ref *iterator = ref_cpy;
2133
0
    opt.shallow_file = alternate_shallow_file;
2134
0
    if (args->deepen)
2135
0
      opt.is_deepening_fetch = 1;
2136
0
    if (check_connected(iterate_ref_map, &iterator, &opt)) {
2137
0
      error(_("remote did not send all necessary objects"));
2138
0
      free_refs(ref_cpy);
2139
0
      ref_cpy = NULL;
2140
0
      rollback_shallow_file(the_repository, &shallow_lock);
2141
0
      goto cleanup;
2142
0
    }
2143
0
    args->connectivity_checked = 1;
2144
0
  }
2145
2146
0
  update_shallow(args, sought, nr_sought, &si);
2147
0
cleanup:
2148
0
  clear_shallow_info(&si);
2149
0
  oid_array_clear(&shallows_scratch);
2150
0
  return ref_cpy;
2151
0
}
2152
2153
static int add_to_object_array(const struct object_id *oid, void *data)
2154
0
{
2155
0
  struct object_array *a = data;
2156
2157
0
  add_object_array(lookup_object(the_repository, oid), "", a);
2158
0
  return 0;
2159
0
}
2160
2161
static void clear_common_flag(struct oidset *s)
2162
0
{
2163
0
  struct oidset_iter iter;
2164
0
  const struct object_id *oid;
2165
0
  oidset_iter_init(s, &iter);
2166
2167
0
  while ((oid = oidset_iter_next(&iter))) {
2168
0
    struct object *obj = lookup_object(the_repository, oid);
2169
0
    obj->flags &= ~COMMON;
2170
0
  }
2171
0
}
2172
2173
void negotiate_using_fetch(const struct oid_array *negotiation_tips,
2174
         const struct string_list *server_options,
2175
         int stateless_rpc,
2176
         int fd[],
2177
         struct oidset *acked_commits)
2178
0
{
2179
0
  struct fetch_negotiator negotiator;
2180
0
  struct packet_reader reader;
2181
0
  struct object_array nt_object_array = OBJECT_ARRAY_INIT;
2182
0
  struct strbuf req_buf = STRBUF_INIT;
2183
0
  int haves_to_send = INITIAL_FLUSH;
2184
0
  int in_vain = 0;
2185
0
  int seen_ack = 0;
2186
0
  int last_iteration = 0;
2187
0
  int negotiation_round = 0;
2188
0
  timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
2189
2190
0
  fetch_negotiator_init(the_repository, &negotiator);
2191
0
  mark_tips(&negotiator, negotiation_tips);
2192
2193
0
  packet_reader_init(&reader, fd[0], NULL, 0,
2194
0
         PACKET_READ_CHOMP_NEWLINE |
2195
0
         PACKET_READ_DIE_ON_ERR_PACKET);
2196
2197
0
  oid_array_for_each((struct oid_array *) negotiation_tips,
2198
0
         add_to_object_array,
2199
0
         &nt_object_array);
2200
2201
0
  trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository);
2202
0
  while (!last_iteration) {
2203
0
    int haves_added;
2204
0
    struct object_id common_oid;
2205
0
    int received_ready = 0;
2206
2207
0
    negotiation_round++;
2208
2209
0
    trace2_region_enter_printf("negotiate_using_fetch", "round",
2210
0
             the_repository, "%d",
2211
0
             negotiation_round);
2212
0
    strbuf_reset(&req_buf);
2213
0
    write_fetch_command_and_capabilities(&req_buf, server_options);
2214
2215
0
    packet_buf_write(&req_buf, "wait-for-done");
2216
2217
0
    haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
2218
0
    in_vain += haves_added;
2219
0
    if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
2220
0
      last_iteration = 1;
2221
2222
0
    trace2_data_intmax("negotiate_using_fetch", the_repository,
2223
0
           "haves_added", haves_added);
2224
0
    trace2_data_intmax("negotiate_using_fetch", the_repository,
2225
0
           "in_vain", in_vain);
2226
2227
    /* Send request */
2228
0
    packet_buf_flush(&req_buf);
2229
0
    if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
2230
0
      die_errno(_("unable to write request to remote"));
2231
2232
    /* Process ACKs/NAKs */
2233
0
    process_section_header(&reader, "acknowledgments", 0);
2234
0
    while (process_ack(&negotiator, &reader, &common_oid,
2235
0
           &received_ready)) {
2236
0
      struct commit *commit = lookup_commit(the_repository,
2237
0
                    &common_oid);
2238
0
      if (commit) {
2239
0
        timestamp_t generation;
2240
2241
0
        parse_commit_or_die(commit);
2242
0
        commit->object.flags |= COMMON;
2243
0
        generation = commit_graph_generation(commit);
2244
0
        if (generation < min_generation)
2245
0
          min_generation = generation;
2246
0
      }
2247
0
      in_vain = 0;
2248
0
      seen_ack = 1;
2249
0
      oidset_insert(acked_commits, &common_oid);
2250
0
    }
2251
0
    if (received_ready)
2252
0
      die(_("unexpected 'ready' from remote"));
2253
0
    else
2254
0
      do_check_stateless_delimiter(stateless_rpc, &reader);
2255
0
    if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
2256
0
             REACH_SCRATCH, 0,
2257
0
             min_generation))
2258
0
      last_iteration = 1;
2259
0
    trace2_region_leave_printf("negotiation", "round",
2260
0
             the_repository, "%d",
2261
0
             negotiation_round);
2262
0
  }
2263
0
  trace2_region_leave("fetch-pack", "negotiate_using_fetch", the_repository);
2264
0
  trace2_data_intmax("negotiate_using_fetch", the_repository,
2265
0
         "total_rounds", negotiation_round);
2266
2267
0
  clear_common_flag(acked_commits);
2268
0
  object_array_clear(&nt_object_array);
2269
0
  negotiator.release(&negotiator);
2270
0
  strbuf_release(&req_buf);
2271
0
}
2272
2273
int report_unmatched_refs(struct ref **sought, int nr_sought)
2274
0
{
2275
0
  int i, ret = 0;
2276
2277
0
  for (i = 0; i < nr_sought; i++) {
2278
0
    if (!sought[i])
2279
0
      continue;
2280
0
    switch (sought[i]->match_status) {
2281
0
    case REF_MATCHED:
2282
0
      continue;
2283
0
    case REF_NOT_MATCHED:
2284
0
      error(_("no such remote ref %s"), sought[i]->name);
2285
0
      break;
2286
0
    case REF_UNADVERTISED_NOT_ALLOWED:
2287
0
      error(_("Server does not allow request for unadvertised object %s"),
2288
0
            sought[i]->name);
2289
0
      break;
2290
0
    }
2291
0
    ret = 1;
2292
0
  }
2293
0
  return ret;
2294
0
}