Line | Count | Source (jump to first uncovered line) |
1 | | /* Command line parsing. |
2 | | Copyright (C) 1996-2015, 2018-2024 Free Software Foundation, Inc. |
3 | | |
4 | | This file is part of GNU Wget. |
5 | | |
6 | | GNU Wget is free software; you can redistribute it and/or modify |
7 | | it under the terms of the GNU General Public License as published by |
8 | | the Free Software Foundation; either version 3 of the License, or |
9 | | (at your option) any later version. |
10 | | |
11 | | GNU Wget is distributed in the hope that it will be useful, |
12 | | but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
14 | | GNU General Public License for more details. |
15 | | |
16 | | You should have received a copy of the GNU General Public License |
17 | | along with Wget. If not, see <http://www.gnu.org/licenses/>. |
18 | | |
19 | | Additional permission under GNU GPL version 3 section 7 |
20 | | |
21 | | If you modify this program, or any covered work, by linking or |
22 | | combining it with the OpenSSL project's OpenSSL library (or a |
23 | | modified version of that library), containing parts covered by the |
24 | | terms of the OpenSSL or SSLeay licenses, the Free Software Foundation |
25 | | grants you additional permission to convey the resulting work. |
26 | | Corresponding Source for a non-source form of such a combination |
27 | | shall include the source code for the parts of OpenSSL used as well |
28 | | as that of the covered work. */ |
29 | | |
30 | | #include "wget.h" |
31 | | |
32 | | #include <stdio.h> |
33 | | #include <stdlib.h> |
34 | | #include <unistd.h> |
35 | | #include <string.h> |
36 | | #include <signal.h> |
37 | | #include <spawn.h> |
38 | | #if defined(ENABLE_NLS) || defined(WINDOWS) |
39 | | # include <locale.h> |
40 | | #endif |
41 | | #include <assert.h> |
42 | | #include <errno.h> |
43 | | #include <time.h> |
44 | | |
45 | | #include "exits.h" |
46 | | #include "utils.h" |
47 | | #include "init.h" |
48 | | #include "retr.h" |
49 | | #include "recur.h" |
50 | | #include "host.h" |
51 | | #include "url.h" |
52 | | #include "progress.h" /* for progress_handle_sigwinch */ |
53 | | #include "convert.h" |
54 | | #include "spider.h" |
55 | | #include "http.h" /* for save_cookies */ |
56 | | #include "hsts.h" /* for initializing hsts_store to NULL */ |
57 | | #include "ptimer.h" |
58 | | #include "warc.h" |
59 | | #include "version.h" |
60 | | #include "c-strcase.h" |
61 | | #include "dirname.h" |
62 | | #include "xmemdup0.h" |
63 | | #include <getopt.h> |
64 | | #include <getpass.h> |
65 | | #include <quote.h> |
66 | | |
67 | | #ifdef TESTING |
68 | | /* Rename the main function so we can have a main() in fuzzing code |
69 | | and call the original main. */ |
70 | | # define main main_wget |
71 | | #endif |
72 | | |
73 | | #ifdef HAVE_METALINK |
74 | | # include <metalink/metalink_parser.h> |
75 | | # include "metalink.h" |
76 | | #endif |
77 | | |
78 | | #ifdef WINDOWS |
79 | | # include <io.h> |
80 | | # include <fcntl.h> |
81 | | #ifndef ENABLE_NLS |
82 | | # include <mbctype.h> |
83 | | #endif |
84 | | #endif |
85 | | |
86 | | #ifdef __VMS |
87 | | # include "vms.h" |
88 | | #endif /* __VMS */ |
89 | | |
90 | | #ifndef PATH_SEPARATOR |
91 | | # define PATH_SEPARATOR '/' |
92 | | #endif |
93 | | |
94 | | #ifndef ENABLE_IRI |
95 | | struct iri dummy_iri; |
96 | | #endif |
97 | | |
98 | | #ifdef HAVE_LIBCARES |
99 | | #include <ares.h> |
100 | | ares_channel ares; |
101 | | #else |
102 | | void *ares; |
103 | | #endif |
104 | | |
105 | | struct options opt; |
106 | | |
107 | | /* defined in version.c */ |
108 | | extern char *system_getrc; |
109 | | /* Used for --version output in print_version */ |
110 | 0 | #define MAX_CHARS_PER_LINE 72 |
111 | 0 | #define TABULATION 4 |
112 | | |
113 | | const char *exec_name; |
114 | | |
115 | | /* Number of successfully downloaded URLs */ |
116 | | int numurls = 0; |
117 | | |
118 | | /* Initialize I18N/L10N. That amounts to invoking setlocale, and |
119 | | setting up gettext's message catalog using bindtextdomain and |
120 | | textdomain. Does nothing if NLS is disabled or missing. */ |
121 | | |
122 | | #if defined(SIGHUP) || defined(SIGUSR1) |
123 | | /* Hangup signal handler. When wget receives SIGHUP or SIGUSR1, it |
124 | | will proceed operation as usual, trying to write into a log file. |
125 | | If that is impossible, the output will be turned off. */ |
126 | | |
127 | | static void |
128 | | redirect_output_signal (int sig) |
129 | 0 | { |
130 | 0 | const char *signal_name = "WTF?!"; |
131 | |
|
132 | 0 | #ifdef SIGHUP |
133 | 0 | if (sig == SIGHUP) |
134 | 0 | signal_name = "SIGHUP"; |
135 | 0 | #endif |
136 | 0 | #ifdef SIGUSR1 |
137 | 0 | if (sig == SIGUSR1) |
138 | 0 | signal_name = "SIGUSR1"; |
139 | 0 | #endif |
140 | |
|
141 | 0 | redirect_output (true,signal_name); |
142 | 0 | progress_schedule_redirect (); |
143 | 0 | signal (sig, redirect_output_signal); |
144 | 0 | } |
145 | | #endif /* defined(SIGHUP) || defined(SIGUSR1) */ |
146 | | |
147 | | static void |
148 | | i18n_initialize (void) |
149 | 0 | { |
150 | | /* ENABLE_NLS implies existence of functions invoked here. */ |
151 | 0 | #ifdef ENABLE_NLS |
152 | | /* Set the current locale. */ |
153 | 0 | setlocale (LC_ALL, ""); |
154 | | /* Set the text message domain. */ |
155 | 0 | bindtextdomain ("wget", LOCALEDIR); |
156 | 0 | bindtextdomain ("wget-gnulib", LOCALEDIR); |
157 | 0 | textdomain ("wget"); |
158 | | #elif defined WINDOWS |
159 | | char MBCP[16] = ""; |
160 | | int CP; |
161 | | |
162 | | CP = _getmbcp(); /* Consider it's different from default. */ |
163 | | if (CP > 0) |
164 | | snprintf(MBCP, sizeof(MBCP), ".%d", CP); |
165 | | setlocale(LC_ALL, MBCP); |
166 | | #endif /* ENABLE_NLS */ |
167 | 0 | } |
168 | | |
169 | | #ifdef HAVE_HSTS |
170 | | /* make the HSTS store global */ |
171 | | hsts_store_t hsts_store; |
172 | | |
173 | | static char* |
174 | | get_hsts_database (void) |
175 | 0 | { |
176 | 0 | if (opt.hsts_file) |
177 | 0 | return xstrdup (opt.hsts_file); |
178 | | |
179 | 0 | if (opt.homedir) |
180 | 0 | { |
181 | 0 | char *dir = ajoin_dir_file(opt.homedir, ".wget-hsts"); |
182 | 0 | return dir; |
183 | 0 | } |
184 | | |
185 | 0 | return NULL; |
186 | 0 | } |
187 | | |
188 | | static void |
189 | | load_hsts (void) |
190 | 0 | { |
191 | 0 | if (!hsts_store) |
192 | 0 | { |
193 | 0 | char *filename = get_hsts_database (); |
194 | |
|
195 | 0 | if (filename) |
196 | 0 | { |
197 | 0 | DEBUGP (("Reading HSTS entries from %s\n", filename)); |
198 | |
|
199 | 0 | hsts_store = hsts_store_open (filename); |
200 | |
|
201 | 0 | if (!hsts_store) |
202 | 0 | logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store at '%s'. " |
203 | 0 | "HSTS will be disabled.\n", |
204 | 0 | filename); |
205 | 0 | } |
206 | 0 | else |
207 | 0 | logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store. HSTS will be disabled.\n"); |
208 | |
|
209 | 0 | xfree (filename); |
210 | 0 | } |
211 | 0 | } |
212 | | |
213 | | static void |
214 | | save_hsts (void) |
215 | 0 | { |
216 | 0 | if (hsts_store) |
217 | 0 | { |
218 | 0 | char *filename = get_hsts_database (); |
219 | |
|
220 | 0 | if (filename && hsts_store_has_changed (hsts_store)) |
221 | 0 | { |
222 | 0 | DEBUGP (("Saving HSTS entries to %s\n", filename)); |
223 | 0 | hsts_store_save (hsts_store, filename); |
224 | 0 | } |
225 | |
|
226 | 0 | hsts_store_close (hsts_store); |
227 | 0 | xfree (hsts_store); |
228 | |
|
229 | 0 | xfree (filename); |
230 | 0 | } |
231 | 0 | } |
232 | | #endif |
233 | | |
234 | | /* Definition of command-line options. */ |
235 | | |
236 | | _Noreturn static void print_help (void); |
237 | | _Noreturn static void print_version (void); |
238 | | |
239 | | #ifdef HAVE_SSL |
240 | | # define IF_SSL(a,b,c,d,e) { a, b, c, d , e }, |
241 | | #else |
242 | | # define IF_SSL(a,b,c,d,e) |
243 | | #endif |
244 | | |
245 | | struct cmdline_option { |
246 | | char long_name[MAX_LONGOPTION]; |
247 | | char short_name; |
248 | | enum { |
249 | | OPT_VALUE, |
250 | | OPT_BOOLEAN, |
251 | | OPT_FUNCALL, |
252 | | /* Non-standard options that have to be handled specially in |
253 | | main(). */ |
254 | | OPT__APPEND_OUTPUT, |
255 | | OPT__CLOBBER, |
256 | | OPT__DONT_REMOVE_LISTING, |
257 | | OPT__EXECUTE, |
258 | | OPT__NO, |
259 | | OPT__PARENT |
260 | | } type; |
261 | | const void *data; /* for standard options */ |
262 | | int argtype; /* for non-standard options */ |
263 | | }; |
264 | | |
265 | | static struct cmdline_option option_data[] = |
266 | | { |
267 | | { "accept", 'A', OPT_VALUE, "accept", -1 }, |
268 | | { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, |
269 | | { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, |
270 | | { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, |
271 | | { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, |
272 | | { "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 }, |
273 | | { "background", 'b', OPT_BOOLEAN, "background", -1 }, |
274 | | { "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 }, |
275 | | { "backups", 0, OPT_BOOLEAN, "backups", -1 }, |
276 | | { "base", 'B', OPT_VALUE, "base", -1 }, |
277 | | { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, |
278 | | #ifdef HAVE_LIBCARES |
279 | | { "bind-dns-address", 0, OPT_VALUE, "binddnsaddress", -1 }, |
280 | | #endif |
281 | | { "body-data", 0, OPT_VALUE, "bodydata", -1 }, |
282 | | { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, |
283 | | IF_SSL ( "ca-certificate", 0, OPT_VALUE, "cacertificate", -1 ) |
284 | | IF_SSL ( "ca-directory", 0, OPT_VALUE, "cadirectory", -1 ) |
285 | | { "cache", 0, OPT_BOOLEAN, "cache", -1 }, |
286 | | IF_SSL ( "certificate", 0, OPT_VALUE, "certificate", -1 ) |
287 | | IF_SSL ( "certificate-type", 0, OPT_VALUE, "certificatetype", -1 ) |
288 | | IF_SSL ( "check-certificate", 0, OPT_BOOLEAN, "checkcertificate", -1 ) |
289 | | { "clobber", 0, OPT__CLOBBER, NULL, optional_argument }, |
290 | | #ifdef HAVE_LIBZ |
291 | | { "compression", 0, OPT_VALUE, "compression", -1 }, |
292 | | #endif |
293 | | { "config", 0, OPT_VALUE, "chooseconfig", -1 }, |
294 | | { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 }, |
295 | | { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, |
296 | | { "convert-file-only", 0, OPT_BOOLEAN, "convertfileonly", -1 }, |
297 | | { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, |
298 | | { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 }, |
299 | | { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 }, |
300 | | { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, |
301 | | IF_SSL ( "crl-file", 0, OPT_VALUE, "crlfile", -1 ) |
302 | | { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 }, |
303 | | { "debug", 'd', OPT_BOOLEAN, "debug", -1 }, |
304 | | { "default-page", 0, OPT_VALUE, "defaultpage", -1 }, |
305 | | { "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 }, |
306 | | { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 }, |
307 | | { "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 }, |
308 | | { "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 }, |
309 | | #ifdef HAVE_LIBCARES |
310 | | { "dns-servers", 0, OPT_VALUE, "dnsservers", -1 }, |
311 | | #endif |
312 | | { "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 }, |
313 | | { "domains", 'D', OPT_VALUE, "domains", -1 }, |
314 | | { "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument }, |
315 | | { "dot-style", 0, OPT_VALUE, "dotstyle", -1 }, /* deprecated */ |
316 | | { "egd-file", 0, OPT_VALUE, "egdfile", -1 }, |
317 | | { "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 }, |
318 | | { "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 }, |
319 | | { "execute", 'e', OPT__EXECUTE, NULL, required_argument }, |
320 | | { "follow-ftp", 0, OPT_BOOLEAN, "followftp", -1 }, |
321 | | { "follow-tags", 0, OPT_VALUE, "followtags", -1 }, |
322 | | { "force-directories", 'x', OPT_BOOLEAN, "dirstruct", -1 }, |
323 | | { "force-html", 'F', OPT_BOOLEAN, "forcehtml", -1 }, |
324 | | { "ftp-password", 0, OPT_VALUE, "ftppassword", -1 }, |
325 | | #ifdef __VMS |
326 | | { "ftp-stmlf", 0, OPT_BOOLEAN, "ftpstmlf", -1 }, |
327 | | #endif /* def __VMS */ |
328 | | { "ftp-user", 0, OPT_VALUE, "ftpuser", -1 }, |
329 | | IF_SSL ( "ftps-clear-data-connection", 0, OPT_BOOLEAN, "ftpscleardataconnection", -1 ) |
330 | | IF_SSL ( "ftps-fallback-to-ftp", 0, OPT_BOOLEAN, "ftpsfallbacktoftp", -1 ) |
331 | | IF_SSL ( "ftps-implicit", 0, OPT_BOOLEAN, "ftpsimplicit", -1 ) |
332 | | IF_SSL ( "ftps-resume-ssl", 0, OPT_BOOLEAN, "ftpsresumessl", -1 ) |
333 | | { "glob", 0, OPT_BOOLEAN, "glob", -1 }, |
334 | | { "header", 0, OPT_VALUE, "header", -1 }, |
335 | | { "help", 'h', OPT_FUNCALL, (void *)print_help, no_argument }, |
336 | | { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 }, |
337 | | #ifdef HAVE_HSTS |
338 | | { "hsts", 0, OPT_BOOLEAN, "hsts", -1}, |
339 | | { "hsts-file", 0, OPT_VALUE, "hstsfile", -1 }, |
340 | | #endif |
341 | | { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */ |
342 | | { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 }, |
343 | | { "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 }, |
344 | | { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */ |
345 | | { "http-password", 0, OPT_VALUE, "httppassword", -1 }, |
346 | | { "http-user", 0, OPT_VALUE, "httpuser", -1 }, |
347 | | IF_SSL ( "https-only", 0, OPT_BOOLEAN, "httpsonly", -1 ) |
348 | | { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 }, |
349 | | { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 }, |
350 | | { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 }, |
351 | | { "include-directories", 'I', OPT_VALUE, "includedirectories", -1 }, |
352 | | #ifdef ENABLE_IPV6 |
353 | | { "inet4-only", '4', OPT_BOOLEAN, "inet4only", -1 }, |
354 | | { "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 }, |
355 | | #endif |
356 | | { "input-file", 'i', OPT_VALUE, "input", -1 }, |
357 | | #ifdef HAVE_METALINK |
358 | | { "input-metalink", 0, OPT_VALUE, "inputmetalink", -1 }, |
359 | | #endif |
360 | | { "iri", 0, OPT_BOOLEAN, "iri", -1 }, |
361 | | { "keep-badhash", 0, OPT_BOOLEAN, "keepbadhash", -1 }, |
362 | | { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 }, |
363 | | { "level", 'l', OPT_VALUE, "reclevel", -1 }, |
364 | | { "limit-rate", 0, OPT_VALUE, "limitrate", -1 }, |
365 | | { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, |
366 | | { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, |
367 | | { "rejected-log", 0, OPT_VALUE, "rejectedlog", -1 }, |
368 | | { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, |
369 | | #ifdef HAVE_METALINK |
370 | | { "metalink-index", 0, OPT_VALUE, "metalinkindex", -1 }, |
371 | | { "metalink-over-http", 0, OPT_BOOLEAN, "metalinkoverhttp", -1 }, |
372 | | #endif |
373 | | { "method", 0, OPT_VALUE, "method", -1 }, |
374 | | { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, |
375 | | { "netrc", 0, OPT_BOOLEAN, "netrc", -1 }, |
376 | | { "no", 'n', OPT__NO, NULL, required_argument }, |
377 | | { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, |
378 | | { "no-config", 0, OPT_BOOLEAN, "noconfig", -1}, |
379 | | { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 }, |
380 | | { "output-document", 'O', OPT_VALUE, "outputdocument", -1 }, |
381 | | { "output-file", 'o', OPT_VALUE, "logfile", -1 }, |
382 | | { "page-requisites", 'p', OPT_BOOLEAN, "pagerequisites", -1 }, |
383 | | { "parent", 0, OPT__PARENT, NULL, optional_argument }, |
384 | | { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 }, |
385 | | { "password", 0, OPT_VALUE, "password", -1 }, |
386 | | IF_SSL ( "pinnedpubkey", 0, OPT_VALUE, "pinnedpubkey", -1 ) |
387 | | { "post-data", 0, OPT_VALUE, "postdata", -1 }, |
388 | | { "post-file", 0, OPT_VALUE, "postfile", -1 }, |
389 | | { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, |
390 | | #ifdef HAVE_METALINK |
391 | | { "preferred-location", 0, OPT_VALUE, "preferredlocation", -1 }, |
392 | | #endif |
393 | | { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, |
394 | | IF_SSL ( "ciphers", 0, OPT_VALUE, "ciphers", -1 ) |
395 | | IF_SSL ( "private-key", 0, OPT_VALUE, "privatekey", -1 ) |
396 | | IF_SSL ( "private-key-type", 0, OPT_VALUE, "privatekeytype", -1 ) |
397 | | { "progress", 0, OPT_VALUE, "progress", -1 }, |
398 | | { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 }, |
399 | | { "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 }, |
400 | | { "proxy", 0, OPT_BOOLEAN, "useproxy", -1 }, |
401 | | { "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */ |
402 | | { "proxy-passwd", 0, OPT_VALUE, "proxypassword", -1 }, /* deprecated */ |
403 | | { "proxy-password", 0, OPT_VALUE, "proxypassword", -1 }, |
404 | | { "proxy-user", 0, OPT_VALUE, "proxyuser", -1 }, |
405 | | { "quiet", 'q', OPT_BOOLEAN, "quiet", -1 }, |
406 | | { "quota", 'Q', OPT_VALUE, "quota", -1 }, |
407 | | { "random-file", 0, OPT_VALUE, "randomfile", -1 }, |
408 | | { "random-wait", 0, OPT_BOOLEAN, "randomwait", -1 }, |
409 | | { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, |
410 | | { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, |
411 | | { "referer", 0, OPT_VALUE, "referer", -1 }, |
412 | | { "regex-type", 0, OPT_VALUE, "regextype", -1 }, |
413 | | { "reject", 'R', OPT_VALUE, "reject", -1 }, |
414 | | { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, |
415 | | { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, |
416 | | { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, |
417 | | { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, |
418 | | { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, |
419 | | { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, |
420 | | { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, |
421 | | { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, |
422 | | { "retry-on-host-error", 0, OPT_BOOLEAN, "retryonhosterror", -1 }, |
423 | | { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 }, |
424 | | { "save-cookies", 0, OPT_VALUE, "savecookies", -1 }, |
425 | | { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 }, |
426 | | IF_SSL ( "secure-protocol", 0, OPT_VALUE, "secureprotocol", -1 ) |
427 | | { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 }, |
428 | | { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 }, |
429 | | { "spider", 0, OPT_BOOLEAN, "spider", -1 }, |
430 | | { "start-pos", 0, OPT_VALUE, "startpos", -1 }, |
431 | | { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, |
432 | | { "timeout", 'T', OPT_VALUE, "timeout", -1 }, |
433 | | { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, |
434 | | { "if-modified-since", 0, OPT_BOOLEAN, "ifmodifiedsince", -1 }, |
435 | | { "tries", 't', OPT_VALUE, "tries", -1 }, |
436 | | { "unlink", 0, OPT_BOOLEAN, "unlink", -1 }, |
437 | | { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 }, |
438 | | #ifndef __VMS |
439 | | { "use-askpass", 0, OPT_VALUE, "useaskpass", -1}, |
440 | | #endif |
441 | | { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 }, |
442 | | { "user", 0, OPT_VALUE, "user", -1 }, |
443 | | { "user-agent", 'U', OPT_VALUE, "useragent", -1 }, |
444 | | { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 }, |
445 | | { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, |
446 | | { "wait", 'w', OPT_VALUE, "wait", -1 }, |
447 | | { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, |
448 | | { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, |
449 | | #ifdef HAVE_LIBZ |
450 | | { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, |
451 | | #endif |
452 | | { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, |
453 | | { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, |
454 | | { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, |
455 | | { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, |
456 | | { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, |
457 | | { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, |
458 | | { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, |
459 | | #ifdef USE_WATT32 |
460 | | { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, |
461 | | #endif |
462 | | #ifdef ENABLE_XATTR |
463 | | { "xattr", 0, OPT_BOOLEAN, "xattr", -1 }, |
464 | | #endif |
465 | | }; |
466 | | |
467 | | #undef IF_SSL |
468 | | |
469 | | /* Return a string that contains S with "no-" prepended. The string |
470 | | is NUL-terminated and allocated off static storage at Wget |
471 | | startup. */ |
472 | | |
473 | | static char * |
474 | | no_prefix (const char *s) |
475 | 0 | { |
476 | 0 | static char buffer[2048]; |
477 | 0 | static char *p = buffer; |
478 | |
|
479 | 0 | char *cp = p; |
480 | 0 | int size = 3 + strlen (s) + 1; /* "no-STRING\0" */ |
481 | 0 | assert(p + size <= buffer + sizeof (buffer)); |
482 | |
|
483 | 0 | cp[0] = 'n', cp[1] = 'o', cp[2] = '-'; |
484 | 0 | strcpy (cp + 3, s); |
485 | 0 | p += size; |
486 | 0 | return cp; |
487 | 0 | } |
488 | | |
489 | | /* The arguments that that main passes to getopt_long. */ |
490 | | static struct option long_options[2 * countof (option_data) + 1]; |
491 | | static char short_options[128]; |
492 | | |
493 | | /* Mapping between short option chars and option_data indices. */ |
494 | | static unsigned char optmap[96]; |
495 | | |
496 | | /* Marker for `--no-FOO' values in long_options. */ |
497 | 0 | #define BOOLEAN_NEG_MARKER 1024 |
498 | | |
499 | | /* Initialize the long_options array used by getopt_long from the data |
500 | | in option_data. */ |
501 | | |
502 | | static void |
503 | | init_switches (void) |
504 | 0 | { |
505 | 0 | static bool initialized; |
506 | 0 | char *p = short_options; |
507 | 0 | size_t i, o = 0; |
508 | |
|
509 | 0 | if (initialized) |
510 | 0 | return; |
511 | 0 | initialized = 1; |
512 | |
|
513 | 0 | for (i = 0; i < countof (option_data); i++) |
514 | 0 | { |
515 | 0 | struct cmdline_option *cmdopt = &option_data[i]; |
516 | 0 | struct option *longopt; |
517 | |
|
518 | 0 | longopt = &long_options[o++]; |
519 | 0 | longopt->name = cmdopt->long_name; |
520 | 0 | longopt->val = i; |
521 | 0 | if (cmdopt->short_name) |
522 | 0 | { |
523 | 0 | *p++ = cmdopt->short_name; |
524 | 0 | optmap[cmdopt->short_name - 32] = longopt - long_options; |
525 | 0 | } |
526 | 0 | switch (cmdopt->type) |
527 | 0 | { |
528 | 0 | case OPT_VALUE: |
529 | 0 | longopt->has_arg = required_argument; |
530 | 0 | if (cmdopt->short_name) |
531 | 0 | *p++ = ':'; |
532 | 0 | break; |
533 | 0 | case OPT_BOOLEAN: |
534 | | /* Specify an optional argument for long options, so that |
535 | | --option=off works the same as --no-option, for |
536 | | compatibility with pre-1.10 Wget. However, don't specify |
537 | | optional arguments short-option booleans because they |
538 | | prevent combining of short options. */ |
539 | 0 | longopt->has_arg = optional_argument; |
540 | | /* For Boolean options, add the "--no-FOO" variant, which is |
541 | | identical to "--foo", except it has opposite meaning and |
542 | | it doesn't allow an argument. */ |
543 | 0 | longopt = &long_options[o++]; |
544 | 0 | longopt->name = no_prefix (cmdopt->long_name); |
545 | 0 | longopt->has_arg = no_argument; |
546 | | /* Mask the value so we'll be able to recognize that we're |
547 | | dealing with the false value. */ |
548 | 0 | longopt->val = i | BOOLEAN_NEG_MARKER; |
549 | 0 | break; |
550 | 0 | default: |
551 | 0 | assert (cmdopt->argtype != -1); |
552 | 0 | longopt->has_arg = cmdopt->argtype; |
553 | 0 | if (cmdopt->short_name) |
554 | 0 | { |
555 | 0 | if (longopt->has_arg == required_argument) |
556 | 0 | *p++ = ':'; |
557 | | /* Don't handle optional_argument */ |
558 | 0 | } |
559 | 0 | } |
560 | 0 | } |
561 | | /* Terminate short_options. */ |
562 | 0 | *p = '\0'; |
563 | | /* No need for xzero(long_options[o]) because its storage is static |
564 | | and it will be zeroed by default. */ |
565 | 0 | assert (o <= countof (long_options)); |
566 | 0 | } |
567 | | |
568 | | /* Print the usage message. */ |
569 | | static int |
570 | | print_usage (_GL_UNUSED int error) |
571 | 0 | { |
572 | | #ifndef TESTING |
573 | | return fprintf (error ? stderr : stdout, |
574 | | _("Usage: %s [OPTION]... [URL]...\n"), exec_name); |
575 | | #else |
576 | 0 | return 0; |
577 | 0 | #endif |
578 | 0 | } |
579 | | |
580 | | /* Print the help message, describing all the available options. If |
581 | | you add an option, be sure to update this list. */ |
582 | | _Noreturn static void |
583 | | print_help (void) |
584 | 0 | { |
585 | | #ifndef TESTING |
586 | | /* We split the help text this way to ease translation of individual |
587 | | entries. */ |
588 | | static const char *help[] = { |
589 | | "\n", |
590 | | N_("\ |
591 | | Mandatory arguments to long options are mandatory for short options too.\n\n"), |
592 | | N_("\ |
593 | | Startup:\n"), |
594 | | N_("\ |
595 | | -V, --version display the version of Wget and exit\n"), |
596 | | N_("\ |
597 | | -h, --help print this help\n"), |
598 | | N_("\ |
599 | | -b, --background go to background after startup\n"), |
600 | | N_("\ |
601 | | -e, --execute=COMMAND execute a `.wgetrc'-style command\n"), |
602 | | "\n", |
603 | | |
604 | | N_("\ |
605 | | Logging and input file:\n"), |
606 | | N_("\ |
607 | | -o, --output-file=FILE log messages to FILE\n"), |
608 | | N_("\ |
609 | | -a, --append-output=FILE append messages to FILE\n"), |
610 | | #ifdef ENABLE_DEBUG |
611 | | N_("\ |
612 | | -d, --debug print lots of debugging information\n"), |
613 | | #endif |
614 | | #ifdef USE_WATT32 |
615 | | N_("\ |
616 | | --wdebug print Watt-32 debug output\n"), |
617 | | #endif |
618 | | N_("\ |
619 | | -q, --quiet quiet (no output)\n"), |
620 | | N_("\ |
621 | | -v, --verbose be verbose (this is the default)\n"), |
622 | | N_("\ |
623 | | -nv, --no-verbose turn off verboseness, without being quiet\n"), |
624 | | N_("\ |
625 | | --report-speed=TYPE output bandwidth as TYPE. TYPE can be bits\n"), |
626 | | N_("\ |
627 | | -i, --input-file=FILE download URLs found in local or external FILE\n"), |
628 | | #ifdef HAVE_METALINK |
629 | | N_("\ |
630 | | --input-metalink=FILE download files covered in local Metalink FILE\n"), |
631 | | #endif |
632 | | N_("\ |
633 | | -F, --force-html treat input file as HTML\n"), |
634 | | N_("\ |
635 | | -B, --base=URL resolves HTML input-file links (-i -F)\n\ |
636 | | relative to URL\n"), |
637 | | N_("\ |
638 | | --config=FILE specify config file to use\n"), |
639 | | N_("\ |
640 | | --no-config do not read any config file\n"), |
641 | | N_("\ |
642 | | --rejected-log=FILE log reasons for URL rejection to FILE\n"), |
643 | | "\n", |
644 | | |
645 | | N_("\ |
646 | | Download:\n"), |
647 | | N_("\ |
648 | | -t, --tries=NUMBER set number of retries to NUMBER (0 unlimits)\n"), |
649 | | N_("\ |
650 | | --retry-connrefused retry even if connection is refused\n"), |
651 | | N_("\ |
652 | | --retry-on-host-error consider host errors as non-fatal, transient errors\n"), |
653 | | N_("\ |
654 | | --retry-on-http-error=ERRORS comma-separated list of HTTP errors to retry\n"), |
655 | | N_("\ |
656 | | -O, --output-document=FILE write documents to FILE\n"), |
657 | | N_("\ |
658 | | -nc, --no-clobber skip downloads that would download to\n\ |
659 | | existing files (overwriting them)\n"), |
660 | | N_("\ |
661 | | --no-netrc don't try to obtain credentials from .netrc\n"), |
662 | | N_("\ |
663 | | -c, --continue resume getting a partially-downloaded file\n"), |
664 | | N_("\ |
665 | | --start-pos=OFFSET start downloading from zero-based position OFFSET\n"), |
666 | | N_("\ |
667 | | --progress=TYPE select progress gauge type\n"), |
668 | | N_("\ |
669 | | --show-progress display the progress bar in any verbosity mode\n"), |
670 | | N_("\ |
671 | | -N, --timestamping don't re-retrieve files unless newer than\n\ |
672 | | local\n"), |
673 | | N_("\ |
674 | | --no-if-modified-since don't use conditional if-modified-since get\n\ |
675 | | requests in timestamping mode\n"), |
676 | | N_("\ |
677 | | --no-use-server-timestamps don't set the local file's timestamp by\n\ |
678 | | the one on the server\n"), |
679 | | N_("\ |
680 | | -S, --server-response print server response\n"), |
681 | | N_("\ |
682 | | --spider don't download anything\n"), |
683 | | N_("\ |
684 | | -T, --timeout=SECONDS set all timeout values to SECONDS\n"), |
685 | | #ifdef HAVE_LIBCARES |
686 | | N_("\ |
687 | | --dns-servers=ADDRESSES list of DNS servers to query (comma separated)\n"), |
688 | | N_("\ |
689 | | --bind-dns-address=ADDRESS bind DNS resolver to ADDRESS (hostname or IP) on local host\n"), |
690 | | #endif |
691 | | N_("\ |
692 | | --dns-timeout=SECS set the DNS lookup timeout to SECS\n"), |
693 | | N_("\ |
694 | | --connect-timeout=SECS set the connect timeout to SECS\n"), |
695 | | N_("\ |
696 | | --read-timeout=SECS set the read timeout to SECS\n"), |
697 | | N_("\ |
698 | | -w, --wait=SECONDS wait SECONDS between retrievals\n\ |
699 | | (applies if more then 1 URL is to be retrieved)\n"), |
700 | | N_("\ |
701 | | --waitretry=SECONDS wait 1..SECONDS between retries of a retrieval\n\ |
702 | | (applies if more then 1 URL is to be retrieved)\n"), |
703 | | N_("\ |
704 | | --random-wait wait from 0.5*WAIT...1.5*WAIT secs between retrievals\n\ |
705 | | (applies if more then 1 URL is to be retrieved)\n"), |
706 | | N_("\ |
707 | | --no-proxy explicitly turn off proxy\n"), |
708 | | N_("\ |
709 | | -Q, --quota=NUMBER set retrieval quota to NUMBER\n"), |
710 | | N_("\ |
711 | | --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host\n"), |
712 | | N_("\ |
713 | | --limit-rate=RATE limit download rate to RATE\n"), |
714 | | N_("\ |
715 | | --no-dns-cache disable caching DNS lookups\n"), |
716 | | N_("\ |
717 | | --restrict-file-names=OS restrict chars in file names to ones OS allows\n"), |
718 | | N_("\ |
719 | | --ignore-case ignore case when matching files/directories\n"), |
720 | | #ifdef ENABLE_IPV6 |
721 | | N_("\ |
722 | | -4, --inet4-only connect only to IPv4 addresses\n"), |
723 | | N_("\ |
724 | | -6, --inet6-only connect only to IPv6 addresses\n"), |
725 | | N_("\ |
726 | | --prefer-family=FAMILY connect first to addresses of specified family,\n\ |
727 | | one of IPv6, IPv4, or none\n"), |
728 | | #endif |
729 | | N_("\ |
730 | | --user=USER set both ftp and http user to USER\n"), |
731 | | N_("\ |
732 | | --password=PASS set both ftp and http password to PASS\n"), |
733 | | N_("\ |
734 | | --ask-password prompt for passwords\n"), |
735 | | #ifndef __VMS |
736 | | N_("\ |
737 | | --use-askpass=COMMAND specify credential handler for requesting \n\ |
738 | | username and password. If no COMMAND is \n\ |
739 | | specified the WGET_ASKPASS or the SSH_ASKPASS \n\ |
740 | | environment variable is used.\n"), |
741 | | #endif |
742 | | N_("\ |
743 | | --no-iri turn off IRI support\n"), |
744 | | N_("\ |
745 | | --local-encoding=ENC use ENC as the local encoding for IRIs\n"), |
746 | | N_("\ |
747 | | --remote-encoding=ENC use ENC as the default remote encoding\n"), |
748 | | N_("\ |
749 | | --unlink remove file before clobber\n"), |
750 | | #ifdef HAVE_METALINK |
751 | | N_("\ |
752 | | --keep-badhash keep files with checksum mismatch (append .badhash)\n"), |
753 | | N_("\ |
754 | | --metalink-index=NUMBER Metalink application/metalink4+xml metaurl ordinal NUMBER\n"), |
755 | | N_("\ |
756 | | --metalink-over-http use Metalink metadata from HTTP response headers\n"), |
757 | | N_("\ |
758 | | --preferred-location preferred location for Metalink resources\n"), |
759 | | #endif |
760 | | #ifdef ENABLE_XATTR |
761 | | N_("\ |
762 | | --xattr turn on storage of metadata in extended file attributes\n"), |
763 | | #endif |
764 | | "\n", |
765 | | |
766 | | N_("\ |
767 | | Directories:\n"), |
768 | | N_("\ |
769 | | -nd, --no-directories don't create directories\n"), |
770 | | N_("\ |
771 | | -x, --force-directories force creation of directories\n"), |
772 | | N_("\ |
773 | | -nH, --no-host-directories don't create host directories\n"), |
774 | | N_("\ |
775 | | --protocol-directories use protocol name in directories\n"), |
776 | | N_("\ |
777 | | -P, --directory-prefix=PREFIX save files to PREFIX/..\n"), |
778 | | N_("\ |
779 | | --cut-dirs=NUMBER ignore NUMBER remote directory components\n"), |
780 | | "\n", |
781 | | |
782 | | N_("\ |
783 | | HTTP options:\n"), |
784 | | N_("\ |
785 | | --http-user=USER set http user to USER\n"), |
786 | | N_("\ |
787 | | --http-password=PASS set http password to PASS\n"), |
788 | | N_("\ |
789 | | --no-cache disallow server-cached data\n"), |
790 | | N_ ("\ |
791 | | --default-page=NAME change the default page name (normally\n\ |
792 | | this is 'index.html'.)\n"), |
793 | | N_("\ |
794 | | -E, --adjust-extension save HTML/CSS documents with proper extensions\n"), |
795 | | N_("\ |
796 | | --ignore-length ignore 'Content-Length' header field\n"), |
797 | | N_("\ |
798 | | --header=STRING insert STRING among the headers\n"), |
799 | | #ifdef HAVE_LIBZ |
800 | | N_("\ |
801 | | --compression=TYPE choose compression, one of auto, gzip and none. (default: none)\n"), |
802 | | #endif |
803 | | N_("\ |
804 | | --max-redirect maximum redirections allowed per page\n"), |
805 | | N_("\ |
806 | | --proxy-user=USER set USER as proxy username\n"), |
807 | | N_("\ |
808 | | --proxy-password=PASS set PASS as proxy password\n"), |
809 | | N_("\ |
810 | | --referer=URL include 'Referer: URL' header in HTTP request\n"), |
811 | | N_("\ |
812 | | --save-headers save the HTTP headers to file\n"), |
813 | | N_("\ |
814 | | -U, --user-agent=AGENT identify as AGENT instead of Wget/VERSION\n"), |
815 | | N_("\ |
816 | | --no-http-keep-alive disable HTTP keep-alive (persistent connections)\n"), |
817 | | N_("\ |
818 | | --no-cookies don't use cookies\n"), |
819 | | N_("\ |
820 | | --load-cookies=FILE load cookies from FILE before session\n"), |
821 | | N_("\ |
822 | | --save-cookies=FILE save cookies to FILE after session\n"), |
823 | | N_("\ |
824 | | --keep-session-cookies load and save session (non-permanent) cookies\n"), |
825 | | N_("\ |
826 | | --post-data=STRING use the POST method; send STRING as the data\n"), |
827 | | N_("\ |
828 | | --post-file=FILE use the POST method; send contents of FILE\n"), |
829 | | N_("\ |
830 | | --method=HTTPMethod use method \"HTTPMethod\" in the request\n"), |
831 | | N_("\ |
832 | | --body-data=STRING send STRING as data. --method MUST be set\n"), |
833 | | N_("\ |
834 | | --body-file=FILE send contents of FILE. --method MUST be set\n"), |
835 | | N_("\ |
836 | | --content-disposition honor the Content-Disposition header when\n\ |
837 | | choosing local file names (EXPERIMENTAL)\n"), |
838 | | N_("\ |
839 | | --content-on-error output the received content on server errors\n"), |
840 | | N_("\ |
841 | | --auth-no-challenge send Basic HTTP authentication information\n\ |
842 | | without first waiting for the server's\n\ |
843 | | challenge\n"), |
844 | | "\n", |
845 | | |
846 | | #ifdef HAVE_SSL |
847 | | N_("\ |
848 | | HTTPS (SSL/TLS) options:\n"), |
849 | | N_("\ |
850 | | --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\ |
851 | | SSLv3, TLSv1, TLSv1_1, TLSv1_2, TLSv1_3 and PFS\n"), |
852 | | N_("\ |
853 | | --https-only only follow secure HTTPS links\n"), |
854 | | N_("\ |
855 | | --no-check-certificate don't validate the server's certificate\n"), |
856 | | N_("\ |
857 | | --certificate=FILE client certificate file\n"), |
858 | | N_("\ |
859 | | --certificate-type=TYPE client certificate type, PEM or DER\n"), |
860 | | N_("\ |
861 | | --private-key=FILE private key file\n"), |
862 | | N_("\ |
863 | | --private-key-type=TYPE private key type, PEM or DER\n"), |
864 | | N_("\ |
865 | | --ca-certificate=FILE file with the bundle of CAs\n"), |
866 | | N_("\ |
867 | | --ca-directory=DIR directory where hash list of CAs is stored\n"), |
868 | | N_("\ |
869 | | --crl-file=FILE file with bundle of CRLs\n"), |
870 | | N_("\ |
871 | | --pinnedpubkey=FILE/HASHES Public key (PEM/DER) file, or any number\n\ |
872 | | of base64 encoded sha256 hashes preceded by\n\ |
873 | | \'sha256//\' and separated by \';\', to verify\n\ |
874 | | peer against\n"), |
875 | | #if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32) |
876 | | N_("\ |
877 | | --random-file=FILE file with random data for seeding the SSL PRNG\n"), |
878 | | #endif |
879 | | #if (defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)) && defined(HAVE_RAND_EGD) |
880 | | N_("\ |
881 | | --egd-file=FILE file naming the EGD socket with random data\n"), |
882 | | #endif |
883 | | "\n", |
884 | | N_("\ |
885 | | --ciphers=STR Set the priority string (GnuTLS) or cipher list string (OpenSSL) directly.\n\ |
886 | | Use with care. This option overrides --secure-protocol.\n\ |
887 | | The format and syntax of this string depend on the specific SSL/TLS engine.\n"), |
888 | | #endif /* HAVE_SSL */ |
889 | | |
890 | | #ifdef HAVE_HSTS |
891 | | N_("\ |
892 | | HSTS options:\n"), |
893 | | N_("\ |
894 | | --no-hsts disable HSTS\n"), |
895 | | N_("\ |
896 | | --hsts-file path of HSTS database (will override default)\n"), |
897 | | "\n", |
898 | | #endif |
899 | | |
900 | | N_("\ |
901 | | FTP options:\n"), |
902 | | #ifdef __VMS |
903 | | N_("\ |
904 | | --ftp-stmlf use Stream_LF format for all binary FTP files\n"), |
905 | | #endif /* def __VMS */ |
906 | | N_("\ |
907 | | --ftp-user=USER set ftp user to USER\n"), |
908 | | N_("\ |
909 | | --ftp-password=PASS set ftp password to PASS\n"), |
910 | | N_("\ |
911 | | --no-remove-listing don't remove '.listing' files\n"), |
912 | | N_("\ |
913 | | --no-glob turn off FTP file name globbing\n"), |
914 | | N_("\ |
915 | | --no-passive-ftp disable the \"passive\" transfer mode\n"), |
916 | | N_("\ |
917 | | --preserve-permissions preserve remote file permissions\n"), |
918 | | N_("\ |
919 | | --retr-symlinks when recursing, get linked-to files (not dir)\n"), |
920 | | "\n", |
921 | | |
922 | | #ifdef HAVE_SSL |
923 | | N_("\ |
924 | | FTPS options:\n"), |
925 | | N_("\ |
926 | | --ftps-implicit use implicit FTPS (default port is 990)\n"), |
927 | | N_("\ |
928 | | --ftps-resume-ssl resume the SSL/TLS session started in the control connection when\n" |
929 | | " opening a data connection\n"), |
930 | | N_("\ |
931 | | --ftps-clear-data-connection cipher the control channel only; all the data will be in plaintext\n"), |
932 | | N_("\ |
933 | | --ftps-fallback-to-ftp fall back to FTP if FTPS is not supported in the target server\n"), |
934 | | #endif |
935 | | |
936 | | N_("\ |
937 | | WARC options:\n"), |
938 | | N_("\ |
939 | | --warc-file=FILENAME save request/response data to a .warc.gz file\n"), |
940 | | N_("\ |
941 | | --warc-header=STRING insert STRING into the warcinfo record\n"), |
942 | | N_("\ |
943 | | --warc-max-size=NUMBER set maximum size of WARC files to NUMBER\n"), |
944 | | N_("\ |
945 | | --warc-cdx write CDX index files\n"), |
946 | | N_("\ |
947 | | --warc-dedup=FILENAME do not store records listed in this CDX file\n"), |
948 | | #ifdef HAVE_LIBZ |
949 | | N_("\ |
950 | | --no-warc-compression do not compress WARC files with GZIP\n"), |
951 | | #endif |
952 | | N_("\ |
953 | | --no-warc-digests do not calculate SHA1 digests\n"), |
954 | | N_("\ |
955 | | --no-warc-keep-log do not store the log file in a WARC record\n"), |
956 | | N_("\ |
957 | | --warc-tempdir=DIRECTORY location for temporary files created by the\n\ |
958 | | WARC writer\n"), |
959 | | "\n", |
960 | | |
961 | | N_("\ |
962 | | Recursive download:\n"), |
963 | | N_("\ |
964 | | -r, --recursive specify recursive download\n"), |
965 | | N_("\ |
966 | | -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite)\n"), |
967 | | N_("\ |
968 | | --delete-after delete files locally after downloading them\n"), |
969 | | N_("\ |
970 | | -k, --convert-links make links in downloaded HTML or CSS point to\n\ |
971 | | local files\n"), |
972 | | N_("\ |
973 | | --convert-file-only convert the file part of the URLs only (usually known as the basename)\n"), |
974 | | N_("\ |
975 | | --backups=N before writing file X, rotate up to N backup files\n"), |
976 | | |
977 | | #ifdef __VMS |
978 | | N_("\ |
979 | | -K, --backup-converted before converting file X, back up as X_orig\n"), |
980 | | #else /* def __VMS */ |
981 | | N_("\ |
982 | | -K, --backup-converted before converting file X, back up as X.orig\n"), |
983 | | #endif /* def __VMS [else] */ |
984 | | N_("\ |
985 | | -m, --mirror shortcut for -N -r -l inf --no-remove-listing\n"), |
986 | | N_("\ |
987 | | -p, --page-requisites get all images, etc. needed to display HTML page\n"), |
988 | | N_("\ |
989 | | --strict-comments turn on strict (SGML) handling of HTML comments\n"), |
990 | | "\n", |
991 | | |
992 | | N_("\ |
993 | | Recursive accept/reject:\n"), |
994 | | N_("\ |
995 | | -A, --accept=LIST comma-separated list of accepted extensions\n"), |
996 | | N_("\ |
997 | | -R, --reject=LIST comma-separated list of rejected extensions\n"), |
998 | | N_("\ |
999 | | --accept-regex=REGEX regex matching accepted URLs\n"), |
1000 | | N_("\ |
1001 | | --reject-regex=REGEX regex matching rejected URLs\n"), |
1002 | | #if defined HAVE_LIBPCRE || defined HAVE_LIBPCRE2 |
1003 | | N_("\ |
1004 | | --regex-type=TYPE regex type (posix|pcre)\n"), |
1005 | | #else |
1006 | | N_("\ |
1007 | | --regex-type=TYPE regex type (posix)\n"), |
1008 | | #endif |
1009 | | N_("\ |
1010 | | -D, --domains=LIST comma-separated list of accepted domains\n"), |
1011 | | N_("\ |
1012 | | --exclude-domains=LIST comma-separated list of rejected domains\n"), |
1013 | | N_("\ |
1014 | | --follow-ftp follow FTP links from HTML documents\n"), |
1015 | | N_("\ |
1016 | | --follow-tags=LIST comma-separated list of followed HTML tags\n"), |
1017 | | N_("\ |
1018 | | --ignore-tags=LIST comma-separated list of ignored HTML tags\n"), |
1019 | | N_("\ |
1020 | | -H, --span-hosts go to foreign hosts when recursive\n"), |
1021 | | N_("\ |
1022 | | -L, --relative follow relative links only\n"), |
1023 | | N_("\ |
1024 | | -I, --include-directories=LIST list of allowed directories\n"), |
1025 | | N_("\ |
1026 | | --trust-server-names use the name specified by the redirection\n\ |
1027 | | URL's last component\n"), |
1028 | | N_("\ |
1029 | | -X, --exclude-directories=LIST list of excluded directories\n"), |
1030 | | N_("\ |
1031 | | -np, --no-parent don't ascend to the parent directory\n"), |
1032 | | "\n", |
1033 | | N_("Email bug reports, questions, discussions to <bug-wget@gnu.org>\n" |
1034 | | "and/or open issues at https://savannah.gnu.org/bugs/?func=additem&group=wget.\n") |
1035 | | }; |
1036 | | |
1037 | | size_t i; |
1038 | | |
1039 | | if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"), |
1040 | | version_string) < 0) |
1041 | | exit (WGET_EXIT_IO_FAIL); |
1042 | | if (print_usage (0) < 0) |
1043 | | exit (WGET_EXIT_IO_FAIL); |
1044 | | |
1045 | | for (i = 0; i < countof (help); i++) |
1046 | | if (fputs (_(help[i]), stdout) < 0) |
1047 | | exit (WGET_EXIT_IO_FAIL); |
1048 | | #endif /* TESTING */ |
1049 | 0 | exit (WGET_EXIT_SUCCESS); |
1050 | 0 | } |
1051 | | |
1052 | | /* Return a human-readable printed representation of INTERVAL, |
1053 | | measured in seconds. */ |
1054 | | |
1055 | | static char * |
1056 | | secs_to_human_time (double interval) |
1057 | 0 | { |
1058 | 0 | static char buf[32]; |
1059 | 0 | int secs = (int) (interval + 0.5); |
1060 | 0 | int hours, mins, days; |
1061 | |
|
1062 | 0 | days = secs / 86400, secs %= 86400; |
1063 | 0 | hours = secs / 3600, secs %= 3600; |
1064 | 0 | mins = secs / 60, secs %= 60; |
1065 | |
|
1066 | 0 | if (days) |
1067 | 0 | snprintf (buf, sizeof(buf), "%dd %dh %dm %ds", days, hours, mins, secs); |
1068 | 0 | else if (hours) |
1069 | 0 | snprintf (buf, sizeof(buf), "%dh %dm %ds", hours, mins, secs); |
1070 | 0 | else if (mins) |
1071 | 0 | snprintf (buf, sizeof(buf), "%dm %ds", mins, secs); |
1072 | 0 | else |
1073 | 0 | snprintf (buf, sizeof(buf), "%ss", print_decimal (interval)); |
1074 | |
|
1075 | 0 | return buf; |
1076 | 0 | } |
1077 | | |
1078 | | static char * |
1079 | | prompt_for_password (void) |
1080 | 0 | { |
1081 | 0 | if (opt.user) |
1082 | 0 | fprintf (stderr, _("Password for user %s: "), quote (opt.user)); |
1083 | 0 | else |
1084 | 0 | fprintf (stderr, _("Password: ")); |
1085 | | #ifndef TESTING |
1086 | | /* gnulib's getpass() uses static variables internally, bad for fuzing */ |
1087 | | return getpass(""); |
1088 | | #else |
1089 | 0 | return xstrdup(""); |
1090 | 0 | #endif |
1091 | 0 | } |
1092 | | |
1093 | | |
1094 | | /* Execute external application opt.use_askpass */ |
1095 | | static void |
1096 | | run_use_askpass (char *question, char **answer) |
1097 | 0 | { |
1098 | 0 | char tmp[1024]; |
1099 | 0 | pid_t pid; |
1100 | 0 | int status; |
1101 | 0 | int com[2]; |
1102 | 0 | ssize_t bytes = 0; |
1103 | 0 | char *argv[3], *p; |
1104 | 0 | posix_spawn_file_actions_t fa; |
1105 | |
|
1106 | 0 | if (pipe (com) == -1) |
1107 | 0 | { |
1108 | 0 | fprintf (stderr, _("Cannot create pipe\n")); |
1109 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1110 | 0 | } |
1111 | |
|
1112 | 0 | status = posix_spawn_file_actions_init (&fa); |
1113 | 0 | if (status) |
1114 | 0 | { |
1115 | 0 | fprintf (stderr, |
1116 | 0 | _("Error initializing spawn file actions for use-askpass: %d\n"), |
1117 | 0 | status); |
1118 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1119 | 0 | } |
1120 | |
|
1121 | 0 | status = posix_spawn_file_actions_adddup2 (&fa, com[1], STDOUT_FILENO); |
1122 | 0 | if (status) |
1123 | 0 | { |
1124 | 0 | fprintf (stderr, |
1125 | 0 | _("Error setting spawn file actions for use-askpass: %d\n"), |
1126 | 0 | status); |
1127 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1128 | 0 | } |
1129 | | |
1130 | | /* C89 initializer lists must be computable at load time, |
1131 | | * thus this explicit initialization. */ |
1132 | 0 | argv[0] = opt.use_askpass; |
1133 | 0 | argv[1] = question; |
1134 | 0 | argv[2] = NULL; |
1135 | |
|
1136 | 0 | status = posix_spawnp (&pid, opt.use_askpass, &fa, NULL, argv, environ); |
1137 | 0 | if (status) |
1138 | 0 | { |
1139 | 0 | fprintf (stderr, "Error spawning %s: %d\n", opt.use_askpass, status); |
1140 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1141 | 0 | } |
1142 | | |
1143 | | /* Parent process reads from child. */ |
1144 | 0 | close (com[1]); |
1145 | 0 | bytes = read (com[0], tmp, sizeof (tmp) - 1); |
1146 | 0 | if (bytes <= 0) |
1147 | 0 | { |
1148 | 0 | fprintf (stderr, |
1149 | 0 | _("Error reading response from command \"%s %s\": %s\n"), |
1150 | 0 | opt.use_askpass, question, strerror (errno)); |
1151 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1152 | 0 | } |
1153 | | |
1154 | | /* Make sure there is a trailing 0 */ |
1155 | 0 | tmp[bytes] = '\0'; |
1156 | | |
1157 | | /* Remove a possible new line */ |
1158 | 0 | if ((p = strpbrk (tmp, "\r\n"))) |
1159 | 0 | bytes = p - tmp; |
1160 | |
|
1161 | 0 | *answer = xmemdup0 (tmp, bytes); |
1162 | 0 | } |
1163 | | |
1164 | | /* set the user name and password*/ |
1165 | | static void |
1166 | | use_askpass (struct url *u) |
1167 | 0 | { |
1168 | 0 | static char question[1024]; |
1169 | |
|
1170 | 0 | if (u->user == NULL || u->user[0] == '\0') |
1171 | 0 | { |
1172 | 0 | snprintf (question, sizeof (question), _("Username for '%s%s': "), |
1173 | 0 | scheme_leading_string(u->scheme), u->host); |
1174 | | /* Prompt for username */ |
1175 | 0 | run_use_askpass (question, &u->user); |
1176 | 0 | if (opt.recursive) |
1177 | 0 | opt.user = xstrdup (u->user); |
1178 | 0 | } |
1179 | |
|
1180 | 0 | if (u->passwd == NULL || u->passwd[0] == '\0') |
1181 | 0 | { |
1182 | 0 | snprintf(question, sizeof (question), _("Password for '%s%s@%s': "), |
1183 | 0 | scheme_leading_string (u->scheme), u->user, u->host); |
1184 | | /* Prompt for password */ |
1185 | 0 | run_use_askpass (question, &u->passwd); |
1186 | 0 | if (opt.recursive) |
1187 | 0 | opt.passwd = xstrdup (u->passwd); |
1188 | 0 | } |
1189 | 0 | } |
1190 | | /* Function that prints the line argument while limiting it |
1191 | | to at most line_length. prefix is printed on the first line |
1192 | | and an appropriate number of spaces are added on subsequent |
1193 | | lines.*/ |
1194 | | static int |
1195 | | format_and_print_line (const char *prefix, const char *line, |
1196 | | int line_length) |
1197 | 0 | { |
1198 | 0 | int remaining_chars; |
1199 | 0 | char *line_dup, *token; |
1200 | |
|
1201 | 0 | assert (prefix != NULL); |
1202 | 0 | assert (line != NULL); |
1203 | 0 | assert (line_length > TABULATION); |
1204 | |
|
1205 | 0 | line_dup = xstrdup (line); |
1206 | |
|
1207 | 0 | if (printf ("%s", prefix) < 0) |
1208 | 0 | { |
1209 | 0 | xfree (line_dup); |
1210 | 0 | return -1; |
1211 | 0 | } |
1212 | | |
1213 | | /* Wrap to new line after prefix. */ |
1214 | 0 | remaining_chars = 0; |
1215 | | |
1216 | | /* We break on spaces. */ |
1217 | 0 | token = strtok (line_dup, " "); |
1218 | 0 | while (token != NULL) |
1219 | 0 | { |
1220 | | /* If however a token is much larger than the maximum |
1221 | | line length, all bets are off and we simply print the |
1222 | | token on the next line. */ |
1223 | 0 | if (remaining_chars <= (int) strlen (token)) |
1224 | 0 | { |
1225 | 0 | if (printf ("\n%*c", TABULATION, ' ') < 0) |
1226 | 0 | { |
1227 | 0 | xfree (line_dup); |
1228 | 0 | return -1; |
1229 | 0 | } |
1230 | 0 | remaining_chars = line_length - TABULATION; |
1231 | 0 | } |
1232 | 0 | if (printf ("%s ", token) < 0) |
1233 | 0 | { |
1234 | 0 | xfree (line_dup); |
1235 | 0 | return -1; |
1236 | 0 | } |
1237 | 0 | remaining_chars -= strlen (token) + 1; /* account for " " */ |
1238 | 0 | token = strtok (NULL, " "); |
1239 | 0 | } |
1240 | | |
1241 | 0 | if (printf ("\n") < 0) |
1242 | 0 | { |
1243 | 0 | xfree (line_dup); |
1244 | 0 | return -1; |
1245 | 0 | } |
1246 | | |
1247 | 0 | xfree (line_dup); |
1248 | 0 | return 0; |
1249 | 0 | } |
1250 | | |
1251 | | _Noreturn static void |
1252 | | print_version (void) |
1253 | 0 | { |
1254 | 0 | const char *wgetrc_title = _("Wgetrc: "); |
1255 | 0 | const char *locale_title = _("Locale: "); |
1256 | 0 | const char *compile_title = _("Compile: "); |
1257 | 0 | const char *link_title = _("Link: "); |
1258 | 0 | char *env_wgetrc, *user_wgetrc; |
1259 | 0 | int i; |
1260 | |
|
1261 | 0 | if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0) |
1262 | 0 | exit (WGET_EXIT_IO_FAIL); |
1263 | |
|
1264 | 0 | for (i = 0; compiled_features[i] != NULL; ) |
1265 | 0 | { |
1266 | 0 | int line_length = MAX_CHARS_PER_LINE; |
1267 | 0 | while ((line_length > 0) && (compiled_features[i] != NULL)) |
1268 | 0 | { |
1269 | 0 | if (printf ("%s ", compiled_features[i]) < 0) |
1270 | 0 | exit (WGET_EXIT_IO_FAIL); |
1271 | 0 | line_length -= (int) strlen (compiled_features[i]) + 2; |
1272 | 0 | i++; |
1273 | 0 | } |
1274 | 0 | if (printf ("\n") < 0) |
1275 | 0 | exit (WGET_EXIT_IO_FAIL); |
1276 | 0 | } |
1277 | 0 | if (printf ("\n") < 0) |
1278 | 0 | exit (WGET_EXIT_IO_FAIL); |
1279 | | |
1280 | | /* Print VMS-specific version info. */ |
1281 | | #ifdef __VMS |
1282 | | if (vms_version_supplement() < 0) |
1283 | | exit (WGET_EXIT_IO_FAIL); |
1284 | | #endif /* def __VMS */ |
1285 | | |
1286 | | /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is |
1287 | | absent. */ |
1288 | 0 | if (printf ("%s\n", wgetrc_title) < 0) |
1289 | 0 | exit (WGET_EXIT_IO_FAIL); |
1290 | |
|
1291 | 0 | env_wgetrc = wgetrc_env_file_name (); |
1292 | 0 | if (env_wgetrc && *env_wgetrc) |
1293 | 0 | { |
1294 | 0 | if (printf (_(" %s (env)\n"), env_wgetrc) < 0) |
1295 | 0 | exit (WGET_EXIT_IO_FAIL); |
1296 | 0 | xfree (env_wgetrc); |
1297 | 0 | } |
1298 | 0 | user_wgetrc = wgetrc_user_file_name (); |
1299 | 0 | if (user_wgetrc) |
1300 | 0 | { |
1301 | 0 | if (printf (_(" %s (user)\n"), user_wgetrc) < 0) |
1302 | 0 | exit (WGET_EXIT_IO_FAIL); |
1303 | 0 | xfree (user_wgetrc); |
1304 | 0 | } |
1305 | 0 | #ifdef SYSTEM_WGETRC |
1306 | 0 | if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0) |
1307 | 0 | exit (WGET_EXIT_IO_FAIL); |
1308 | 0 | #endif |
1309 | |
|
1310 | 0 | #ifdef ENABLE_NLS |
1311 | 0 | if (format_and_print_line (locale_title, |
1312 | 0 | LOCALEDIR, |
1313 | 0 | MAX_CHARS_PER_LINE) < 0) |
1314 | 0 | exit (WGET_EXIT_IO_FAIL); |
1315 | 0 | #endif /* def ENABLE_NLS */ |
1316 | |
|
1317 | 0 | if (compilation_string != NULL) |
1318 | 0 | if (format_and_print_line (compile_title, |
1319 | 0 | compilation_string, |
1320 | 0 | MAX_CHARS_PER_LINE) < 0) |
1321 | 0 | exit (WGET_EXIT_IO_FAIL); |
1322 | |
|
1323 | 0 | if (link_string != NULL) |
1324 | 0 | if (format_and_print_line (link_title, |
1325 | 0 | link_string, |
1326 | 0 | MAX_CHARS_PER_LINE) < 0) |
1327 | 0 | exit (WGET_EXIT_IO_FAIL); |
1328 | |
|
1329 | 0 | if (printf ("\n") < 0) |
1330 | 0 | exit (WGET_EXIT_IO_FAIL); |
1331 | | |
1332 | | /* TRANSLATORS: When available, an actual copyright character |
1333 | | (circle-c) should be used in preference to "(C)". */ |
1334 | 0 | if (printf (_("\ |
1335 | 0 | Copyright (C) %s Free Software Foundation, Inc.\n"), "2015") < 0) |
1336 | 0 | exit (WGET_EXIT_IO_FAIL); |
1337 | 0 | if (fputs (_("\ |
1338 | 0 | License GPLv3+: GNU GPL version 3 or later\n\ |
1339 | 0 | <http://www.gnu.org/licenses/gpl.html>.\n\ |
1340 | 0 | This is free software: you are free to change and redistribute it.\n\ |
1341 | 0 | There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) |
1342 | 0 | exit (WGET_EXIT_IO_FAIL); |
1343 | | /* TRANSLATORS: When available, please use the proper diacritics for |
1344 | | names such as this one. See en_US.po for reference. */ |
1345 | 0 | if (fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"), |
1346 | 0 | stdout) < 0) |
1347 | 0 | exit (WGET_EXIT_IO_FAIL); |
1348 | 0 | if (fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"), |
1349 | 0 | stdout) < 0) |
1350 | 0 | exit (WGET_EXIT_IO_FAIL); |
1351 | |
|
1352 | 0 | exit (WGET_EXIT_SUCCESS); |
1353 | 0 | } |
1354 | | |
1355 | | const char *program_name; /* Needed by lib/error.c. */ |
1356 | | const char *program_argstring; /* Needed by wget_warc.c. */ |
1357 | | struct ptimer *timer; |
1358 | | int cleaned_up; |
1359 | | |
1360 | | int |
1361 | | main (int argc, char **argv) |
1362 | 0 | { |
1363 | 0 | char *p; |
1364 | 0 | int i, ret, longindex; |
1365 | 0 | int nurls; |
1366 | 0 | int argstring_length; |
1367 | 0 | bool use_userconfig = false; |
1368 | 0 | bool noconfig = false; |
1369 | 0 | bool append_to_log = false; |
1370 | |
|
1371 | 0 | cleaned_up = 0; /* do cleanup later */ |
1372 | |
|
1373 | 0 | timer = ptimer_new (); |
1374 | 0 | double start_time = ptimer_measure (timer); |
1375 | |
|
1376 | 0 | total_downloaded_bytes = 0; |
1377 | 0 | program_name = argv[0]; |
1378 | |
|
1379 | 0 | i18n_initialize (); |
1380 | | |
1381 | | /* Construct the name of the executable, without the directory part. */ |
1382 | | #ifdef __VMS |
1383 | | /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ |
1384 | | exec_name = vms_basename (argv[0]); |
1385 | | #else /* def __VMS */ |
1386 | 0 | exec_name = base_name (argv[0]); |
1387 | 0 | #endif /* def __VMS [else] */ |
1388 | |
|
1389 | | #ifdef WINDOWS |
1390 | | /* Drop extension (typically .EXE) from executable filename. */ |
1391 | | windows_main ((char **) &exec_name); |
1392 | | #endif |
1393 | | |
1394 | | /* Construct the arguments string. */ |
1395 | 0 | for (argstring_length = 1, i = 1; i < argc; i++) |
1396 | 0 | argstring_length += strlen (argv[i]) + 3 + 1; |
1397 | 0 | program_argstring = p = malloc (argstring_length); |
1398 | 0 | if (p == NULL) |
1399 | 0 | { |
1400 | 0 | fprintf (stderr, _("Memory allocation problem\n")); |
1401 | 0 | exit (WGET_EXIT_PARSE_ERROR); |
1402 | 0 | } |
1403 | 0 | for (i = 1; i < argc; i++) |
1404 | 0 | { |
1405 | 0 | int arglen; |
1406 | |
|
1407 | 0 | *p++ = '"'; |
1408 | 0 | arglen = strlen (argv[i]); |
1409 | 0 | memcpy (p, argv[i], arglen); |
1410 | 0 | p += arglen; |
1411 | 0 | *p++ = '"'; |
1412 | 0 | *p++ = ' '; |
1413 | 0 | } |
1414 | 0 | *p = '\0'; |
1415 | | |
1416 | | /* Load the hard-coded defaults. */ |
1417 | 0 | defaults (); |
1418 | 0 | opt.homedir = home_dir(); |
1419 | |
|
1420 | 0 | init_switches (); |
1421 | | |
1422 | | /* This separate getopt_long is needed to find the user config file |
1423 | | option ("--config") and parse it before the other user options. */ |
1424 | 0 | longindex = -1; |
1425 | |
|
1426 | 0 | while ((getopt_long (argc, argv, |
1427 | 0 | short_options, long_options, &longindex)) != -1) |
1428 | 0 | { |
1429 | 0 | int confval; |
1430 | 0 | struct cmdline_option *config_opt; |
1431 | | |
1432 | | /* There is no short option for "--config". */ |
1433 | 0 | if (longindex >= 0) |
1434 | 0 | { |
1435 | 0 | confval = long_options[longindex].val; |
1436 | 0 | config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; |
1437 | 0 | if (strcmp (config_opt->long_name, "no-config") == 0) |
1438 | 0 | { |
1439 | 0 | noconfig = true; |
1440 | 0 | break; |
1441 | 0 | } |
1442 | 0 | else if (strcmp (config_opt->long_name, "config") == 0) |
1443 | 0 | { |
1444 | 0 | file_stats_t flstats; |
1445 | 0 | use_userconfig = true; |
1446 | 0 | memset(&flstats, 0, sizeof(flstats)); |
1447 | 0 | if (file_exists_p(optarg, &flstats) && run_wgetrc (optarg, &flstats)) |
1448 | 0 | break; |
1449 | 0 | else |
1450 | 0 | { |
1451 | 0 | fprintf (stderr, _("Exiting due to error in %s\n"), optarg); |
1452 | 0 | exit (WGET_EXIT_PARSE_ERROR); |
1453 | 0 | } |
1454 | 0 | } |
1455 | 0 | } |
1456 | 0 | } |
1457 | | |
1458 | | /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ |
1459 | 0 | if (noconfig == false && use_userconfig == false) |
1460 | 0 | if ((ret = initialize ())) |
1461 | 0 | return ret; |
1462 | | |
1463 | 0 | opterr = 0; |
1464 | 0 | optind = 0; |
1465 | |
|
1466 | 0 | longindex = -1; |
1467 | 0 | while ((ret = getopt_long (argc, argv, |
1468 | 0 | short_options, long_options, &longindex)) != -1) |
1469 | 0 | { |
1470 | 0 | int val; |
1471 | 0 | struct cmdline_option *cmdopt; |
1472 | | |
1473 | | /* If LONGINDEX is unchanged, it means RET is referring a short |
1474 | | option. */ |
1475 | 0 | if (longindex == -1) |
1476 | 0 | { |
1477 | 0 | if (ret == '?') |
1478 | 0 | { |
1479 | 0 | print_usage (1); |
1480 | 0 | fprintf (stderr, "\n"); |
1481 | 0 | fprintf (stderr, _("Try `%s --help' for more options.\n"), |
1482 | 0 | exec_name); |
1483 | 0 | exit (WGET_EXIT_PARSE_ERROR); |
1484 | 0 | } |
1485 | | /* Find the short option character in the mapping. */ |
1486 | 0 | longindex = optmap[ret - 32]; |
1487 | 0 | } |
1488 | 0 | val = long_options[longindex].val; |
1489 | | |
1490 | | /* Use the retrieved value to locate the option in the |
1491 | | option_data array, and to see if we're dealing with the |
1492 | | negated "--no-FOO" variant of the boolean option "--foo". */ |
1493 | 0 | cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER]; |
1494 | 0 | switch (cmdopt->type) |
1495 | 0 | { |
1496 | 0 | case OPT_VALUE: |
1497 | 0 | setoptval (cmdopt->data, optarg, cmdopt->long_name); |
1498 | 0 | break; |
1499 | 0 | case OPT_BOOLEAN: |
1500 | 0 | if (optarg) |
1501 | | /* The user has specified a value -- use it. */ |
1502 | 0 | setoptval (cmdopt->data, optarg, cmdopt->long_name); |
1503 | 0 | else |
1504 | 0 | { |
1505 | | /* NEG is true for `--no-FOO' style boolean options. */ |
1506 | 0 | bool neg = !!(val & BOOLEAN_NEG_MARKER); |
1507 | 0 | setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name); |
1508 | 0 | } |
1509 | 0 | break; |
1510 | 0 | case OPT_FUNCALL: |
1511 | 0 | { |
1512 | 0 | void (*func) (void) = (void (*) (void)) cmdopt->data; |
1513 | 0 | func (); |
1514 | 0 | } |
1515 | 0 | break; |
1516 | 0 | case OPT__APPEND_OUTPUT: |
1517 | 0 | setoptval ("logfile", optarg, cmdopt->long_name); |
1518 | 0 | append_to_log = true; |
1519 | 0 | break; |
1520 | 0 | case OPT__EXECUTE: |
1521 | 0 | if (optarg) /* check silences static analyzer */ |
1522 | 0 | run_command (optarg); |
1523 | 0 | break; |
1524 | 0 | case OPT__NO: |
1525 | 0 | { |
1526 | | /* We support real --no-FOO flags now, but keep these |
1527 | | short options for convenience and backward |
1528 | | compatibility. */ |
1529 | 0 | for (p = optarg; p && *p; p++) |
1530 | 0 | switch (*p) |
1531 | 0 | { |
1532 | 0 | case 'v': |
1533 | 0 | setoptval ("verbose", "0", cmdopt->long_name); |
1534 | 0 | break; |
1535 | 0 | case 'H': |
1536 | 0 | setoptval ("addhostdir", "0", cmdopt->long_name); |
1537 | 0 | break; |
1538 | 0 | case 'd': |
1539 | 0 | setoptval ("dirstruct", "0", cmdopt->long_name); |
1540 | 0 | break; |
1541 | 0 | case 'c': |
1542 | 0 | setoptval ("noclobber", "1", cmdopt->long_name); |
1543 | 0 | break; |
1544 | 0 | case 'p': |
1545 | 0 | setoptval ("noparent", "1", cmdopt->long_name); |
1546 | 0 | break; |
1547 | 0 | default: |
1548 | 0 | fprintf (stderr, _("%s: illegal option -- `-n%c'\n"), |
1549 | 0 | exec_name, *p); |
1550 | 0 | print_usage (1); |
1551 | 0 | fprintf (stderr, "\n"); |
1552 | 0 | fprintf (stderr, _("Try `%s --help' for more options.\n"), |
1553 | 0 | exec_name); |
1554 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1555 | 0 | } |
1556 | 0 | break; |
1557 | 0 | } |
1558 | 0 | case OPT__PARENT: |
1559 | 0 | case OPT__CLOBBER: |
1560 | 0 | { |
1561 | | /* The wgetrc commands are named noparent and noclobber, |
1562 | | so we must revert the meaning of the cmdline options |
1563 | | before passing the value to setoptval. */ |
1564 | 0 | bool flag = true; |
1565 | 0 | if (optarg) |
1566 | 0 | flag = (*optarg == '1' || c_tolower (*optarg) == 'y' |
1567 | 0 | || (c_tolower (optarg[0]) == 'o' |
1568 | 0 | && c_tolower (optarg[1]) == 'n')); |
1569 | 0 | setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber", |
1570 | 0 | flag ? "0" : "1", cmdopt->long_name); |
1571 | 0 | break; |
1572 | 0 | } |
1573 | 0 | case OPT__DONT_REMOVE_LISTING: |
1574 | 0 | setoptval ("removelisting", "0", cmdopt->long_name); |
1575 | 0 | break; |
1576 | 0 | } |
1577 | | |
1578 | 0 | longindex = -1; |
1579 | 0 | } |
1580 | | |
1581 | 0 | nurls = argc - optind; |
1582 | | |
1583 | | /* Initialize logging ASAP. */ |
1584 | 0 | log_init (opt.lfilename, append_to_log); |
1585 | | |
1586 | | /* If we do not have Debug support compiled in AND Wget is invoked with the |
1587 | | * --debug switch, instead of failing, we silently turn it into a no-op. For |
1588 | | * this no-op, we explicitly set opt.debug to false and hence none of the |
1589 | | * Debug output messages will be printed. |
1590 | | */ |
1591 | | #ifndef ENABLE_DEBUG |
1592 | | if (opt.debug) |
1593 | | { |
1594 | | fprintf (stderr, _("Debugging support not compiled in. " |
1595 | | "Ignoring --debug flag.\n")); |
1596 | | opt.debug = false; |
1597 | | } |
1598 | | #endif |
1599 | | |
1600 | | /* All user options have now been processed, so it's now safe to do |
1601 | | interoption dependency checks. */ |
1602 | |
|
1603 | 0 | if (opt.noclobber && (opt.convert_links || opt.convert_file_only)) |
1604 | 0 | { |
1605 | 0 | fprintf (stderr, |
1606 | 0 | opt.convert_links ? |
1607 | 0 | _("Both --no-clobber and --convert-links were specified," |
1608 | 0 | " only --convert-links will be used.\n") : |
1609 | 0 | _("Both --no-clobber and --convert-file-only were specified," |
1610 | 0 | " only --convert-file-only will be used.\n")); |
1611 | 0 | opt.noclobber = false; |
1612 | 0 | } |
1613 | |
|
1614 | 0 | if (opt.reclevel == 0) |
1615 | 0 | opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */ |
1616 | |
|
1617 | 0 | if (opt.spider || opt.delete_after) |
1618 | 0 | opt.no_dirstruct = true; |
1619 | |
|
1620 | 0 | if (opt.page_requisites && !opt.recursive) |
1621 | 0 | { |
1622 | | /* Don't set opt.recursive here because it would confuse the FTP |
1623 | | code. Instead, call retrieve_tree below when either |
1624 | | page_requisites or recursive is requested. */ |
1625 | 0 | opt.reclevel = 0; |
1626 | 0 | if (!opt.no_dirstruct) |
1627 | 0 | opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */ |
1628 | 0 | } |
1629 | |
|
1630 | 0 | if (opt.verbose == -1) |
1631 | 0 | opt.verbose = !opt.quiet; |
1632 | |
|
1633 | 0 | if (!opt.verbose && opt.show_progress == -1) |
1634 | 0 | opt.show_progress = false; |
1635 | |
|
1636 | 0 | if (opt.quiet && opt.show_progress == -1) |
1637 | 0 | opt.show_progress = false; |
1638 | | |
1639 | | /* Sanity checks. */ |
1640 | 0 | if (opt.verbose && opt.quiet) |
1641 | 0 | { |
1642 | 0 | fprintf (stderr, _("Can't be verbose and quiet at the same time.\n")); |
1643 | 0 | print_usage (1); |
1644 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1645 | 0 | } |
1646 | 0 | if (opt.timestamping && opt.noclobber) |
1647 | 0 | { |
1648 | 0 | fprintf (stderr, _("\ |
1649 | 0 | Can't timestamp and not clobber old files at the same time.\n")); |
1650 | 0 | print_usage (1); |
1651 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1652 | 0 | } |
1653 | 0 | #ifdef ENABLE_IPV6 |
1654 | 0 | if (opt.ipv4_only && opt.ipv6_only) |
1655 | 0 | { |
1656 | 0 | fprintf (stderr, |
1657 | 0 | _("Cannot specify both --inet4-only and --inet6-only.\n")); |
1658 | 0 | print_usage (1); |
1659 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1660 | 0 | } |
1661 | 0 | #endif |
1662 | 0 | if (opt.output_document) |
1663 | 0 | { |
1664 | 0 | if ((opt.convert_links || opt.convert_file_only) |
1665 | 0 | && (nurls > 1 || opt.page_requisites || opt.recursive)) |
1666 | 0 | { |
1667 | 0 | fputs (_("\ |
1668 | 0 | Cannot specify both -k or --convert-file-only and -O if multiple URLs are given, or in combination\n\ |
1669 | 0 | with -p or -r. See the manual for details.\n\n"), stderr); |
1670 | 0 | print_usage (1); |
1671 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1672 | 0 | } |
1673 | 0 | if (opt.page_requisites |
1674 | 0 | || opt.recursive) |
1675 | 0 | { |
1676 | 0 | logprintf (LOG_NOTQUIET, "%s", _("\ |
1677 | 0 | WARNING: combining -O with -r or -p will mean that all downloaded content\n\ |
1678 | 0 | will be placed in the single file you specified.\n\n")); |
1679 | 0 | } |
1680 | 0 | if (opt.timestamping) |
1681 | 0 | { |
1682 | 0 | logprintf (LOG_NOTQUIET, "%s", _("\ |
1683 | 0 | WARNING: timestamping does nothing in combination with -O. See the manual\n\ |
1684 | 0 | for details.\n\n")); |
1685 | 0 | opt.timestamping = false; |
1686 | 0 | } |
1687 | 0 | if (opt.noclobber && file_exists_p(opt.output_document, NULL)) |
1688 | 0 | { |
1689 | | /* Check if output file exists; if it does, exit. */ |
1690 | 0 | logprintf (LOG_VERBOSE, |
1691 | 0 | _("File %s already there; not retrieving.\n"), |
1692 | 0 | quote (opt.output_document)); |
1693 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1694 | 0 | } |
1695 | 0 | } |
1696 | |
|
1697 | 0 | if (opt.warc_filename != 0) |
1698 | 0 | { |
1699 | 0 | if (opt.noclobber) |
1700 | 0 | { |
1701 | 0 | fprintf (stderr, |
1702 | 0 | _("WARC output does not work with --no-clobber, " |
1703 | 0 | "--no-clobber will be disabled.\n")); |
1704 | 0 | opt.noclobber = false; |
1705 | 0 | } |
1706 | 0 | if (opt.timestamping) |
1707 | 0 | { |
1708 | 0 | fprintf (stderr, |
1709 | 0 | _("WARC output does not work with timestamping, " |
1710 | 0 | "timestamping will be disabled.\n")); |
1711 | 0 | opt.timestamping = false; |
1712 | 0 | } |
1713 | 0 | if (opt.spider) |
1714 | 0 | { |
1715 | 0 | fprintf (stderr, |
1716 | 0 | _("WARC output does not work with --spider.\n")); |
1717 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1718 | 0 | } |
1719 | 0 | if (opt.always_rest || opt.start_pos >= 0) |
1720 | 0 | { |
1721 | 0 | fprintf (stderr, |
1722 | 0 | _("WARC output does not work with --continue or" |
1723 | 0 | " --start-pos, they will be disabled.\n")); |
1724 | 0 | opt.always_rest = false; |
1725 | 0 | opt.start_pos = -1; |
1726 | 0 | } |
1727 | 0 | if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) |
1728 | 0 | { |
1729 | 0 | fprintf (stderr, |
1730 | 0 | _("Digests are disabled; WARC deduplication will " |
1731 | 0 | "not find duplicate records.\n")); |
1732 | 0 | } |
1733 | 0 | if (opt.warc_keep_log) |
1734 | 0 | { |
1735 | 0 | opt.progress_type = xstrdup ("dot"); |
1736 | 0 | } |
1737 | 0 | } |
1738 | |
|
1739 | 0 | #ifdef HAVE_LIBZ |
1740 | 0 | if (opt.always_rest || opt.start_pos >= 0) |
1741 | 0 | { |
1742 | 0 | if (opt.compression == compression_auto) |
1743 | 0 | { |
1744 | | /* Compression does not work with --continue or --start-pos. |
1745 | | Since compression was not explicitly set, it will be disabled. */ |
1746 | 0 | opt.compression = compression_none; |
1747 | 0 | } |
1748 | 0 | else if (opt.compression != compression_none) |
1749 | 0 | { |
1750 | 0 | fprintf (stderr, |
1751 | 0 | _("Compression does not work with --continue or" |
1752 | 0 | " --start-pos, they will be disabled.\n")); |
1753 | 0 | opt.always_rest = false; |
1754 | 0 | opt.start_pos = -1; |
1755 | 0 | } |
1756 | 0 | } |
1757 | 0 | #endif |
1758 | |
|
1759 | 0 | if (opt.ask_passwd && opt.passwd) |
1760 | 0 | { |
1761 | 0 | fprintf (stderr, |
1762 | 0 | _("Cannot specify both --ask-password and --password.\n")); |
1763 | 0 | print_usage (1); |
1764 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1765 | 0 | } |
1766 | |
|
1767 | 0 | if (opt.ask_passwd && !(opt.user || opt.http_user || opt.ftp_user)) |
1768 | 0 | { |
1769 | 0 | fprintf(stderr, |
1770 | 0 | _("WARNING: No username set with --ask-password. This is usually not what you want.\n")); |
1771 | 0 | } |
1772 | |
|
1773 | 0 | if (opt.start_pos >= 0 && opt.always_rest) |
1774 | 0 | { |
1775 | 0 | fprintf (stderr, |
1776 | 0 | _("Specifying both --start-pos and --continue is not " |
1777 | 0 | "recommended; --continue will be disabled.\n")); |
1778 | 0 | opt.always_rest = false; |
1779 | 0 | } |
1780 | |
|
1781 | 0 | if (!nurls && !opt.input_filename |
1782 | | #ifdef HAVE_METALINK |
1783 | | && !opt.input_metalink |
1784 | | #endif |
1785 | 0 | ) |
1786 | 0 | { |
1787 | | /* No URL specified. */ |
1788 | | #ifndef TESTING |
1789 | | fprintf (stderr, _("%s: missing URL\n"), exec_name); |
1790 | | print_usage (1); |
1791 | | fprintf (stderr, "\n"); |
1792 | | /* #### Something nicer should be printed here -- similar to the |
1793 | | pre-1.5 `--help' page. */ |
1794 | | fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); |
1795 | | #endif |
1796 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1797 | 0 | } |
1798 | | |
1799 | | /* Compile the regular expressions. */ |
1800 | 0 | switch (opt.regex_type) |
1801 | 0 | { |
1802 | | #ifdef HAVE_LIBPCRE2 |
1803 | | case regex_type_pcre: |
1804 | | opt.regex_compile_fun = compile_pcre2_regex; |
1805 | | opt.regex_match_fun = match_pcre2_regex; |
1806 | | break; |
1807 | | #endif |
1808 | | #ifdef HAVE_LIBPCRE |
1809 | | case regex_type_pcre: |
1810 | | opt.regex_compile_fun = compile_pcre_regex; |
1811 | | opt.regex_match_fun = match_pcre_regex; |
1812 | | break; |
1813 | | #endif |
1814 | | |
1815 | 0 | case regex_type_posix: |
1816 | 0 | default: |
1817 | 0 | opt.regex_compile_fun = compile_posix_regex; |
1818 | 0 | opt.regex_match_fun = match_posix_regex; |
1819 | 0 | break; |
1820 | 0 | } |
1821 | 0 | if (opt.acceptregex_s) |
1822 | 0 | { |
1823 | 0 | opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); |
1824 | 0 | if (!opt.acceptregex) |
1825 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1826 | 0 | } |
1827 | 0 | if (opt.rejectregex_s) |
1828 | 0 | { |
1829 | 0 | opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); |
1830 | 0 | if (!opt.rejectregex) |
1831 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1832 | 0 | } |
1833 | 0 | if (opt.post_data || opt.post_file_name) |
1834 | 0 | { |
1835 | 0 | if (opt.post_data && opt.post_file_name) |
1836 | 0 | { |
1837 | 0 | fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); |
1838 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1839 | 0 | } |
1840 | 0 | else if (opt.method) |
1841 | 0 | { |
1842 | 0 | fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " |
1843 | 0 | "--method expects data through --body-data and --body-file options\n")); |
1844 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1845 | 0 | } |
1846 | 0 | } |
1847 | 0 | if (opt.body_data || opt.body_file) |
1848 | 0 | { |
1849 | 0 | if (!opt.method) |
1850 | 0 | { |
1851 | 0 | fprintf (stderr, _("You must specify a method through --method=HTTPMethod " |
1852 | 0 | "to use with --body-data or --body-file.\n")); |
1853 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1854 | 0 | } |
1855 | 0 | else if (opt.body_data && opt.body_file) |
1856 | 0 | { |
1857 | 0 | fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); |
1858 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1859 | 0 | } |
1860 | 0 | } |
1861 | | |
1862 | | /* Set various options as required for opt.method. */ |
1863 | | |
1864 | | /* When user specifies HEAD as the method, we do not wish to download any |
1865 | | files. Hence, set wget to run in spider mode. */ |
1866 | 0 | if (opt.method && c_strcasecmp (opt.method, "HEAD") == 0) |
1867 | 0 | setoptval ("spider", "1", "spider"); |
1868 | | |
1869 | | /* Convert post_data to body-data and post_file_name to body-file options. |
1870 | | This is required so as to remove redundant code later on in gethttp(). |
1871 | | The --post-data and --post-file options may also be removed in |
1872 | | the future hence it makes sense to convert them to aliases for |
1873 | | the more generic --method options. |
1874 | | This MUST occur only after the sanity checks so as to prevent the |
1875 | | user from setting both post and body options simultaneously. |
1876 | | */ |
1877 | 0 | if (opt.post_data || opt.post_file_name) |
1878 | 0 | { |
1879 | 0 | setoptval ("method", "POST", "method"); |
1880 | 0 | if (opt.post_data) |
1881 | 0 | { |
1882 | 0 | setoptval ("bodydata", opt.post_data, "body-data"); |
1883 | 0 | xfree(opt.post_data); |
1884 | 0 | } |
1885 | 0 | else |
1886 | 0 | { |
1887 | 0 | setoptval ("bodyfile", opt.post_file_name, "body-file"); |
1888 | 0 | xfree(opt.post_file_name); |
1889 | 0 | } |
1890 | 0 | } |
1891 | |
|
1892 | 0 | #ifdef ENABLE_IRI |
1893 | 0 | if (opt.enable_iri) |
1894 | 0 | { |
1895 | 0 | if (opt.locale && !check_encoding_name (opt.locale)) |
1896 | 0 | xfree (opt.locale); |
1897 | |
|
1898 | 0 | if (!opt.locale) |
1899 | 0 | opt.locale = find_locale (); |
1900 | |
|
1901 | 0 | if (opt.encoding_remote && !check_encoding_name (opt.encoding_remote)) |
1902 | 0 | xfree (opt.encoding_remote); |
1903 | 0 | } |
1904 | | #else |
1905 | | memset (&dummy_iri, 0, sizeof (dummy_iri)); |
1906 | | if (opt.enable_iri || opt.locale || opt.encoding_remote) |
1907 | | { |
1908 | | /* sXXXav : be more specific... */ |
1909 | | fprintf (stderr, _("This version does not have support for IRIs\n")); |
1910 | | exit (WGET_EXIT_GENERIC_ERROR); |
1911 | | } |
1912 | | #endif |
1913 | |
|
1914 | 0 | if (opt.ask_passwd) |
1915 | 0 | { |
1916 | 0 | opt.passwd = prompt_for_password (); |
1917 | |
|
1918 | 0 | if (opt.passwd == NULL || opt.passwd[0] == '\0') |
1919 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1920 | 0 | } |
1921 | |
|
1922 | 0 | if (opt.use_askpass) |
1923 | 0 | { |
1924 | 0 | if (opt.use_askpass[0] == '\0') |
1925 | 0 | { |
1926 | 0 | fprintf (stderr, |
1927 | 0 | _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n")); |
1928 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
1929 | 0 | } |
1930 | 0 | } |
1931 | |
|
1932 | | #ifdef USE_WATT32 |
1933 | | if (opt.wdebug) |
1934 | | dbug_init(); |
1935 | | sock_init(); |
1936 | | #elif ! defined TESTING |
1937 | | if (opt.background) |
1938 | | { |
1939 | | bool logfile_changed = fork_to_background (); |
1940 | | |
1941 | | if (logfile_changed) |
1942 | | log_init (opt.lfilename, append_to_log); |
1943 | | } |
1944 | | #endif |
1945 | | |
1946 | | /* Initialize progress. Have to do this after the options are |
1947 | | processed so we know where the log file is. */ |
1948 | 0 | if (opt.show_progress) |
1949 | 0 | set_progress_implementation (opt.progress_type); |
1950 | | |
1951 | | /* Open WARC file. */ |
1952 | 0 | if (opt.warc_filename != 0) |
1953 | 0 | warc_init (); |
1954 | |
|
1955 | 0 | DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", |
1956 | 0 | version_string, OS_TYPE)); |
1957 | | |
1958 | | /* Open the output filename if necessary. */ |
1959 | | |
1960 | | /* 2005-04-17 SMS. |
1961 | | Note that having the output_stream ("-O") file opened here for an FTP |
1962 | | URL rather than in getftp() (ftp.c) (and the http equivalent) rather |
1963 | | limits the ability in VMS to open the file differently for ASCII |
1964 | | versus binary FTP there. (Of course, doing it here allows a open |
1965 | | failure to be detected immediately, without first connecting to the |
1966 | | server.) |
1967 | | */ |
1968 | 0 | if (opt.output_document) |
1969 | 0 | { |
1970 | 0 | if (HYPHENP (opt.output_document)) |
1971 | 0 | { |
1972 | | #ifdef WINDOWS |
1973 | | _setmode (_fileno (stdout), _O_BINARY); |
1974 | | #endif |
1975 | 0 | output_stream = stdout; |
1976 | 0 | } |
1977 | 0 | else |
1978 | 0 | { |
1979 | 0 | struct stat st; |
1980 | |
|
1981 | | #ifdef __VMS |
1982 | | /* Common fopen() optional arguments: |
1983 | | sequential access only, access callback function. |
1984 | | */ |
1985 | | # define FOPEN_OPT_ARGS , "fop=sqo", "acc", acc_cb, &open_id |
1986 | | int open_id = 7; |
1987 | | #else /* def __VMS */ |
1988 | 0 | # define FOPEN_OPT_ARGS |
1989 | 0 | #endif /* def __VMS [else] */ |
1990 | |
|
1991 | 0 | if (opt.unlink_requested) |
1992 | 0 | { |
1993 | 0 | unlink(opt.output_document); |
1994 | 0 | } |
1995 | |
|
1996 | 0 | output_stream = fopen (opt.output_document, |
1997 | 0 | opt.always_rest ? "ab" : "wb" |
1998 | 0 | FOPEN_OPT_ARGS); |
1999 | 0 | if (output_stream == NULL) |
2000 | 0 | { |
2001 | 0 | perror (opt.output_document); |
2002 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
2003 | 0 | } |
2004 | 0 | if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode)) |
2005 | 0 | output_stream_regular = true; |
2006 | 0 | } |
2007 | 0 | if (!output_stream_regular && (opt.convert_links || opt.recursive)) |
2008 | 0 | { |
2009 | 0 | fprintf (stderr, _("-k or -r can be used together with -O only if \ |
2010 | 0 | outputting to a regular file.\n")); |
2011 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
2012 | 0 | } |
2013 | 0 | if (!output_stream_regular && (opt.convert_links || opt.convert_file_only)) |
2014 | 0 | { |
2015 | 0 | fprintf (stderr, _("--convert-links or --convert-file-only can be used together \ |
2016 | 0 | only if outputting to a regular file.\n")); |
2017 | 0 | exit (WGET_EXIT_GENERIC_ERROR); |
2018 | 0 | } |
2019 | 0 | } |
2020 | |
|
2021 | | #ifdef HAVE_LIBCARES |
2022 | | if (opt.bind_dns_address || opt.dns_servers) |
2023 | | { |
2024 | | if (ares_library_init (ARES_LIB_INIT_ALL)) |
2025 | | { |
2026 | | fprintf (stderr, _("Failed to init libcares\n")); |
2027 | | exit (WGET_EXIT_GENERIC_ERROR); |
2028 | | } |
2029 | | |
2030 | | if (ares_init (&ares) != ARES_SUCCESS) |
2031 | | { |
2032 | | fprintf (stderr, _("Failed to init c-ares channel\n")); |
2033 | | exit (WGET_EXIT_GENERIC_ERROR); |
2034 | | } |
2035 | | |
2036 | | if (opt.bind_dns_address) |
2037 | | { |
2038 | | struct in_addr a4; |
2039 | | #ifdef ENABLE_IPV6 |
2040 | | struct in6_addr a6; |
2041 | | #endif |
2042 | | |
2043 | | if (inet_pton (AF_INET, opt.bind_dns_address, &a4) == 1) |
2044 | | { |
2045 | | ares_set_local_ip4 (ares, ntohl (a4.s_addr)); |
2046 | | } |
2047 | | #ifdef ENABLE_IPV6 |
2048 | | else if (inet_pton (AF_INET6, opt.bind_dns_address, &a6) == 1) |
2049 | | { |
2050 | | ares_set_local_ip6 (ares, (unsigned char *) &a6); |
2051 | | } |
2052 | | #endif |
2053 | | else |
2054 | | { |
2055 | | fprintf (stderr, _("Failed to parse IP address '%s'\n"), opt.bind_dns_address); |
2056 | | exit (WGET_EXIT_GENERIC_ERROR); |
2057 | | } |
2058 | | } |
2059 | | |
2060 | | if (opt.dns_servers) |
2061 | | { |
2062 | | int result; |
2063 | | |
2064 | | if ((result = ares_set_servers_csv (ares, opt.dns_servers)) != ARES_SUCCESS) |
2065 | | { |
2066 | | fprintf (stderr, _("Failed to set DNS server(s) '%s' (%d)\n"), opt.dns_servers, result); |
2067 | | exit (WGET_EXIT_GENERIC_ERROR); |
2068 | | } |
2069 | | } |
2070 | | } |
2071 | | #endif |
2072 | |
|
2073 | | #ifdef __VMS |
2074 | | /* Set global ODS5 flag according to the specified destination (if |
2075 | | any), otherwise according to the current default device. |
2076 | | */ |
2077 | | if (output_stream == NULL) |
2078 | | set_ods5_dest( "SYS$DISK"); |
2079 | | else if (output_stream != stdout) |
2080 | | set_ods5_dest( opt.output_document); |
2081 | | #endif /* def __VMS */ |
2082 | |
|
2083 | | #ifdef WINDOWS |
2084 | | ws_startup (); |
2085 | | #endif |
2086 | |
|
2087 | 0 | #ifdef SIGHUP |
2088 | | /* Setup the signal handler to redirect output when hangup is |
2089 | | received. */ |
2090 | 0 | if (signal(SIGHUP, SIG_IGN) != SIG_IGN) |
2091 | 0 | signal(SIGHUP, redirect_output_signal); |
2092 | 0 | #endif |
2093 | | /* ...and do the same for SIGUSR1. */ |
2094 | 0 | #ifdef SIGUSR1 |
2095 | 0 | signal (SIGUSR1, redirect_output_signal); |
2096 | 0 | #endif |
2097 | 0 | #ifdef SIGPIPE |
2098 | | /* Writing to a closed socket normally signals SIGPIPE, and the |
2099 | | process exits. What we want is to ignore SIGPIPE and just check |
2100 | | for the return value of write(). */ |
2101 | 0 | signal (SIGPIPE, SIG_IGN); |
2102 | 0 | #endif |
2103 | 0 | #ifdef SIGWINCH |
2104 | 0 | signal (SIGWINCH, progress_handle_sigwinch); |
2105 | 0 | #endif |
2106 | |
|
2107 | 0 | #ifdef HAVE_HSTS |
2108 | | /* Load the HSTS database. |
2109 | | Maybe all the URLs are FTP(S), in which case HSTS would not be needed, |
2110 | | but this is the best place to do it, and it shouldn't be a critical |
2111 | | performance hit. |
2112 | | */ |
2113 | 0 | if (opt.hsts) |
2114 | 0 | load_hsts (); |
2115 | 0 | #endif |
2116 | | |
2117 | | /* Retrieve the URLs from argument list. */ |
2118 | 0 | for (i = 0; i < nurls; i++, optind++) |
2119 | 0 | { |
2120 | 0 | char *t; |
2121 | 0 | char *filename = NULL, *redirected_URL = NULL; |
2122 | 0 | int dt = 0, url_err; |
2123 | | /* Need to do a new struct iri every time, because |
2124 | | * retrieve_url may modify it in some circumstances, |
2125 | | * currently. */ |
2126 | 0 | struct iri *iri = iri_new (); |
2127 | 0 | struct url *url_parsed; |
2128 | |
|
2129 | 0 | t = maybe_prepend_scheme (argv[optind]); |
2130 | 0 | if (!t) |
2131 | 0 | t = argv[optind]; |
2132 | |
|
2133 | 0 | set_uri_encoding (iri, opt.locale, true); |
2134 | 0 | url_parsed = url_parse (t, &url_err, iri, true); |
2135 | |
|
2136 | 0 | if (!url_parsed) |
2137 | 0 | { |
2138 | 0 | logprintf (LOG_NOTQUIET, "%s: %s.\n", t, url_error (url_err)); |
2139 | 0 | inform_exit_status (URLERROR); |
2140 | 0 | } |
2141 | 0 | else |
2142 | 0 | { |
2143 | | /* Request credentials if use_askpass is set. */ |
2144 | 0 | if (opt.use_askpass) |
2145 | 0 | use_askpass (url_parsed); |
2146 | |
|
2147 | 0 | if ((opt.recursive || opt.page_requisites) |
2148 | 0 | && ((url_scheme (t) != SCHEME_FTP |
2149 | 0 | #ifdef HAVE_SSL |
2150 | 0 | && url_scheme (t) != SCHEME_FTPS |
2151 | 0 | #endif |
2152 | 0 | ) |
2153 | 0 | || url_uses_proxy (url_parsed))) |
2154 | 0 | { |
2155 | 0 | int old_follow_ftp = opt.follow_ftp; |
2156 | | |
2157 | | /* Turn opt.follow_ftp on in case of recursive FTP retrieval */ |
2158 | 0 | if (url_scheme (t) == SCHEME_FTP |
2159 | 0 | #ifdef HAVE_SSL |
2160 | 0 | || url_scheme (t) == SCHEME_FTPS |
2161 | 0 | #endif |
2162 | 0 | ) |
2163 | 0 | opt.follow_ftp = 1; |
2164 | |
|
2165 | 0 | retrieve_tree (url_parsed, NULL); |
2166 | |
|
2167 | 0 | opt.follow_ftp = old_follow_ftp; |
2168 | 0 | } |
2169 | 0 | else |
2170 | 0 | { |
2171 | 0 | retrieve_url (url_parsed, t, &filename, &redirected_URL, NULL, |
2172 | 0 | &dt, opt.recursive, iri, true); |
2173 | 0 | } |
2174 | |
|
2175 | 0 | if (opt.delete_after && filename != NULL && file_exists_p (filename, NULL)) |
2176 | 0 | { |
2177 | 0 | DEBUGP (("Removing file due to --delete-after in main():\n")); |
2178 | 0 | logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); |
2179 | 0 | if (unlink (filename)) |
2180 | 0 | logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); |
2181 | 0 | } |
2182 | 0 | xfree (redirected_URL); |
2183 | 0 | xfree (filename); |
2184 | 0 | url_free (url_parsed); |
2185 | 0 | } |
2186 | |
|
2187 | 0 | iri_free (iri); |
2188 | |
|
2189 | 0 | if (t != argv[optind]) |
2190 | 0 | xfree (t); |
2191 | 0 | } |
2192 | | |
2193 | | /* And then from the input file, if any. */ |
2194 | 0 | if (opt.input_filename) |
2195 | 0 | { |
2196 | 0 | int count; |
2197 | 0 | int status; |
2198 | 0 | status = retrieve_from_file (opt.input_filename, opt.force_html, &count); |
2199 | 0 | inform_exit_status (status); |
2200 | 0 | if (!count) |
2201 | 0 | logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"), |
2202 | 0 | opt.input_filename); |
2203 | 0 | } |
2204 | |
|
2205 | | #ifdef HAVE_METALINK |
2206 | | /* Finally, from metlink file, if any. */ |
2207 | | if (opt.input_metalink) |
2208 | | { |
2209 | | metalink_error_t meta_err; |
2210 | | uerr_t retr_err; |
2211 | | metalink_t *metalink; |
2212 | | |
2213 | | meta_err = metalink_parse_file (opt.input_metalink, &metalink); |
2214 | | |
2215 | | if (meta_err) |
2216 | | { |
2217 | | logprintf (LOG_NOTQUIET, _("Unable to parse metalink file %s.\n"), |
2218 | | opt.input_metalink); |
2219 | | retr_err = METALINK_PARSE_ERROR; |
2220 | | } |
2221 | | else |
2222 | | { |
2223 | | /* We need to sort the resources if preferred location |
2224 | | was specified by the user. */ |
2225 | | if (opt.preferred_location && opt.preferred_location[0]) |
2226 | | { |
2227 | | metalink_file_t **mfile_ptr; |
2228 | | for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++) |
2229 | | { |
2230 | | metalink_resource_t **mres_ptr; |
2231 | | metalink_file_t *mfile = *mfile_ptr; |
2232 | | size_t mres_count = 0; |
2233 | | |
2234 | | for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++) |
2235 | | mres_count++; |
2236 | | |
2237 | | stable_sort (mfile->resources, |
2238 | | mres_count, |
2239 | | sizeof (metalink_resource_t *), |
2240 | | metalink_res_cmp); |
2241 | | } |
2242 | | } |
2243 | | retr_err = retrieve_from_metalink (metalink); |
2244 | | if (retr_err != RETROK) |
2245 | | { |
2246 | | logprintf (LOG_NOTQUIET, |
2247 | | _("Could not download all resources from %s.\n"), |
2248 | | quote (opt.input_metalink)); |
2249 | | } |
2250 | | metalink_delete (metalink); |
2251 | | } |
2252 | | inform_exit_status (retr_err); |
2253 | | } |
2254 | | #endif /* HAVE_METALINK */ |
2255 | | |
2256 | | /* Print broken links. */ |
2257 | 0 | if (opt.recursive && opt.spider) |
2258 | 0 | print_broken_links (); |
2259 | | |
2260 | | /* Print the downloaded sum. */ |
2261 | 0 | if ((opt.recursive || opt.page_requisites |
2262 | 0 | || nurls > 1 |
2263 | 0 | || (opt.input_filename && total_downloaded_bytes != 0)) |
2264 | 0 | && |
2265 | 0 | total_downloaded_bytes != 0) |
2266 | 0 | { |
2267 | 0 | double end_time = ptimer_measure (timer); |
2268 | 0 | char *wall_time = xstrdup (secs_to_human_time (end_time - start_time)); |
2269 | 0 | char *download_time = xstrdup (secs_to_human_time (total_download_time)); |
2270 | |
|
2271 | 0 | ptimer_destroy (timer); timer = NULL; |
2272 | |
|
2273 | 0 | logprintf (LOG_NOTQUIET, |
2274 | 0 | _("FINISHED --%s--\nTotal wall clock time: %s\n" |
2275 | 0 | "Downloaded: %d files, %s in %s (%s)\n"), |
2276 | 0 | datetime_str (time (NULL)), |
2277 | 0 | wall_time, |
2278 | 0 | numurls, |
2279 | 0 | human_readable (total_downloaded_bytes, 10, 1), |
2280 | 0 | download_time, |
2281 | 0 | retr_rate (total_downloaded_bytes, total_download_time)); |
2282 | 0 | xfree (wall_time); |
2283 | 0 | xfree (download_time); |
2284 | | |
2285 | | /* Print quota warning, if exceeded. */ |
2286 | 0 | if (opt.quota && total_downloaded_bytes > opt.quota) |
2287 | 0 | logprintf (LOG_NOTQUIET, |
2288 | 0 | _("Download quota of %s EXCEEDED!\n"), |
2289 | 0 | human_readable (opt.quota, 10, 1)); |
2290 | 0 | } |
2291 | |
|
2292 | 0 | if (opt.cookies_output) |
2293 | 0 | save_cookies (); |
2294 | |
|
2295 | 0 | #ifdef HAVE_HSTS |
2296 | 0 | if (opt.hsts && hsts_store) |
2297 | 0 | save_hsts (); |
2298 | 0 | #endif |
2299 | |
|
2300 | 0 | if ((opt.convert_links || opt.convert_file_only) && !opt.delete_after) |
2301 | 0 | convert_all_links (); |
2302 | |
|
2303 | 0 | cleanup (); |
2304 | |
|
2305 | 0 | exit (get_exit_status ()); |
2306 | 0 | } |
2307 | | |
2308 | | /* |
2309 | | * vim: et ts=2 sw=2 |
2310 | | */ |