Line | Count | Source (jump to first uncovered line) |
1 | | /* hfile_gcs.c -- Google Cloud Storage backend for low-level file streams. |
2 | | |
3 | | Copyright (C) 2016, 2021 Genome Research Ltd. |
4 | | |
5 | | Author: John Marshall <jm18@sanger.ac.uk> |
6 | | |
7 | | Permission is hereby granted, free of charge, to any person obtaining a copy |
8 | | of this software and associated documentation files (the "Software"), to deal |
9 | | in the Software without restriction, including without limitation the rights |
10 | | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
11 | | copies of the Software, and to permit persons to whom the Software is |
12 | | furnished to do so, subject to the following conditions: |
13 | | |
14 | | The above copyright notice and this permission notice shall be included in |
15 | | all copies or substantial portions of the Software. |
16 | | |
17 | | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
18 | | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
19 | | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
20 | | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
21 | | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
22 | | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
23 | | DEALINGS IN THE SOFTWARE. */ |
24 | | |
25 | | #define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h |
26 | | #include <config.h> |
27 | | |
28 | | #include <stdarg.h> |
29 | | #include <stdio.h> |
30 | | #include <stdlib.h> |
31 | | #include <string.h> |
32 | | #include <errno.h> |
33 | | |
34 | | #include "htslib/hts.h" |
35 | | #include "htslib/kstring.h" |
36 | | #include "hfile_internal.h" |
37 | | #ifdef ENABLE_PLUGINS |
38 | | #include "version.h" |
39 | | #endif |
40 | | |
41 | | static hFILE * |
42 | | gcs_rewrite(const char *gsurl, const char *mode, int mode_has_colon, |
43 | | va_list *argsp) |
44 | 76 | { |
45 | 76 | const char *bucket, *path, *access_token, *requester_pays_project; |
46 | 76 | kstring_t mode_colon = { 0, 0, NULL }; |
47 | 76 | kstring_t url = { 0, 0, NULL }; |
48 | 76 | kstring_t auth_hdr = { 0, 0, NULL }; |
49 | 76 | kstring_t requester_pays_hdr = { 0, 0, NULL }; |
50 | 76 | hFILE *fp = NULL; |
51 | | |
52 | | // GCS URL format is gs[+SCHEME]://BUCKET/PATH |
53 | | |
54 | 76 | if (gsurl[2] == '+') { |
55 | 0 | bucket = strchr(gsurl, ':') + 1; |
56 | 0 | kputsn(&gsurl[3], bucket - &gsurl[3], &url); |
57 | 0 | } |
58 | 76 | else { |
59 | 76 | kputs("https:", &url); |
60 | 76 | bucket = &gsurl[3]; |
61 | 76 | } |
62 | 94 | while (*bucket == '/') kputc(*bucket++, &url); |
63 | | |
64 | 76 | path = bucket + strcspn(bucket, "/?#"); |
65 | | |
66 | 76 | kputsn(bucket, path - bucket, &url); |
67 | 76 | if (strchr(mode, 'r')) kputs(".storage-download", &url); |
68 | 0 | else if (strchr(mode, 'w')) kputs(".storage-upload", &url); |
69 | 0 | else kputs(".storage", &url); |
70 | 76 | kputs(".googleapis.com", &url); |
71 | | |
72 | 76 | kputs(path, &url); |
73 | | |
74 | 76 | if (hts_verbose >= 8) |
75 | 0 | fprintf(stderr, "[M::gcs_open] rewrote URL as %s\n", url.s); |
76 | | |
77 | | // TODO Find the access token in a more standard way |
78 | 76 | access_token = getenv("GCS_OAUTH_TOKEN"); |
79 | | |
80 | 76 | if (access_token) { |
81 | 0 | kputs("Authorization: Bearer ", &auth_hdr); |
82 | 0 | kputs(access_token, &auth_hdr); |
83 | 0 | } |
84 | | |
85 | 76 | requester_pays_project = getenv("GCS_REQUESTER_PAYS_PROJECT"); |
86 | | |
87 | 76 | if (requester_pays_project) { |
88 | 0 | kputs("X-Goog-User-Project: ", &requester_pays_hdr); |
89 | 0 | kputs(requester_pays_project, &requester_pays_hdr); |
90 | 0 | } |
91 | | |
92 | 76 | if (argsp || mode_has_colon || auth_hdr.l > 0 || requester_pays_hdr.l > 0) { |
93 | 0 | if (! mode_has_colon) { |
94 | 0 | kputs(mode, &mode_colon); |
95 | 0 | kputc(':', &mode_colon); |
96 | 0 | mode = mode_colon.s; |
97 | 0 | } |
98 | |
|
99 | 0 | if (auth_hdr.l > 0 && requester_pays_hdr.l > 0) { |
100 | 0 | fp = hopen( |
101 | 0 | url.s, mode, "va_list", argsp, |
102 | 0 | "httphdr:l", |
103 | 0 | auth_hdr.s, |
104 | 0 | requester_pays_hdr.s, |
105 | 0 | NULL, |
106 | 0 | NULL |
107 | 0 | ); |
108 | |
|
109 | 0 | } |
110 | 0 | else { |
111 | 0 | fp = hopen(url.s, mode, "va_list", argsp, |
112 | 0 | "httphdr", (auth_hdr.l > 0)? auth_hdr.s : NULL, NULL); |
113 | 0 | } |
114 | 0 | } |
115 | 76 | else |
116 | 76 | fp = hopen(url.s, mode); |
117 | | |
118 | 76 | free(mode_colon.s); |
119 | 76 | free(url.s); |
120 | 76 | free(auth_hdr.s); |
121 | 76 | free(requester_pays_hdr.s); |
122 | 76 | return fp; |
123 | 76 | } |
124 | | |
125 | | static hFILE *gcs_open(const char *url, const char *mode) |
126 | 76 | { |
127 | 76 | return gcs_rewrite(url, mode, 0, NULL); |
128 | 76 | } |
129 | | |
130 | | static hFILE *gcs_vopen(const char *url, const char *mode_colon, va_list args0) |
131 | 0 | { |
132 | | // Need to use va_copy() as we can only take the address of an actual |
133 | | // va_list object, not that of a parameter as its type may have decayed. |
134 | 0 | va_list args; |
135 | 0 | va_copy(args, args0); |
136 | 0 | hFILE *fp = gcs_rewrite(url, mode_colon, 1, &args); |
137 | 0 | va_end(args); |
138 | 0 | return fp; |
139 | 0 | } |
140 | | |
141 | | int PLUGIN_GLOBAL(hfile_plugin_init,_gcs)(struct hFILE_plugin *self) |
142 | 1 | { |
143 | 1 | static const struct hFILE_scheme_handler handler = |
144 | 1 | { gcs_open, hfile_always_remote, "Google Cloud Storage", |
145 | 1 | 2000 + 50, gcs_vopen |
146 | 1 | }; |
147 | | |
148 | | #ifdef ENABLE_PLUGINS |
149 | | // Embed version string for examination via strings(1) or what(1) |
150 | | static const char id[] = "@(#)hfile_gcs plugin (htslib)\t" HTS_VERSION_TEXT; |
151 | | if (hts_verbose >= 9) |
152 | | fprintf(stderr, "[M::hfile_gcs.init] version %s\n", strchr(id, '\t')+1); |
153 | | #endif |
154 | | |
155 | 1 | self->name = "Google Cloud Storage"; |
156 | 1 | hfile_add_scheme_handler("gs", &handler); |
157 | 1 | hfile_add_scheme_handler("gs+http", &handler); |
158 | 1 | hfile_add_scheme_handler("gs+https", &handler); |
159 | 1 | return 0; |
160 | 1 | } |