/src/duckdb/src/main/extension/extension_install.cpp
Line | Count | Source |
1 | | #include "duckdb/common/exception/http_exception.hpp" |
2 | | #include "duckdb/common/gzip_file_system.hpp" |
3 | | #include "duckdb/common/http_util.hpp" |
4 | | #include "duckdb/common/local_file_system.hpp" |
5 | | #include "duckdb/main/database_file_opener.hpp" |
6 | | #include "duckdb/common/serializer/binary_serializer.hpp" |
7 | | #include "duckdb/common/string_util.hpp" |
8 | | #include "duckdb/common/types/uuid.hpp" |
9 | | #include "duckdb/main/client_data.hpp" |
10 | | #include "duckdb/main/extension_helper.hpp" |
11 | | #include "duckdb/main/extension_install_info.hpp" |
12 | | #include "duckdb/main/secret/secret.hpp" |
13 | | #include "duckdb/main/secret/secret_manager.hpp" |
14 | | #include "duckdb/main/settings.hpp" |
15 | | #include "duckdb/common/windows_undefs.hpp" |
16 | | |
17 | | #include <fstream> |
18 | | |
19 | | namespace duckdb { |
20 | | |
21 | | //===--------------------------------------------------------------------===// |
22 | | // Install Extension |
23 | | //===--------------------------------------------------------------------===// |
24 | 0 | const string ExtensionHelper::NormalizeVersionTag(const string &version_tag) { |
25 | 0 | if (!version_tag.empty() && version_tag[0] != 'v') { |
26 | 0 | return "v" + version_tag; |
27 | 0 | } |
28 | 0 | return version_tag; |
29 | 0 | } |
30 | | |
31 | 0 | bool ExtensionHelper::IsRelease(const string &version_tag) { |
32 | 0 | return !StringUtil::Contains(version_tag, "-dev"); |
33 | 0 | } |
34 | | |
35 | 0 | const string ExtensionHelper::GetVersionDirectoryName() { |
36 | | #ifdef DUCKDB_WASM_VERSION |
37 | | return DUCKDB_QUOTE_DEFINE(DUCKDB_WASM_VERSION); |
38 | | #endif |
39 | 0 | if (IsRelease(DuckDB::LibraryVersion())) { |
40 | 0 | return NormalizeVersionTag(DuckDB::LibraryVersion()); |
41 | 0 | } else { |
42 | 0 | return DuckDB::SourceID(); |
43 | 0 | } |
44 | 0 | } |
45 | | |
46 | 0 | const vector<string> ExtensionHelper::PathComponents() { |
47 | 0 | return vector<string> {GetVersionDirectoryName(), DuckDB::Platform()}; |
48 | 0 | } |
49 | | |
50 | 0 | string ExtensionHelper::ExtensionInstallDocumentationLink(const string &extension_name) { |
51 | 0 | auto components = PathComponents(); |
52 | |
|
53 | 0 | string link = "https://duckdb.org/docs/current/extensions/troubleshooting"; |
54 | |
|
55 | 0 | if (components.size() >= 2) { |
56 | 0 | link += "?version=" + components[0] + "&platform=" + components[1] + "&extension=" + extension_name; |
57 | 0 | } |
58 | |
|
59 | 0 | return link; |
60 | 0 | } |
61 | | |
62 | 0 | vector<duckdb::string> ExtensionHelper::DefaultExtensionFolders(FileSystem &fs) { |
63 | 0 | vector<duckdb::string> default_folders; |
64 | | // These fallbacks are necessary if the user doesn't use the CMake build. |
65 | | #ifndef DUCKDB_EXTENSION_DIRECTORIES |
66 | | #ifdef _WIN32 |
67 | | #define DUCKDB_EXTENSION_DIRECTORIES "~\\.duckdb\\extensions" |
68 | | #else |
69 | | #define DUCKDB_EXTENSION_DIRECTORIES "~/.duckdb/extensions" |
70 | | #endif |
71 | | #endif |
72 | 0 | string dirs_string(DUCKDB_EXTENSION_DIRECTORIES); |
73 | | |
74 | | // Skip if empty |
75 | 0 | if (dirs_string.empty()) { |
76 | 0 | return default_folders; |
77 | 0 | } |
78 | | |
79 | | // Split the string by separator |
80 | 0 | auto directories = StringUtil::Split(dirs_string, ';'); |
81 | |
|
82 | 0 | for (auto &dir : directories) { |
83 | | // Skip empty directories |
84 | 0 | if (dir.empty()) { |
85 | 0 | continue; |
86 | 0 | } |
87 | | |
88 | 0 | default_folders.push_back(dir); |
89 | 0 | } |
90 | |
|
91 | 0 | return default_folders; |
92 | 0 | } |
93 | | |
94 | 0 | vector<string> ExtensionHelper::GetExtensionDirectoryPath(ClientContext &context) { |
95 | 0 | auto &db = DatabaseInstance::GetDatabase(context); |
96 | 0 | auto &fs = FileSystem::GetFileSystem(context); |
97 | 0 | return GetExtensionDirectoryPath(db, fs); |
98 | 0 | } |
99 | | |
100 | 0 | vector<string> ExtensionHelper::GetExtensionDirectoryPath(DatabaseInstance &db, FileSystem &fs) { |
101 | 0 | vector<string> extension_directories; |
102 | 0 | auto &config = db.config; |
103 | |
|
104 | 0 | auto custom_extension_directory = Settings::Get<ExtensionDirectorySetting>(config); |
105 | 0 | if (!custom_extension_directory.empty()) { |
106 | 0 | extension_directories.push_back(custom_extension_directory); |
107 | 0 | } |
108 | |
|
109 | 0 | if (!config.options.extension_directories.empty()) { |
110 | | // Add all configured extension directories |
111 | 0 | for (const auto &dir : config.options.extension_directories) { |
112 | 0 | extension_directories.push_back(dir); |
113 | 0 | } |
114 | 0 | } |
115 | 0 | if (extension_directories.empty()) { |
116 | | // Add default extension directory if no custom directories configured |
117 | 0 | for (const auto &default_dir : ExtensionHelper::DefaultExtensionFolders(fs)) { |
118 | 0 | extension_directories.push_back(default_dir); |
119 | 0 | } |
120 | 0 | } |
121 | | |
122 | | // Process all directories with common path operations |
123 | 0 | auto path_components = PathComponents(); |
124 | 0 | for (auto &extension_directory : extension_directories) { |
125 | | // convert random separators to platform-canonic |
126 | 0 | extension_directory = fs.ConvertSeparators(extension_directory); |
127 | | // expand ~ in extension directory |
128 | 0 | extension_directory = fs.ExpandPath(extension_directory); |
129 | | |
130 | | // Add path components (version and platform) |
131 | 0 | for (auto &path_ele : path_components) { |
132 | 0 | extension_directory = fs.JoinPath(extension_directory, path_ele); |
133 | 0 | } |
134 | 0 | } |
135 | |
|
136 | 0 | return extension_directories; |
137 | 0 | } |
138 | | |
139 | 0 | string ExtensionHelper::ExtensionDirectory(DatabaseInstance &db, FileSystem &fs) { |
140 | | #ifdef WASM_LOADABLE_EXTENSIONS |
141 | | throw PermissionException("ExtensionDirectory functionality is not supported in duckdb-wasm"); |
142 | | #endif |
143 | 0 | auto extension_directories = GetExtensionDirectoryPath(db, fs); |
144 | | // TODO: This should never be the case given the implementation of GetExtensionDirectoryPath |
145 | | // should we still keep this check? |
146 | 0 | D_ASSERT(!extension_directories.empty()); |
147 | |
|
148 | 0 | string extension_directory = extension_directories[0]; // Use first/primary directory |
149 | 0 | { |
150 | 0 | if (!fs.DirectoryExists(extension_directory)) { |
151 | 0 | string home_directory = fs.GetHomeDirectory(); |
152 | 0 | if (extension_directory.rfind(home_directory, 0) == 0 && !fs.DirectoryExists(home_directory)) { |
153 | 0 | throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET " |
154 | 0 | "home_directory='/path/to/dir' option.", |
155 | 0 | home_directory); |
156 | 0 | } |
157 | 0 | fs.CreateDirectoriesRecursive(extension_directory); |
158 | 0 | } |
159 | 0 | } |
160 | 0 | D_ASSERT(fs.DirectoryExists(extension_directory)); |
161 | |
|
162 | 0 | return extension_directory; |
163 | 0 | } |
164 | | |
165 | 0 | string ExtensionHelper::ExtensionDirectory(ClientContext &context) { |
166 | 0 | auto &db = DatabaseInstance::GetDatabase(context); |
167 | 0 | auto &fs = FileSystem::GetFileSystem(context); |
168 | 0 | return ExtensionDirectory(db, fs); |
169 | 0 | } |
170 | | |
171 | 0 | bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &message) { |
172 | 0 | auto lowercase_extension_name = StringUtil::Lower(extension_name); |
173 | 0 | vector<string> candidates; |
174 | 0 | for (idx_t ext_count = ExtensionHelper::DefaultExtensionCount(), i = 0; i < ext_count; i++) { |
175 | 0 | candidates.emplace_back(ExtensionHelper::GetDefaultExtension(i).name); |
176 | 0 | } |
177 | 0 | for (idx_t ext_count = ExtensionHelper::ExtensionAliasCount(), i = 0; i < ext_count; i++) { |
178 | 0 | candidates.emplace_back(ExtensionHelper::GetInternalExtensionAlias(i).alias); |
179 | 0 | } |
180 | 0 | auto closest_extensions = StringUtil::TopNJaroWinkler(candidates, lowercase_extension_name); |
181 | 0 | message = StringUtil::CandidatesMessage(closest_extensions, "Candidate extensions"); |
182 | 0 | for (auto &closest : closest_extensions) { |
183 | 0 | if (closest == lowercase_extension_name) { |
184 | 0 | message = "Extension \"" + extension_name + "\" is an existing extension.\n"; |
185 | 0 | return true; |
186 | 0 | } |
187 | 0 | } |
188 | 0 | return false; |
189 | 0 | } |
190 | | |
191 | | unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DatabaseInstance &db, FileSystem &fs, |
192 | | const string &extension, |
193 | 0 | ExtensionInstallOptions &options) { |
194 | | #ifdef WASM_LOADABLE_EXTENSIONS |
195 | | // Install is currently a no-op |
196 | | return nullptr; |
197 | | #endif |
198 | 0 | string local_path = ExtensionDirectory(db, fs); |
199 | 0 | return InstallExtensionInternal(db, fs, local_path, extension, options); |
200 | 0 | } |
201 | | |
202 | | unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, |
203 | 0 | ExtensionInstallOptions &options) { |
204 | | #ifdef WASM_LOADABLE_EXTENSIONS |
205 | | // Install is currently a no-op |
206 | | return nullptr; |
207 | | #endif |
208 | 0 | auto &db = DatabaseInstance::GetDatabase(context); |
209 | 0 | auto &fs = FileSystem::GetFileSystem(context); |
210 | 0 | string local_path = ExtensionDirectory(context); |
211 | 0 | return InstallExtensionInternal(db, fs, local_path, extension, options, context); |
212 | 0 | } |
213 | | |
214 | 0 | static unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) { |
215 | 0 | auto source_file = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); |
216 | 0 | file_size = source_file->GetFileSize(); |
217 | 0 | auto in_buffer = make_unsafe_uniq_array<data_t>(file_size); |
218 | 0 | source_file->Read(QueryContext(), in_buffer.get(), file_size); |
219 | 0 | source_file->Close(); |
220 | 0 | return in_buffer; |
221 | 0 | } |
222 | | |
223 | | static void WriteExtensionFileToDisk(QueryContext &query_context, FileSystem &fs, const string &path, void *data, |
224 | 0 | idx_t data_size, DBConfig &config) { |
225 | 0 | if (!Settings::Get<AllowUnsignedExtensionsSetting>(config)) { |
226 | 0 | const bool signature_valid = ExtensionHelper::CheckExtensionBufferSignature( |
227 | 0 | static_cast<char *>(data), data_size, Settings::Get<AllowCommunityExtensionsSetting>(config)); |
228 | 0 | if (!signature_valid) { |
229 | 0 | throw IOException("Attempting to install an extension file that doesn't have a valid signature, see " |
230 | 0 | "https://duckdb.org/docs/current/operations_manual/securing_duckdb/securing_extensions"); |
231 | 0 | } |
232 | 0 | } |
233 | | |
234 | | // Now signature has been checked (if signature checking is enabled) |
235 | | |
236 | | // Open target_file, at this points ending with '.duckdb_extension' |
237 | 0 | auto target_file = |
238 | 0 | fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_APPEND | |
239 | 0 | FileFlags::FILE_FLAGS_FILE_CREATE_NEW | FileFlags::FILE_FLAGS_ENABLE_EXTENSION_INSTALL); |
240 | | // Write content to the file |
241 | 0 | target_file->Write(query_context, data, data_size); |
242 | |
|
243 | 0 | target_file->Close(); |
244 | 0 | target_file.reset(); |
245 | 0 | } |
246 | | |
247 | 0 | static void WriteExtensionMetadataFileToDisk(FileSystem &fs, const string &path, ExtensionInstallInfo &metadata) { |
248 | 0 | auto file_writer = BufferedFileWriter(fs, path); |
249 | 0 | BinarySerializer::Serialize(metadata, file_writer); |
250 | 0 | file_writer.Sync(); |
251 | 0 | } |
252 | | |
253 | | string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const DatabaseInstance> db, |
254 | 0 | const ExtensionRepository &repository, const string &version) { |
255 | 0 | string versioned_path; |
256 | 0 | if (!version.empty()) { |
257 | 0 | versioned_path = "/${NAME}/" + version + "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension"; |
258 | 0 | } else { |
259 | 0 | versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension"; |
260 | 0 | } |
261 | | #ifdef WASM_LOADABLE_EXTENSIONS |
262 | | versioned_path = versioned_path + ".wasm"; |
263 | | #else |
264 | 0 | versioned_path = versioned_path + CompressionExtensionFromType(FileCompressionType::GZIP); |
265 | 0 | #endif |
266 | 0 | string url_template = repository.path + versioned_path; |
267 | 0 | return url_template; |
268 | 0 | } |
269 | | |
270 | 0 | string ExtensionHelper::ExtensionFinalizeUrlTemplate(const string &url_template, const string &extension_name) { |
271 | 0 | auto url = StringUtil::Replace(url_template, "${REVISION}", GetVersionDirectoryName()); |
272 | 0 | url = StringUtil::Replace(url, "${PLATFORM}", DuckDB::Platform()); |
273 | 0 | url = StringUtil::Replace(url, "${NAME}", extension_name); |
274 | 0 | return url; |
275 | 0 | } |
276 | | |
277 | | static void CheckExtensionMetadataOnInstall(DatabaseInstance &db, void *in_buffer, idx_t file_size, |
278 | 0 | ExtensionInstallInfo &info, const string &extension_name) { |
279 | 0 | if (file_size < ParsedExtensionMetaData::FOOTER_SIZE) { |
280 | 0 | throw IOException("Failed to install '%s', file too small to be a valid DuckDB extension!", extension_name); |
281 | 0 | } |
282 | | |
283 | 0 | auto parsed_metadata = ExtensionHelper::ParseExtensionMetaData(static_cast<char *>(in_buffer) + |
284 | 0 | (file_size - ParsedExtensionMetaData::FOOTER_SIZE)); |
285 | |
|
286 | 0 | auto metadata_mismatch_error = parsed_metadata.GetInvalidMetadataError(); |
287 | |
|
288 | 0 | if (!metadata_mismatch_error.empty() && !Settings::Get<AllowExtensionsMetadataMismatchSetting>(db)) { |
289 | 0 | throw IOException("Failed to install '%s'\n%s", extension_name, metadata_mismatch_error); |
290 | 0 | } |
291 | | |
292 | 0 | info.version = parsed_metadata.extension_version; |
293 | 0 | } |
294 | | |
295 | | // Note: since this method is not atomic, this can fail in different ways, that should all be handled properly by |
296 | | // DuckDB: |
297 | | // 1. Crash after extension removal: extension is now uninstalled, metadata file still present |
298 | | // 2. Crash after metadata removal: extension is now uninstalled, extension dir is clean |
299 | | // 3. Crash after extension move: extension is now uninstalled, new metadata file present |
300 | | static void WriteExtensionFiles(QueryContext &query_context, FileSystem &fs, const string &temp_path, |
301 | | const string &local_extension_path, void *in_buffer, idx_t file_size, |
302 | 0 | ExtensionInstallInfo &info, DBConfig &config) { |
303 | | // temp_path ends with '.duckdb_extension' |
304 | 0 | if (!StringUtil::EndsWith(temp_path, ".duckdb_extension")) { |
305 | 0 | throw InternalException("Extension install temp_path of '%s' is not valid, should end in '.duckdb_extension'", |
306 | 0 | temp_path); |
307 | 0 | } |
308 | | // local_extension_path ends with '.duckdb_extension', and given it will be written only after signature checks, |
309 | | // it's now loadable |
310 | 0 | if (!StringUtil::EndsWith(local_extension_path, ".duckdb_extension")) { |
311 | 0 | throw InternalException("Extension install local_extension_path of '%s' is not valid, should end in " |
312 | 0 | "'.duckdb_extension'", |
313 | 0 | temp_path); |
314 | 0 | } |
315 | | |
316 | | // Write extension to tmp file |
317 | 0 | WriteExtensionFileToDisk(query_context, fs, temp_path, in_buffer, file_size, config); |
318 | | // When this exit, signature has already being checked (if enabled by config) |
319 | | |
320 | | // Write metadata to tmp file |
321 | 0 | auto metadata_tmp_path = temp_path + ".info"; |
322 | 0 | auto metadata_file_path = local_extension_path + ".info"; |
323 | 0 | WriteExtensionMetadataFileToDisk(fs, metadata_tmp_path, info); |
324 | |
|
325 | 0 | fs.MoveFile(metadata_tmp_path, metadata_file_path); |
326 | 0 | fs.MoveFile(temp_path, local_extension_path); |
327 | 0 | } |
328 | | |
329 | | // Install an extension using a filesystem |
330 | | static unique_ptr<ExtensionInstallInfo> DirectInstallExtension(DatabaseInstance &db, FileSystem &fs, const string &path, |
331 | | const string &temp_path, const string &extension_name, |
332 | | const string &local_extension_path, |
333 | | ExtensionInstallOptions &options, |
334 | 0 | optional_ptr<ClientContext> context) { |
335 | 0 | string extension; |
336 | 0 | string file; |
337 | 0 | if (fs.IsRemoteFile(path, extension)) { |
338 | 0 | file = path; |
339 | | // Try autoloading httpfs for loading extensions over https |
340 | 0 | if (context) { |
341 | 0 | auto &db = DatabaseInstance::GetDatabase(*context); |
342 | 0 | if (extension == "httpfs" && !db.ExtensionIsLoaded("httpfs") && |
343 | 0 | Settings::Get<AutoloadKnownExtensionsSetting>(*context)) { |
344 | 0 | ExtensionHelper::AutoLoadExtension(*context, "httpfs"); |
345 | 0 | } |
346 | 0 | } |
347 | 0 | } else { |
348 | 0 | file = fs.ConvertSeparators(path); |
349 | 0 | } |
350 | | |
351 | | // Check if file exists |
352 | 0 | bool exists = fs.FileExists(file); |
353 | | |
354 | | // Recheck without .gz |
355 | 0 | if (!exists && StringUtil::EndsWith(file, CompressionExtensionFromType(FileCompressionType::GZIP))) { |
356 | 0 | file = file.substr(0, file.size() - 3); |
357 | 0 | exists = fs.FileExists(file); |
358 | 0 | } |
359 | | |
360 | | // Throw error on failure |
361 | 0 | if (!exists) { |
362 | 0 | if (!fs.IsRemoteFile(file)) { |
363 | 0 | throw IOException("Failed to install local extension \"%s\", no access to the file at PATH \"%s\"\n", |
364 | 0 | extension_name, file); |
365 | 0 | } |
366 | 0 | if (StringUtil::StartsWith(file, "https://")) { |
367 | 0 | throw IOException("Failed to install remote extension \"%s\" from url \"%s\"", extension_name, file); |
368 | 0 | } |
369 | 0 | } |
370 | | |
371 | 0 | idx_t file_size; |
372 | 0 | auto in_buffer = ReadExtensionFileFromDisk(fs, file, file_size); |
373 | |
|
374 | 0 | ExtensionInstallInfo info; |
375 | |
|
376 | 0 | string decompressed_data; |
377 | 0 | void *extension_decompressed; |
378 | 0 | idx_t extension_decompressed_size; |
379 | |
|
380 | 0 | if (GZipFileSystem::CheckIsZip(const_char_ptr_cast(in_buffer.get()), file_size)) { |
381 | 0 | decompressed_data = GZipFileSystem::UncompressGZIPString(const_char_ptr_cast(in_buffer.get()), file_size); |
382 | 0 | extension_decompressed = (void *)decompressed_data.data(); |
383 | 0 | extension_decompressed_size = decompressed_data.size(); |
384 | 0 | } else { |
385 | 0 | extension_decompressed = (void *)in_buffer.get(); |
386 | 0 | extension_decompressed_size = file_size; |
387 | 0 | } |
388 | |
|
389 | 0 | CheckExtensionMetadataOnInstall(db, extension_decompressed, extension_decompressed_size, info, extension_name); |
390 | |
|
391 | 0 | if (!options.repository) { |
392 | 0 | info.mode = ExtensionInstallMode::CUSTOM_PATH; |
393 | 0 | info.full_path = file; |
394 | 0 | } else { |
395 | 0 | info.mode = ExtensionInstallMode::REPOSITORY; |
396 | 0 | info.full_path = file; |
397 | 0 | info.repository_url = options.repository->path; |
398 | 0 | } |
399 | |
|
400 | 0 | QueryContext query_context(context); |
401 | 0 | WriteExtensionFiles(query_context, fs, temp_path, local_extension_path, extension_decompressed, |
402 | 0 | extension_decompressed_size, info, db.config); |
403 | |
|
404 | 0 | return make_uniq<ExtensionInstallInfo>(info); |
405 | 0 | } |
406 | | |
407 | | #ifndef DUCKDB_DISABLE_EXTENSION_LOAD |
408 | | static unique_ptr<ExtensionInstallInfo> InstallFromHttpUrl(DatabaseInstance &db, const string &url, |
409 | | const string &extension_name, const string &temp_path, |
410 | | const string &local_extension_path, |
411 | | ExtensionInstallOptions &options, |
412 | 0 | optional_ptr<ClientContext> context) { |
413 | 0 | unique_ptr<ExtensionInstallInfo> install_info; |
414 | 0 | { |
415 | 0 | auto &fs = FileSystem::GetLocal(db); |
416 | 0 | if (fs.FileExists(local_extension_path + ".info")) { |
417 | 0 | try { |
418 | 0 | install_info = |
419 | 0 | ExtensionInstallInfo::TryReadInfoFile(fs, local_extension_path + ".info", extension_name); |
420 | 0 | } catch (...) { |
421 | 0 | if (!options.force_install) { |
422 | | // We are going to rewrite the file anyhow, so this is fine |
423 | 0 | throw; |
424 | 0 | } |
425 | 0 | } |
426 | 0 | } |
427 | 0 | } |
428 | | |
429 | 0 | HTTPHeaders headers(db); |
430 | 0 | if (options.use_etags && install_info && !install_info->etag.empty()) { |
431 | 0 | headers.Insert("If-None-Match", StringUtil::Format("%s", install_info->etag)); |
432 | 0 | } |
433 | |
|
434 | 0 | auto &http_util = HTTPUtil::Get(db); |
435 | 0 | unique_ptr<HTTPParams> params; |
436 | 0 | if (context) { |
437 | 0 | params = http_util.InitializeParameters(*context, url); |
438 | 0 | } else { |
439 | 0 | params = http_util.InitializeParameters(db, url); |
440 | 0 | } |
441 | | |
442 | | // Unclear what's peculiar about extension install flow, but those two parameters are needed |
443 | | // to avoid lengthy retry on 304 |
444 | 0 | params->follow_location = false; |
445 | 0 | params->keep_alive = false; |
446 | |
|
447 | 0 | GetRequestInfo get_request(url, headers, *params, nullptr, nullptr); |
448 | 0 | get_request.try_request = true; |
449 | |
|
450 | 0 | auto response = http_util.Request(get_request); |
451 | 0 | if (!response->Success()) { |
452 | | // if we should not retry or exceeded the number of retries - bubble up the error |
453 | 0 | string message; |
454 | 0 | ExtensionHelper::CreateSuggestions(extension_name, message); |
455 | |
|
456 | 0 | auto documentation_link = ExtensionHelper::ExtensionInstallDocumentationLink(extension_name); |
457 | 0 | if (!documentation_link.empty()) { |
458 | 0 | message += "\nFor more info, visit " + documentation_link; |
459 | 0 | } |
460 | 0 | if (response->HasRequestError()) { |
461 | | // request error - this means something went wrong performing the request |
462 | 0 | throw IOException("Failed to download extension \"%s\" at URL \"%s\"\n%s (ERROR %s)", extension_name, url, |
463 | 0 | message, response->GetRequestError()); |
464 | 0 | } |
465 | | // if this was not a request error this means the server responded - report the response status and response |
466 | 0 | throw HTTPException(*response, "Failed to download extension \"%s\" at URL \"%s\" (HTTP %n)\n%s", |
467 | 0 | extension_name, url, int(response->status), message); |
468 | 0 | } |
469 | 0 | if (response->status == HTTPStatusCode::NotModified_304 && install_info) { |
470 | 0 | return install_info; |
471 | 0 | } |
472 | | |
473 | 0 | string decompressed_body; |
474 | 0 | void *extension_data; |
475 | 0 | idx_t extension_size; |
476 | |
|
477 | 0 | if (GZipFileSystem::CheckIsZip(response->body.data(), response->body.size())) { |
478 | 0 | decompressed_body = GZipFileSystem::UncompressGZIPString(response->body); |
479 | 0 | extension_data = (void *)decompressed_body.data(); |
480 | 0 | extension_size = decompressed_body.size(); |
481 | 0 | } else { |
482 | 0 | extension_data = (void *)response->body.data(); |
483 | 0 | extension_size = response->body.size(); |
484 | 0 | } |
485 | |
|
486 | 0 | ExtensionInstallInfo info; |
487 | 0 | CheckExtensionMetadataOnInstall(db, extension_data, extension_size, info, extension_name); |
488 | 0 | if (response->HasHeader("ETag")) { |
489 | 0 | info.etag = response->GetHeaderValue("ETag"); |
490 | 0 | } |
491 | |
|
492 | 0 | if (options.repository) { |
493 | 0 | info.mode = ExtensionInstallMode::REPOSITORY; |
494 | 0 | info.full_path = url; |
495 | 0 | info.repository_url = options.repository->path; |
496 | 0 | } else { |
497 | 0 | info.mode = ExtensionInstallMode::CUSTOM_PATH; |
498 | 0 | info.full_path = url; |
499 | 0 | } |
500 | |
|
501 | 0 | QueryContext query_context(context); |
502 | 0 | auto fs = FileSystem::CreateLocal(); |
503 | 0 | WriteExtensionFiles(query_context, *fs, temp_path, local_extension_path, extension_data, extension_size, info, |
504 | 0 | db.config); |
505 | |
|
506 | 0 | return make_uniq<ExtensionInstallInfo>(info); |
507 | 0 | } |
508 | | |
509 | | // Install an extension using a hand-rolled http request |
510 | | static unique_ptr<ExtensionInstallInfo> InstallFromRepository(DatabaseInstance &db, FileSystem &fs, const string &url, |
511 | | const string &extension_name, const string &temp_path, |
512 | | const string &local_extension_path, |
513 | | ExtensionInstallOptions &options, |
514 | 0 | optional_ptr<ClientContext> context) { |
515 | 0 | string url_template = ExtensionHelper::ExtensionUrlTemplate(db, *options.repository, options.version); |
516 | 0 | string generated_url = ExtensionHelper::ExtensionFinalizeUrlTemplate(url_template, extension_name); |
517 | | |
518 | | // Special handling for http repository: avoid using regular filesystem (note: the filesystem is not used here) |
519 | 0 | if (HTTPUtil::IsHTTPProtocol(options.repository->path)) { |
520 | 0 | if (db.ExtensionIsLoaded("httpfs")) { |
521 | 0 | HTTPUtil::BumpToSecureProtocol(generated_url); |
522 | 0 | } |
523 | 0 | return InstallFromHttpUrl(db, generated_url, extension_name, temp_path, local_extension_path, options, context); |
524 | 0 | } |
525 | | |
526 | | // Default case, let the FileSystem figure it out |
527 | 0 | return DirectInstallExtension(db, fs, generated_url, temp_path, extension_name, local_extension_path, options, |
528 | 0 | context); |
529 | 0 | } |
530 | | |
531 | | static void ThrowErrorOnMismatchingExtensionOrigin(FileSystem &fs, const string &local_extension_path, |
532 | | const string &extension_name, const string &extension, |
533 | 0 | optional_ptr<ExtensionRepository> repository) { |
534 | 0 | auto install_info = ExtensionInstallInfo::TryReadInfoFile(fs, local_extension_path + ".info", extension_name); |
535 | |
|
536 | 0 | string format_string = "Installing extension '%s' failed. The extension is already installed " |
537 | 0 | "but the origin is different.\n" |
538 | 0 | "Currently installed extension is from %s '%s', while the extension to be " |
539 | 0 | "installed is from %s '%s'.\n" |
540 | 0 | "To solve this rerun this command with `FORCE INSTALL`"; |
541 | 0 | string repo = "repository"; |
542 | 0 | string custom_path = "custom_path"; |
543 | |
|
544 | 0 | if (install_info) { |
545 | 0 | if (install_info->mode == ExtensionInstallMode::REPOSITORY && repository && |
546 | 0 | install_info->repository_url != repository->path) { |
547 | 0 | throw InvalidInputException(format_string, extension_name, repo, install_info->repository_url, repo, |
548 | 0 | repository->path); |
549 | 0 | } |
550 | 0 | if (install_info->mode == ExtensionInstallMode::REPOSITORY && ExtensionHelper::IsFullPath(extension)) { |
551 | 0 | throw InvalidInputException(format_string, extension_name, repo, install_info->repository_url, custom_path, |
552 | 0 | extension); |
553 | 0 | } |
554 | 0 | } |
555 | 0 | } |
556 | | #endif // DUCKDB_DISABLE_EXTENSION_LOAD |
557 | | |
558 | | unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtensionInternal(DatabaseInstance &db, FileSystem &fs, |
559 | | const string &local_path, |
560 | | const string &extension, |
561 | | ExtensionInstallOptions &options, |
562 | 0 | optional_ptr<ClientContext> context) { |
563 | | #ifdef DUCKDB_DISABLE_EXTENSION_LOAD |
564 | | throw PermissionException("Installing external extensions is disabled through a compile time flag"); |
565 | | #else |
566 | |
|
567 | 0 | auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension)); |
568 | 0 | string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension"); |
569 | 0 | string temp_path = |
570 | 0 | local_extension_path + ".tmp-" + UUID::ToString(UUID::GenerateRandomUUID()) + ".duckdb_extension"; |
571 | |
|
572 | 0 | if (fs.FileExists(local_extension_path) && !options.force_install) { |
573 | | // File exists: throw error if origin mismatches |
574 | 0 | if (options.throw_on_origin_mismatch && !Settings::Get<AllowExtensionsMetadataMismatchSetting>(db) && |
575 | 0 | fs.FileExists(local_extension_path + ".info")) { |
576 | 0 | ThrowErrorOnMismatchingExtensionOrigin(fs, local_extension_path, extension_name, extension, |
577 | 0 | options.repository); |
578 | 0 | } |
579 | | |
580 | | // File exists, but that's okay, install is now a NOP |
581 | 0 | return nullptr; |
582 | 0 | } |
583 | | |
584 | 0 | fs.TryRemoveFile(temp_path); |
585 | |
|
586 | 0 | if (ExtensionHelper::IsFullPath(extension) && options.repository) { |
587 | 0 | throw InvalidInputException("Cannot pass both a repository and a full path url"); |
588 | 0 | } |
589 | | |
590 | | // Resolve default repository if there is none set |
591 | 0 | ExtensionRepository resolved_repository; |
592 | 0 | if (!ExtensionHelper::IsFullPath(extension) && !options.repository) { |
593 | 0 | resolved_repository = ExtensionRepository::GetDefaultRepository(db.config); |
594 | 0 | options.repository = resolved_repository; |
595 | 0 | } |
596 | | |
597 | | // Install extension from local, direct url |
598 | 0 | if (ExtensionHelper::IsFullPath(extension) && !FileSystem::IsRemoteFile(extension)) { |
599 | 0 | auto &local_fs = FileSystem::GetLocal(db); |
600 | 0 | return DirectInstallExtension(db, local_fs, extension, temp_path, extension, local_extension_path, options, |
601 | 0 | context); |
602 | 0 | } |
603 | | |
604 | | // Install extension from local url based on a repository (Note that this will install it as a local file) |
605 | 0 | if (options.repository && !FileSystem::IsRemoteFile(options.repository->path)) { |
606 | 0 | auto &local_fs = FileSystem::GetLocal(db); |
607 | 0 | return InstallFromRepository(db, local_fs, extension, extension_name, temp_path, local_extension_path, options, |
608 | 0 | context); |
609 | 0 | } |
610 | | |
611 | | #ifdef DISABLE_DUCKDB_REMOTE_INSTALL |
612 | | throw BinderException("Remote extension installation is disabled through configuration"); |
613 | | #else |
614 | | |
615 | | // Full path direct installation |
616 | 0 | if (IsFullPath(extension)) { |
617 | 0 | if (StringUtil::StartsWith(extension, "http://")) { |
618 | | // HTTP takes separate path to avoid dependency on httpfs extension |
619 | 0 | return InstallFromHttpUrl(db, extension, extension_name, temp_path, local_extension_path, options, context); |
620 | 0 | } |
621 | | |
622 | | // Direct installation from local or remote path |
623 | 0 | return DirectInstallExtension(db, fs, extension, temp_path, extension, local_extension_path, options, context); |
624 | 0 | } |
625 | | |
626 | | // Repository installation |
627 | 0 | return InstallFromRepository(db, fs, extension, extension_name, temp_path, local_extension_path, options, context); |
628 | 0 | #endif |
629 | 0 | #endif |
630 | 0 | } |
631 | | |
632 | | } // namespace duckdb |