/src/gdal/port/cpl_google_cloud.cpp

Source
/**********************************************************************
 * Project:  CPL - Common Portability Library
 * Purpose:  Google Cloud Storage routines
 * Author:   Even Rouault <even.rouault at spatialys.com>
 *
 **********************************************************************
 * Copyright (c) 2017, Even Rouault <even.rouault at spatialys.com>
 *
 * SPDX-License-Identifier: MIT
 ****************************************************************************/

#include "cpl_google_cloud.h"
#include "cpl_vsi_error.h"
#include "cpl_sha1.h"
#include "cpl_sha256.h"
#include "cpl_time.h"
#include "cpl_http.h"
#include "cpl_mem_cache.h"
#include "cpl_aws.h"
#include "cpl_json.h"

#include <mutex>
#include <utility>

#ifdef HAVE_CURL

static bool bFirstTimeForDebugMessage = true;

struct GOA2ManagerCache
{
    struct ManagerWithMutex
    {
        std::mutex oMutex{};
        GOA2Manager oManager{};

        explicit ManagerWithMutex(const GOA2Manager &oManagerIn)
            : oManager(oManagerIn)
        {
        }
    };

    std::mutex oMutexGOA2ManagerCache{};
    lru11::Cache<std::string, std::shared_ptr<ManagerWithMutex>>
        oGOA2ManagerCache{};

    std::string GetBearer(const GOA2Manager &oManager)
    {
        const std::string osKey(oManager.GetKey());
        std::shared_ptr<ManagerWithMutex> poSharedManager;
        {
            std::lock_guard oLock(oMutexGOA2ManagerCache);
            if (!oGOA2ManagerCache.tryGet(osKey, poSharedManager))
            {
                poSharedManager = std::make_shared<ManagerWithMutex>(oManager);
                oGOA2ManagerCache.insert(osKey, poSharedManager);
            }
        }
        {
            std::lock_guard oLock(poSharedManager->oMutex);
            const char *pszBearer = poSharedManager->oManager.GetBearer();
            return std::string(pszBearer ? pszBearer : "");
        }
    }

    void clear()
    {
        std::lock_guard oLock(oMutexGOA2ManagerCache);
        oGOA2ManagerCache.clear();
    }

    static GOA2ManagerCache &GetSingleton()
    {
        static GOA2ManagerCache goGOA2ManagerCache;
        return goGOA2ManagerCache;
    }
};

/************************************************************************/
/*                   CPLIsMachineForSureGCEInstance()                   */
/************************************************************************/

/** Returns whether the current machine is surely a Google Compute Engine
 * instance.
 *
 * This does a very quick check without network access.
 * Note: only works for Linux GCE instances.
 *
 * Also detects Google Cloud Run services, jobs, and worker pools on all
 * platforms.
 *
 * @return true if the current machine is surely a GCE instance or Cloud Run.
 */
bool CPLIsMachineForSureGCEInstance()
{
    if (CPLTestBool(CPLGetConfigOption("CPL_MACHINE_IS_GCE", "NO")))
    {
        return true;
    }

    // Check for Google Cloud Run environment
    if (CPLGetConfigOption("CLOUD_RUN_TIMEOUT_SECONDS", nullptr) != nullptr ||
        CPLGetConfigOption("CLOUD_RUN_JOB", nullptr) != nullptr ||
        CPLGetConfigOption("CLOUD_RUN_WORKER_POOL", nullptr) != nullptr)
    {
        return true;
    }

#ifdef __linux
    // If /sys/class/dmi/id/product_name exists, it contains "Google Compute
    // Engine"
    bool bIsGCEInstance = false;
    if (CPLTestBool(CPLGetConfigOption("CPL_GCE_CHECK_LOCAL_FILES", "YES")))
    {
        static bool bIsGCEInstanceStatic = []()
        {
            bool bIsGCE = false;
            VSILFILE *fp = VSIFOpenL("/sys/class/dmi/id/product_name", "rb");
            if (fp)
            {
                const char *pszLine = CPLReadLineL(fp);
                bIsGCE =
                    pszLine && STARTS_WITH_CI(pszLine, "Google Compute Engine");
                VSIFCloseL(fp);
            }
            return bIsGCE;
        }();
        bIsGCEInstance = bIsGCEInstanceStatic;
    }
    return bIsGCEInstance;
#else
    return false;
#endif
}

/************************************************************************/
/*                 CPLIsMachinePotentiallyGCEInstance()                 */
/************************************************************************/

/** Returns whether the current machine is potentially a Google Compute Engine
 * instance.
 *
 * This does a very quick check without network access. To confirm if the
 * machine is effectively a GCE instance, metadata.google.internal must be
 * queried.
 *
 * Also detects Google Cloud Run services, jobs, and worker pools.
 *
 * @return true if the current machine is potentially a GCE instance or Cloud
 * Run.
 */
bool CPLIsMachinePotentiallyGCEInstance()
{
    // Check for Google Cloud Run environment first (platform-independent)
    // This must be done before platform-specific checks to ensure Cloud Run
    // detection works on all platforms.
    if (CPLGetConfigOption("CLOUD_RUN_TIMEOUT_SECONDS", nullptr) != nullptr ||
        CPLGetConfigOption("CLOUD_RUN_JOB", nullptr) != nullptr ||
        CPLGetConfigOption("CLOUD_RUN_WORKER_POOL", nullptr) != nullptr)
    {
        return true;
    }

#ifdef __linux
    bool bIsMachinePotentialGCEInstance = true;
    if (CPLTestBool(CPLGetConfigOption("CPL_GCE_CHECK_LOCAL_FILES", "YES")))
    {
        bIsMachinePotentialGCEInstance = CPLIsMachineForSureGCEInstance();
    }
    return bIsMachinePotentialGCEInstance;
#elif defined(_WIN32)
    // We might add later a way of detecting if we run on GCE using WMI
    // See https://cloud.google.com/compute/docs/instances/managing-instances
    // For now, unconditionally try
    return true;
#else
    // At time of writing GCE instances can be only Linux or Windows
    return false;
#endif
}

//! @cond Doxygen_Suppress

/************************************************************************/
/*                            GetGSHeaders()                            */
/************************************************************************/

static struct curl_slist *GetGSHeaders(const std::string &osPathForOption,
                                       const std::string &osVerb,
                                       struct curl_slist *psHeaders,
                                       const std::string &osCanonicalResource,
                                       const std::string &osSecretAccessKey,
                                       const std::string &osAccessKeyId)
{
    if (osSecretAccessKey.empty())
    {
        // GS_NO_SIGN_REQUEST=YES case
        return psHeaders;
    }

    std::string osDate = VSIGetPathSpecificOption(osPathForOption.c_str(),
                                                  "CPL_GS_TIMESTAMP", "");
    if (osDate.empty())
    {
        osDate = IVSIS3LikeHandleHelper::GetRFC822DateTime();
    }

    std::map<std::string, std::string> oSortedMapHeaders;
    std::string osCanonicalizedHeaders(
        IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
            oSortedMapHeaders, psHeaders, "x-goog-"));

    // See https://cloud.google.com/storage/docs/migrating
    std::string osStringToSign;
    osStringToSign += osVerb + "\n";
    osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-MD5") + "\n";
    osStringToSign += CPLAWSGetHeaderVal(psHeaders, "Content-Type") + "\n";
    osStringToSign += osDate + "\n";
    osStringToSign += osCanonicalizedHeaders;
    osStringToSign += osCanonicalResource;
#ifdef DEBUG_VERBOSE
    CPLDebug("GS", "osStringToSign = %s", osStringToSign.c_str());
#endif

    GByte abySignature[CPL_SHA1_HASH_SIZE] = {};
    CPL_HMAC_SHA1(osSecretAccessKey.c_str(), osSecretAccessKey.size(),
                  osStringToSign.c_str(), osStringToSign.size(), abySignature);

    char *pszBase64 = CPLBase64Encode(sizeof(abySignature), abySignature);
    std::string osAuthorization("GOOG1 ");
    osAuthorization += osAccessKeyId;
    osAuthorization += ":";
    osAuthorization += pszBase64;
    CPLFree(pszBase64);

    psHeaders =
        curl_slist_append(psHeaders, CPLSPrintf("Date: %s", osDate.c_str()));
    psHeaders = curl_slist_append(
        psHeaders, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));

    return psHeaders;
}

/************************************************************************/
/*                         VSIGSHandleHelper()                          */
/************************************************************************/
VSIGSHandleHelper::VSIGSHandleHelper(const std::string &osEndpoint,
                                     const std::string &osBucketObjectKey,
                                     const std::string &osSecretAccessKey,
                                     const std::string &osAccessKeyId,
                                     bool bUseAuthenticationHeader,
                                     const GOA2Manager &oManager,
                                     const std::string &osUserProject)
    : m_osURL(osEndpoint + CPLAWSURLEncode(osBucketObjectKey, false)),
      m_osEndpoint(osEndpoint), m_osBucketObjectKey(osBucketObjectKey),
      m_osSecretAccessKey(osSecretAccessKey), m_osAccessKeyId(osAccessKeyId),
      m_bUseAuthenticationHeader(bUseAuthenticationHeader),
      m_oManager(oManager), m_osUserProject(osUserProject)
{
    if (m_osBucketObjectKey.find('/') == std::string::npos)
        m_osURL += "/";
}

/************************************************************************/
/*                         ~VSIGSHandleHelper()                         */
/************************************************************************/

VSIGSHandleHelper::~VSIGSHandleHelper()
{
}

/************************************************************************/
/*                 GetConfigurationFromAWSConfigFiles()                 */
/************************************************************************/

bool VSIGSHandleHelper::GetConfigurationFromConfigFile(
    std::string &osSecretAccessKey, std::string &osAccessKeyId,
    std::string &osOAuth2RefreshToken, std::string &osOAuth2ClientId,
    std::string &osOAuth2ClientSecret, std::string &osCredentials)
{
#ifdef _WIN32
    const char *pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
    constexpr char SEP_STRING[] = "\\";
#else
    const char *pszHome = CPLGetConfigOption("HOME", nullptr);
    constexpr char SEP_STRING[] = "/";
#endif

    // GDAL specific config option (mostly for testing purpose, but also
    // used in production in some cases)
    const char *pszCredentials =
        CPLGetConfigOption("CPL_GS_CREDENTIALS_FILE", nullptr);
    if (pszCredentials)
    {
        osCredentials = pszCredentials;
    }
    else
    {
        osCredentials = pszHome ? pszHome : "";
        osCredentials += SEP_STRING;
        osCredentials += ".boto";
    }

    VSILFILE *fp = VSIFOpenL(osCredentials.c_str(), "rb");
    if (fp != nullptr)
    {
        const char *pszLine;
        bool bInCredentials = false;
        bool bInOAuth2 = false;
        while ((pszLine = CPLReadLineL(fp)) != nullptr)
        {
            if (pszLine[0] == '[')
            {
                bInCredentials = false;
                bInOAuth2 = false;

                if (std::string(pszLine) == "[Credentials]")
                    bInCredentials = true;
                else if (std::string(pszLine) == "[OAuth2]")
                    bInOAuth2 = true;
            }
            else if (bInCredentials)
            {
                char *pszKey = nullptr;
                const char *pszValue = CPLParseNameValue(pszLine, &pszKey);
                if (pszKey && pszValue)
                {
                    if (EQUAL(pszKey, "gs_access_key_id"))
                        osAccessKeyId = CPLString(pszValue).Trim();
                    else if (EQUAL(pszKey, "gs_secret_access_key"))
                        osSecretAccessKey = CPLString(pszValue).Trim();
                    else if (EQUAL(pszKey, "gs_oauth2_refresh_token"))
                        osOAuth2RefreshToken = CPLString(pszValue).Trim();
                }
                CPLFree(pszKey);
            }
            else if (bInOAuth2)
            {
                char *pszKey = nullptr;
                const char *pszValue = CPLParseNameValue(pszLine, &pszKey);
                if (pszKey && pszValue)
                {
                    if (EQUAL(pszKey, "client_id"))
                        osOAuth2ClientId = CPLString(pszValue).Trim();
                    else if (EQUAL(pszKey, "client_secret"))
                        osOAuth2ClientSecret = CPLString(pszValue).Trim();
                }
                CPLFree(pszKey);
            }
        }
        VSIFCloseL(fp);
    }

    return (!osAccessKeyId.empty() && !osSecretAccessKey.empty()) ||
           !osOAuth2RefreshToken.empty();
}

/************************************************************************/
/*                          GetConfiguration()                          */
/************************************************************************/

bool VSIGSHandleHelper::GetConfiguration(const std::string &osPathForOption,
                                         CSLConstList papszOptions,
                                         std::string &osSecretAccessKey,
                                         std::string &osAccessKeyId,
                                         bool &bUseAuthenticationHeader,
                                         GOA2Manager &oManager)
{
    osSecretAccessKey.clear();
    osAccessKeyId.clear();
    bUseAuthenticationHeader = false;

    if (CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(),
                                             "GS_NO_SIGN_REQUEST", "NO")))
    {
        return true;
    }

    osSecretAccessKey = VSIGetPathSpecificOption(osPathForOption.c_str(),
                                                 "GS_SECRET_ACCESS_KEY", "");
    if (!osSecretAccessKey.empty())
    {
        osAccessKeyId = VSIGetPathSpecificOption(osPathForOption.c_str(),
                                                 "GS_ACCESS_KEY_ID", "");
        if (osAccessKeyId.empty())
        {
            VSIError(VSIE_InvalidCredentials,
                     "GS_ACCESS_KEY_ID configuration option not defined");
            bFirstTimeForDebugMessage = false;
            return false;
        }

        if (bFirstTimeForDebugMessage)
        {
            CPLDebug("GS", "Using GS_SECRET_ACCESS_KEY and "
                           "GS_ACCESS_KEY_ID configuration options");
        }
        bFirstTimeForDebugMessage = false;
        return true;
    }

    const std::string osHeaderFile = VSIGetPathSpecificOption(
        osPathForOption.c_str(), "GDAL_HTTP_HEADER_FILE", "");
    bool bMayWarnDidNotFindAuth = false;
    if (!osHeaderFile.empty())
    {
        bool bFoundAuth = false;
        VSILFILE *fp = nullptr;
        // Do not allow /vsicurl/ access from /vsicurl because of
        // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
        // cause use of memory after free
        if (strstr(osHeaderFile.c_str(), "/vsicurl/") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsicurl?") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsis3/") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsigs/") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsiaz/") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsioss/") == nullptr &&
            strstr(osHeaderFile.c_str(), "/vsiswift/") == nullptr)
        {
            fp = VSIFOpenL(osHeaderFile.c_str(), "rb");
        }
        if (fp == nullptr)
        {
            CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s",
                     osHeaderFile.c_str());
        }
        else
        {
            const char *pszLine = nullptr;
            while ((pszLine = CPLReadLineL(fp)) != nullptr)
            {
                if (STARTS_WITH_CI(pszLine, "Authorization:"))
                {
                    bFoundAuth = true;
                    break;
                }
            }
            VSIFCloseL(fp);
            if (!bFoundAuth)
                bMayWarnDidNotFindAuth = true;
        }
        if (bFoundAuth)
        {
            if (bFirstTimeForDebugMessage)
            {
                CPLDebug("GS", "Using GDAL_HTTP_HEADER_FILE=%s",
                         osHeaderFile.c_str());
            }
            bFirstTimeForDebugMessage = false;
            bUseAuthenticationHeader = true;
            return true;
        }
    }

    const char *pszHeaders = VSIGetPathSpecificOption(
        osPathForOption.c_str(), "GDAL_HTTP_HEADERS", nullptr);
    if (pszHeaders && strstr(pszHeaders, "Authorization:") != nullptr)
    {
        bUseAuthenticationHeader = true;
        return true;
    }

    std::string osRefreshToken(VSIGetPathSpecificOption(
        osPathForOption.c_str(), "GS_OAUTH2_REFRESH_TOKEN", ""));
    if (!osRefreshToken.empty())
    {
        std::string osClientId = VSIGetPathSpecificOption(
            osPathForOption.c_str(), "GS_OAUTH2_CLIENT_ID", "");
        std::string osClientSecret = VSIGetPathSpecificOption(
            osPathForOption.c_str(), "GS_OAUTH2_CLIENT_SECRET", "");

        int nCount =
            (!osClientId.empty() ? 1 : 0) + (!osClientSecret.empty() ? 1 : 0);
        if (nCount == 1)
        {
            CPLError(CE_Failure, CPLE_NotSupported,
                     "Either both or none of GS_OAUTH2_CLIENT_ID and "
                     "GS_OAUTH2_CLIENT_SECRET must be set");
            return false;
        }

        if (bFirstTimeForDebugMessage)
        {
            std::string osMsg(
                "Using GS_OAUTH2_REFRESH_TOKEN configuration option");
            if (osClientId.empty())
                osMsg += " and GDAL default client_id/client_secret";
            else
                osMsg += " and GS_OAUTH2_CLIENT_ID and GS_OAUTH2_CLIENT_SECRET";
            CPLDebug("GS", "%s", osMsg.c_str());
        }
        bFirstTimeForDebugMessage = false;

        return oManager.SetAuthFromRefreshToken(
            osRefreshToken.c_str(), osClientId.c_str(), osClientSecret.c_str(),
            nullptr);
    }

    std::string osJsonFile(CSLFetchNameValueDef(
        papszOptions, "GOOGLE_APPLICATION_CREDENTIALS",
        VSIGetPathSpecificOption(osPathForOption.c_str(),
                                 "GOOGLE_APPLICATION_CREDENTIALS", "")));
    if (!osJsonFile.empty())
    {
        CPLJSONDocument oDoc;
        if (!oDoc.Load(osJsonFile))
        {
            return false;
        }

        // JSON file can be of type 'service_account' or 'authorized_user'
        std::string osJsonFileType = oDoc.GetRoot().GetString("type");

        if (strcmp(osJsonFileType.c_str(), "service_account") == 0)
        {
            CPLString osPrivateKey = oDoc.GetRoot().GetString("private_key");
            osPrivateKey.replaceAll("\\n", "\n")
                .replaceAll("\n\n", "\n")
                .replaceAll("\r", "");
            std::string osClientEmail =
                oDoc.GetRoot().GetString("client_email");
            const char *pszScope = CSLFetchNameValueDef(
                papszOptions, "GS_OAUTH2_SCOPE",
                VSIGetPathSpecificOption(
                    osPathForOption.c_str(), "GS_OAUTH2_SCOPE",
                    "https://www.googleapis.com/auth/devstorage.read_write"));

            return oManager.SetAuthFromServiceAccount(
                osPrivateKey.c_str(), osClientEmail.c_str(), pszScope, nullptr,
                nullptr);
        }
        else if (strcmp(osJsonFileType.c_str(), "authorized_user") == 0)
        {
            std::string osClientId = oDoc.GetRoot().GetString("client_id");
            std::string osClientSecret =
                oDoc.GetRoot().GetString("client_secret");
            osRefreshToken = oDoc.GetRoot().GetString("refresh_token");

            return oManager.SetAuthFromRefreshToken(
                osRefreshToken.c_str(), osClientId.c_str(),
                osClientSecret.c_str(), nullptr);
        }
        return false;
    }

    CPLString osPrivateKey = CSLFetchNameValueDef(
        papszOptions, "GS_OAUTH2_PRIVATE_KEY",
        VSIGetPathSpecificOption(osPathForOption.c_str(),
                                 "GS_OAUTH2_PRIVATE_KEY", ""));
    std::string osPrivateKeyFile = CSLFetchNameValueDef(
        papszOptions, "GS_OAUTH2_PRIVATE_KEY_FILE",
        VSIGetPathSpecificOption(osPathForOption.c_str(),
                                 "GS_OAUTH2_PRIVATE_KEY_FILE", ""));
    if (!osPrivateKey.empty() || !osPrivateKeyFile.empty())
    {
        if (!osPrivateKeyFile.empty())
        {
            VSILFILE *fp = VSIFOpenL(osPrivateKeyFile.c_str(), "rb");
            if (fp == nullptr)
            {
                CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
                         osPrivateKeyFile.c_str());
                bFirstTimeForDebugMessage = false;
                return false;
            }
            else
            {
                char *pabyBuffer = static_cast<char *>(CPLMalloc(32768));
                size_t nRead = VSIFReadL(pabyBuffer, 1, 32768, fp);
                osPrivateKey.assign(pabyBuffer, nRead);
                VSIFCloseL(fp);
                CPLFree(pabyBuffer);
            }
        }
        osPrivateKey.replaceAll("\\n", "\n")
            .replaceAll("\n\n", "\n")
            .replaceAll("\r", "");

        std::string osClientEmail = CSLFetchNameValueDef(
            papszOptions, "GS_OAUTH2_CLIENT_EMAIL",
            VSIGetPathSpecificOption(osPathForOption.c_str(),
                                     "GS_OAUTH2_CLIENT_EMAIL", ""));
        if (osClientEmail.empty())
        {
            CPLError(CE_Failure, CPLE_AppDefined,
                     "GS_OAUTH2_CLIENT_EMAIL not defined");
            bFirstTimeForDebugMessage = false;
            return false;
        }
        const char *pszScope = CSLFetchNameValueDef(
            papszOptions, "GS_OAUTH2_SCOPE",
            VSIGetPathSpecificOption(
                osPathForOption.c_str(), "GS_OAUTH2_SCOPE",
                "https://www.googleapis.com/auth/devstorage.read_write"));

        if (bFirstTimeForDebugMessage)
        {
            CPLDebug("GS",
                     "Using %s, GS_OAUTH2_CLIENT_EMAIL and GS_OAUTH2_SCOPE=%s "
                     "configuration options",
                     !osPrivateKeyFile.empty() ? "GS_OAUTH2_PRIVATE_KEY_FILE"
                                               : "GS_OAUTH2_PRIVATE_KEY",
                     pszScope);
        }
        bFirstTimeForDebugMessage = false;

        return oManager.SetAuthFromServiceAccount(osPrivateKey.c_str(),
                                                  osClientEmail.c_str(),
                                                  pszScope, nullptr, nullptr);
    }

    // Next try reading from ~/.boto
    std::string osCredentials;
    std::string osOAuth2RefreshToken;
    std::string osOAuth2ClientId;
    std::string osOAuth2ClientSecret;
    if (GetConfigurationFromConfigFile(osSecretAccessKey, osAccessKeyId,
                                       osOAuth2RefreshToken, osOAuth2ClientId,
                                       osOAuth2ClientSecret, osCredentials))
    {
        if (!osOAuth2RefreshToken.empty())
        {
            std::string osClientId =
                CPLGetConfigOption("GS_OAUTH2_CLIENT_ID", "");
            std::string osClientSecret =
                CPLGetConfigOption("GS_OAUTH2_CLIENT_SECRET", "");
            bool bClientInfoFromEnv = false;
            bool bClientInfoFromFile = false;

            const int nCountClientIdSecret = (!osClientId.empty() ? 1 : 0) +
                                             (!osClientSecret.empty() ? 1 : 0);
            if (nCountClientIdSecret == 1)
            {
                CPLError(CE_Failure, CPLE_NotSupported,
                         "Either both or none of GS_OAUTH2_CLIENT_ID and "
                         "GS_OAUTH2_CLIENT_SECRET must be set");
                return false;
            }
            else if (nCountClientIdSecret == 2)
            {
                bClientInfoFromEnv = true;
            }
            else if (nCountClientIdSecret == 0)
            {
                int nCountOAuth2IdSecret = (!osOAuth2ClientId.empty() ? 1 : 0);
                nCountOAuth2IdSecret += (!osOAuth2ClientSecret.empty() ? 1 : 0);
                if (nCountOAuth2IdSecret == 1)
                {
                    CPLError(CE_Failure, CPLE_NotSupported,
                             "Either both or none of client_id and "
                             "client_secret from %s must be set",
                             osCredentials.c_str());
                    return false;
                }
                else if (nCountOAuth2IdSecret == 2)
                {
                    osClientId = std::move(osOAuth2ClientId);
                    osClientSecret = std::move(osOAuth2ClientSecret);
                    bClientInfoFromFile = true;
                }
            }

            if (bFirstTimeForDebugMessage)
            {
                CPLString osMsg;
                osMsg.Printf("Using gs_oauth2_refresh_token from %s",
                             osCredentials.c_str());
                if (bClientInfoFromEnv)
                    osMsg += " and GS_OAUTH2_CLIENT_ID and "
                             "GS_OAUTH2_CLIENT_SECRET configuration options";
                else if (bClientInfoFromFile)
                    osMsg +=
                        CPLSPrintf(" and client_id and client_secret from %s",
                                   osCredentials.c_str());
                else
                    osMsg += " and GDAL default client_id/client_secret";
                CPLDebug("GS", "%s", osMsg.c_str());
            }
            bFirstTimeForDebugMessage = false;

            return oManager.SetAuthFromRefreshToken(
                osOAuth2RefreshToken.c_str(), osClientId.c_str(),
                osClientSecret.c_str(), nullptr);
        }
        else
        {
            if (bFirstTimeForDebugMessage)
            {
                CPLDebug(
                    "GS",
                    "Using gs_access_key_id and gs_secret_access_key from %s",
                    osCredentials.c_str());
            }
            bFirstTimeForDebugMessage = false;
            return true;
        }
    }

    // Some Travis-CI workers are GCE machines, and for some tests, we don't
    // want this code path to be taken. And on AppVeyor/Window, we would also
    // attempt a network access
    if (!CPLTestBool(CPLGetConfigOption("CPL_GCE_SKIP", "NO")) &&
        CPLIsMachinePotentiallyGCEInstance())
    {
        oManager.SetAuthFromGCE(nullptr);

        if (!GOA2ManagerCache::GetSingleton().GetBearer(oManager).empty())
        {
            CPLDebug("GS", "Using GCE inherited permissions");

            bFirstTimeForDebugMessage = false;
            return true;
        }
    }

    if (bMayWarnDidNotFindAuth)
    {
        CPLDebug("GS", "Cannot find Authorization header in %s",
                 CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", ""));
    }

    CPLString osMsg;
    osMsg.Printf(
        "No valid GCS credentials found. "
        "For authenticated requests, you need to set "
        "GS_SECRET_ACCESS_KEY, GS_ACCESS_KEY_ID, GS_OAUTH2_REFRESH_TOKEN, "
        "GOOGLE_APPLICATION_CREDENTIALS, or other configuration "
        "options, or create a %s file. Consult "
        "https://gdal.org/en/stable/user/"
        "virtual_file_systems.html#vsigs-google-cloud-storage-files "
        "for more details. "
        "For unauthenticated requests on public resources, set the "
        "GS_NO_SIGN_REQUEST configuration option to YES.",
        osCredentials.c_str());

    CPLDebug("GS", "%s", osMsg.c_str());
    VSIError(VSIE_InvalidCredentials, "%s", osMsg.c_str());
    return false;
}

/************************************************************************/
/*                            BuildFromURI()                            */
/************************************************************************/

VSIGSHandleHelper *VSIGSHandleHelper::BuildFromURI(
    const char *pszURI, const char * /*pszFSPrefix*/,
    const char *pszURIForPathSpecificOption, CSLConstList papszOptions)
{
    std::string osPathForOption("/vsigs/");
    osPathForOption +=
        pszURIForPathSpecificOption ? pszURIForPathSpecificOption : pszURI;

    // pszURI == bucket/object
    const std::string osBucketObject(pszURI);
    std::string osEndpoint(VSIGetPathSpecificOption(osPathForOption.c_str(),
                                                    "CPL_GS_ENDPOINT", ""));
    if (osEndpoint.empty())
        osEndpoint = "https://storage.googleapis.com/";

    std::string osSecretAccessKey;
    std::string osAccessKeyId;
    bool bUseAuthenticationHeader;
    GOA2Manager oManager;

    if (!GetConfiguration(osPathForOption, papszOptions, osSecretAccessKey,
                          osAccessKeyId, bUseAuthenticationHeader, oManager))
    {
        return nullptr;
    }

    // https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooguserproject
    // The Project ID for an existing Google Cloud project to bill for access
    // charges associated with the request.
    const std::string osUserProject = VSIGetPathSpecificOption(
        osPathForOption.c_str(), "GS_USER_PROJECT", "");

    return new VSIGSHandleHelper(osEndpoint, osBucketObject, osSecretAccessKey,
                                 osAccessKeyId, bUseAuthenticationHeader,
                                 oManager, osUserProject);
}

/************************************************************************/
/*                             RebuildURL()                             */
/************************************************************************/

void VSIGSHandleHelper::RebuildURL()
{
    m_osURL = m_osEndpoint + CPLAWSURLEncode(m_osBucketObjectKey, false);
    if (!m_osBucketObjectKey.empty() &&
        m_osBucketObjectKey.find('/') == std::string::npos)
        m_osURL += "/";
    m_osURL += GetQueryString(false);
}

/************************************************************************/
/*                            UsesHMACKey()                             */
/************************************************************************/

bool VSIGSHandleHelper::UsesHMACKey() const
{
    return m_oManager.GetAuthMethod() == GOA2Manager::NONE;
}

/************************************************************************/
/*                           GetCurlHeaders()                           */
/************************************************************************/

struct curl_slist *
VSIGSHandleHelper::GetCurlHeaders(const std::string &osVerb,
                                  struct curl_slist *psHeaders, const void *,
                                  size_t) const
{
    if (m_bUseAuthenticationHeader)
        return psHeaders;

    if (!m_osUserProject.empty())
    {
        psHeaders =
            curl_slist_append(psHeaders, CPLSPrintf("x-goog-user-project: %s",
                                                    m_osUserProject.c_str()));
    }

    if (m_oManager.GetAuthMethod() != GOA2Manager::NONE)
    {
        const std::string osBearer =
            GOA2ManagerCache::GetSingleton().GetBearer(m_oManager);
        if (!osBearer.empty())
        {
            psHeaders = curl_slist_append(
                psHeaders,
                CPLSPrintf("Authorization: Bearer %s", osBearer.c_str()));
        }
        return psHeaders;
    }

    std::string osCanonicalResource(
        "/" + CPLAWSURLEncode(m_osBucketObjectKey, false));
    if (!m_osBucketObjectKey.empty() &&
        m_osBucketObjectKey.find('/') == std::string::npos)
        osCanonicalResource += "/";
    else
    {
        const auto osQueryString(GetQueryString(false));
        if (osQueryString == "?uploads" || osQueryString == "?acl")
            osCanonicalResource += osQueryString;
    }

    // If accessing a Google Cloud account from a GC VM, check that
    // Google is still a sponsor, and if not, make some (kind) noise.
    if ((m_oManager.GetAuthMethod() == GOA2Manager::GCE &&
         cpl::starts_with(m_osEndpoint, "https://storage.googleapis.com/"))
#ifdef DEBUG
        || CPLTestBool(CPLGetConfigOption("GDAL_TEST_NAME_AND_SHAME", "NO"))
#endif
    )
    {
        static const bool bCheckSponsoring = []()
        {
            if (!CPLTestBool(CPLGetConfigOption("GDAL_NAME_AND_SHAME", "YES")))
                return true;

            const std::string osCacheDir = []()
            {
#ifdef _WIN32
                const char *pszHome =
                    CPLGetConfigOption("USERPROFILE", nullptr);
#else
                const char *pszHome = CPLGetConfigOption("HOME", nullptr);
#endif
                if (pszHome != nullptr)
                {
                    return CPLFormFilenameSafe(pszHome, ".gdal", nullptr);
                }
                else
                {
                    const char *pszDir = CPLGetConfigOption("TEMP", "/tmp");
                    VSIStatBufL sStat;
                    if (VSIStatL(pszDir, &sStat) == 0)
                    {
                        const char *pszUsername =
                            CPLGetConfigOption("USERNAME", nullptr);
                        if (pszUsername == nullptr)
                            pszUsername = CPLGetConfigOption("USER", nullptr);

                        if (pszUsername != nullptr)
                        {
                            return CPLFormFilenameSafe(
                                pszDir, CPLSPrintf(".gdal_%s", pszUsername),
                                nullptr);
                        }
                    }
                }
                return std::string();
            }();
            if (!osCacheDir.empty())
            {
                VSIStatBufL sStat;
                if (VSIStatL(osCacheDir.c_str(), &sStat) != 0)
                    VSIMkdir(osCacheDir.c_str(), 0755);
                const std::string osCloudCheck = CPLFormFilenameSafe(
                    osCacheDir.c_str(), "cloud_check_gcs.txt", nullptr);
                // Sidereal day, why not? "Aim for the stars, expect dust"
                constexpr int ONE_DAY_IN_SECS = 86164;
                if (VSIStatL(osCloudCheck.c_str(), &sStat) == 0 &&
                    sStat.st_mtime + ONE_DAY_IN_SECS >= time(nullptr))
                {
                    CPLDebugOnly("GDAL", "%s checked", osCloudCheck.c_str());
                }
                else
                {
                    FILE *f = fopen(osCloudCheck.c_str(), "wb");
                    if (f)
                        fclose(f);

                    const auto PingURL = [](const char *pszURL)
                    {
                        CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler);
                        const char *const apszOptions[] = {
                            "CUSTOMREQUEST=HEAD", "TIMEOUT=1", nullptr};
                        auto res = CPLHTTPFetch(pszURL, apszOptions);
                        const bool bOK = res && !res->pszErrBuf;
                        CPLHTTPDestroyResult(res);
                        return bOK;
                    };
                    if (!PingURL("https://gdal.org/en/latest/sponsors/"
                                 "did_google_sponsor.html") &&
                        // check that gdal.org is responding to avoid false positive
                        PingURL("https://gdal.org/en/latest/index.html"))
                    {
                        const auto CPLE_NonCooperativeSponsor = CPLE_AppDefined;
                        CPLError(
                            CE_Warning, CPLE_NonCooperativeSponsor,
                            "Due to lack of resources, Google Cloud Storage "
                            "access is undergoing minimal maintenance and may "
                            "be removed in the future unless Google Cloud "
                            "re-evaluates its decision to stop sponsoring "
                            "GDAL. If you are interested in keeping this "
                            "functionality please get in touch with your "
                            "Google Cloud representative.");
                    }
                }
            }
            return true;
        }();
        CPL_IGNORE_RET_VAL(bCheckSponsoring);
    }

    return GetGSHeaders("/vsigs/" + m_osBucketObjectKey, osVerb, psHeaders,
                        osCanonicalResource, m_osSecretAccessKey,
                        m_osAccessKeyId);
}

/************************************************************************/
/*                             ClearCache()                             */
/************************************************************************/

void VSIGSHandleHelper::ClearCache()
{
    GOA2ManagerCache::GetSingleton().clear();
    bFirstTimeForDebugMessage = true;
}

/************************************************************************/
/*                            GetSignedURL()                            */
/************************************************************************/

std::string VSIGSHandleHelper::GetSignedURL(CSLConstList papszOptions)
{
    if (!((!m_osAccessKeyId.empty() && !m_osSecretAccessKey.empty()) ||
          m_oManager.GetAuthMethod() == GOA2Manager::SERVICE_ACCOUNT))
    {
        CPLError(CE_Failure, CPLE_NotSupported,
                 "Signed URL for Google Cloud Storage is only available with "
                 "AWS style authentication with "
                 "GS_ACCESS_KEY_ID+GS_SECRET_ACCESS_KEY, "
                 "or with service account authentication");
        return std::string();
    }

    GIntBig nStartDate = static_cast<GIntBig>(time(nullptr));
    const char *pszStartDate = CSLFetchNameValue(papszOptions, "START_DATE");
    if (pszStartDate)
    {
        int nYear, nMonth, nDay, nHour, nMin, nSec;
        if (sscanf(pszStartDate, "%04d%02d%02dT%02d%02d%02dZ", &nYear, &nMonth,
                   &nDay, &nHour, &nMin, &nSec) == 6)
        {
            struct tm brokendowntime;
            brokendowntime.tm_year = nYear - 1900;
            brokendowntime.tm_mon = nMonth - 1;
            brokendowntime.tm_mday = nDay;
            brokendowntime.tm_hour = nHour;
            brokendowntime.tm_min = nMin;
            brokendowntime.tm_sec = nSec;
            nStartDate = CPLYMDHMSToUnixTime(&brokendowntime);
        }
    }
    GIntBig nExpiresIn =
        nStartDate +
        atoi(CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600"));
    std::string osExpires(CSLFetchNameValueDef(
        papszOptions, "EXPIRES", CPLSPrintf(CPL_FRMT_GIB, nExpiresIn)));

    std::string osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));

    std::string osCanonicalizedResource(
        "/" + CPLAWSURLEncode(m_osBucketObjectKey, false));

    std::string osStringToSign;
    osStringToSign += osVerb + "\n";
    osStringToSign += "\n";  // Content_MD5
    osStringToSign += "\n";  // Content_Type
    osStringToSign += osExpires + "\n";
    // osStringToSign += // Canonicalized_Extension_Headers
    osStringToSign += osCanonicalizedResource;
#ifdef DEBUG_VERBOSE
    CPLDebug("GS", "osStringToSign = %s", osStringToSign.c_str());
#endif

    if (!m_osAccessKeyId.empty())
    {
        // No longer documented but actually works !
        GByte abySignature[CPL_SHA1_HASH_SIZE] = {};
        CPL_HMAC_SHA1(m_osSecretAccessKey.c_str(), m_osSecretAccessKey.size(),
                      osStringToSign.c_str(), osStringToSign.size(),
                      abySignature);

        char *pszBase64 = CPLBase64Encode(sizeof(abySignature), abySignature);
        std::string osSignature(pszBase64);
        CPLFree(pszBase64);

        ResetQueryParameters();
        AddQueryParameter("GoogleAccessId", m_osAccessKeyId);
        AddQueryParameter("Expires", osExpires);
        AddQueryParameter("Signature", osSignature);
    }
    else
    {
        unsigned nSignatureLen = 0;
        GByte *pabySignature = CPL_RSA_SHA256_Sign(
            m_oManager.GetPrivateKey().c_str(), osStringToSign.data(),
            static_cast<unsigned>(osStringToSign.size()), &nSignatureLen);
        if (pabySignature == nullptr)
            return std::string();
        char *pszBase64 = CPLBase64Encode(nSignatureLen, pabySignature);
        CPLFree(pabySignature);
        std::string osSignature(pszBase64);
        CPLFree(pszBase64);

        ResetQueryParameters();
        AddQueryParameter("GoogleAccessId", m_oManager.GetClientEmail());
        AddQueryParameter("Expires", osExpires);
        AddQueryParameter("Signature", osSignature);
    }
    return m_osURL;
}

#endif

//! @endcond

Coverage Report

Created: 2026-06-30 08:33