/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */

/* vim: set ts=8 sts=2 et sw=2 tw=80: */

/* This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "mozilla/dom/cache/DBSchema.h"

#include "ipc/IPCMessageUtils.h"

#include "mozilla/BasePrincipal.h"

#include "mozilla/dom/HeadersBinding.h"

#include "mozilla/dom/InternalHeaders.h"

#include "mozilla/dom/InternalResponse.h"

#include "mozilla/dom/RequestBinding.h"

#include "mozilla/dom/ResponseBinding.h"

#include "mozilla/dom/cache/CacheTypes.h"

#include "mozilla/dom/cache/SavedTypes.h"

#include "mozilla/dom/cache/Types.h"

#include "mozilla/dom/cache/TypeUtils.h"

#include "mozilla/net/MozURL.h"

#include "mozIStorageConnection.h"

#include "mozIStorageStatement.h"

#include "mozStorageHelper.h"

#include "nsCOMPtr.h"

#include "nsCRT.h"

#include "nsHttp.h"

#include "nsIContentPolicy.h"

#include "nsICryptoHash.h"

#include "nsNetCID.h"

#include "nsPrintfCString.h"

#include "nsTArray.h"

namespace mozilla {

namespace dom {

namespace cache {

namespace db {

const int32_t kFirstShippedSchemaVersion = 15;

namespace {

// ## Firefox 57 Cache API v25/v26/v27 Schema Hack Info

// ### Overview

// In Firefox 57 we introduced Cache API schema version 26 and Quota Manager

// schema v3 to support tracking padding for opaque responses.  Unfortunately,

// Firefox 57 is a big release that may potentially result in users downgrading

// to Firefox 56 due to 57 retiring add-ons.  These schema changes have the

// unfortunate side-effect of causing QuotaManager and all its clients to break

// if the user downgrades to 56.  In order to avoid making a bad situation

// worse, we're now retrofitting 57 so that Firefox 56 won't freak out.

//

// ### Implementation

// We're introducing a new schema version 27 that uses an on-disk schema version

// of v25.  We differentiate v25 from v27 by the presence of the column added

// by v26.  This translates to:

// - v25: on-disk schema=25, no "response_padding_size" column in table

//   "entries".

// - v26: on-disk schema=26, yes "response_padding_size" column in table

//   "entries".

// - v27: on-disk schema=25, yes "response_padding_size" column in table

//   "entries".

//

// ### Fallout

// Firefox 57 is happy because it sees schema 27 and everything is as it

// expects.

//

// Firefox 56 non-DEBUG build is fine/happy, but DEBUG builds will not be.

// - Our QuotaClient will invoke `NS_WARNING("Unknown Cache file found!");`

//   at QuotaManager init time.  This is harmless but annoying and potentially

//   misleading.

// - The DEBUG-only Validate() call will error out whenever an attempt is made

//   to open a DOM Cache database because it will notice the schema is broken

//   and there is no attempt at recovery.

//

const int32_t kHackyDowngradeSchemaVersion = 25;

const int32_t kHackyPaddingSizePresentVersion = 27;

//

// Update this whenever the DB schema is changed.

const int32_t kLatestSchemaVersion = 27;

// ---------

// The following constants define the SQL schema.  These are defined in the

// same order the SQL should be executed in CreateOrMigrateSchema().  They are

// broken out as constants for convenient use in validation and migration.

// ---------

// The caches table is the single source of truth about what Cache

// objects exist for the origin.  The contents of the Cache are stored

// in the entries table that references back to caches.

//

// The caches table is also referenced from storage.  Rows in storage

// represent named Cache objects.  There are cases, however, where

// a Cache can still exist, but not be in a named Storage.  For example,

// when content is still using the Cache after CacheStorage::Delete()

// has been run.

//

// For now, the caches table mainly exists for data integrity with

// foreign keys, but could be expanded to contain additional cache object

// information.

//

// AUTOINCREMENT is necessary to prevent CacheId values from being reused.

const char* const kTableCaches =

  "CREATE TABLE caches ("

    "id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT "

  ")";

// Security blobs are quite large and duplicated for every Response from

// the same https origin.  This table is used to de-duplicate this data.

const char* const kTableSecurityInfo =

  "CREATE TABLE security_info ("

    "id INTEGER NOT NULL PRIMARY KEY, "

    "hash BLOB NOT NULL, "  // first 8-bytes of the sha1 hash of data column

    "data BLOB NOT NULL, "  // full security info data, usually a few KB

    "refcount INTEGER NOT NULL"

  ")";

// Index the smaller hash value instead of the large security data blob.

const char* const kIndexSecurityInfoHash =

  "CREATE INDEX security_info_hash_index ON security_info (hash)";

const char* const kTableEntries =

  "CREATE TABLE entries ("

    "id INTEGER NOT NULL PRIMARY KEY, "

    "request_method TEXT NOT NULL, "

    "request_url_no_query TEXT NOT NULL, "

    "request_url_no_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash

    "request_url_query TEXT NOT NULL, "

    "request_url_query_hash BLOB NOT NULL, "    // first 8-bytes of sha1 hash

    "request_referrer TEXT NOT NULL, "

    "request_headers_guard INTEGER NOT NULL, "

    "request_mode INTEGER NOT NULL, "

    "request_credentials INTEGER NOT NULL, "

    "request_contentpolicytype INTEGER NOT NULL, "

    "request_cache INTEGER NOT NULL, "

    "request_body_id TEXT NULL, "

    "response_type INTEGER NOT NULL, "

    "response_status INTEGER NOT NULL, "

    "response_status_text TEXT NOT NULL, "

    "response_headers_guard INTEGER NOT NULL, "

    "response_body_id TEXT NULL, "

    "response_security_info_id INTEGER NULL REFERENCES security_info(id), "

    "response_principal_info TEXT NOT NULL, "

    "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, "

    "request_redirect INTEGER NOT NULL, "

    "request_referrer_policy INTEGER NOT NULL, "

    "request_integrity TEXT NOT NULL, "

    "request_url_fragment TEXT NOT NULL, "

    "response_padding_size INTEGER NULL "

    // New columns must be added at the end of table to migrate and

    // validate properly.

  ")";

// Create an index to support the QueryCache() matching algorithm.  This

// needs to quickly find entries in a given Cache that match the request

// URL.  The url query is separated in order to support the ignoreSearch

// option.  Finally, we index hashes of the URL values instead of the

// actual strings to avoid excessive disk bloat.  The index will duplicate

// the contents of the columsn in the index.  The hash index will prune

// the vast majority of values from the query result so that normal

// scanning only has to be done on a few values to find an exact URL match.

const char* const kIndexEntriesRequest =

  "CREATE INDEX entries_request_match_index "

            "ON entries (cache_id, request_url_no_query_hash, "

                        "request_url_query_hash)";

const char* const kTableRequestHeaders =

  "CREATE TABLE request_headers ("

    "name TEXT NOT NULL, "

    "value TEXT NOT NULL, "

    "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"

  ")";

const char* const kTableResponseHeaders =

  "CREATE TABLE response_headers ("

    "name TEXT NOT NULL, "

    "value TEXT NOT NULL, "

    "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"

  ")";

// We need an index on response_headers, but not on request_headers,

// because we quickly need to determine if a VARY header is present.

const char* const kIndexResponseHeadersName =

  "CREATE INDEX response_headers_name_index "

            "ON response_headers (name)";

const char* const kTableResponseUrlList =

  "CREATE TABLE response_url_list ("

    "url TEXT NOT NULL, "

    "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE"

  ")";

// NOTE: key allows NULL below since that is how "" is represented

//       in a BLOB column.  We use BLOB to avoid encoding issues

//       with storing DOMStrings.

const char* const kTableStorage =

  "CREATE TABLE storage ("

    "namespace INTEGER NOT NULL, "

    "key BLOB NULL, "

    "cache_id INTEGER NOT NULL REFERENCES caches(id), "

    "PRIMARY KEY(namespace, key) "

  ")";

// ---------

// End schema definition

// ---------

const int32_t kMaxEntriesPerStatement = 255;

const uint32_t kPageSize = 4 * 1024;

// Grow the database in chunks to reduce fragmentation

const uint32_t kGrowthSize = 32 * 1024;

const uint32_t kGrowthPages = kGrowthSize / kPageSize;

static_assert(kGrowthSize % kPageSize == 0,

              "Growth size must be multiple of page size");

// Only release free pages when we have more than this limit

const int32_t kMaxFreePages = kGrowthPages;

// Limit WAL journal to a reasonable size

const uint32_t kWalAutoCheckpointSize = 512 * 1024;

const uint32_t kWalAutoCheckpointPages = kWalAutoCheckpointSize / kPageSize;

static_assert(kWalAutoCheckpointSize % kPageSize == 0,

              "WAL checkpoint size must be multiple of page size");

} // namespace

// If any of the static_asserts below fail, it means that you have changed

// the corresponding WebIDL enum in a way that may be incompatible with the

// existing data stored in the DOM Cache.  You would need to update the Cache

// database schema accordingly and adjust the failing static_assert.

static_assert(int(HeadersGuardEnum::None) == 0 &&

              int(HeadersGuardEnum::Request) == 1 &&

              int(HeadersGuardEnum::Request_no_cors) == 2 &&

              int(HeadersGuardEnum::Response) == 3 &&

              int(HeadersGuardEnum::Immutable) == 4 &&

              int(HeadersGuardEnum::EndGuard_) == 5,

              "HeadersGuardEnum values are as expected");

static_assert(int(ReferrerPolicy::_empty) == 0 &&

              int(ReferrerPolicy::No_referrer) == 1 &&

              int(ReferrerPolicy::No_referrer_when_downgrade) == 2 &&

              int(ReferrerPolicy::Origin) == 3 &&

              int(ReferrerPolicy::Origin_when_cross_origin) == 4 &&

              int(ReferrerPolicy::Unsafe_url) == 5 &&

              int(ReferrerPolicy::Same_origin) == 6 &&

              int(ReferrerPolicy::Strict_origin) == 7 &&

              int(ReferrerPolicy::Strict_origin_when_cross_origin) == 8 &&

              int(ReferrerPolicy::EndGuard_) == 9,

              "ReferrerPolicy values are as expected");

static_assert(int(RequestMode::Same_origin) == 0 &&

              int(RequestMode::No_cors) == 1 &&

              int(RequestMode::Cors) == 2 &&

              int(RequestMode::Navigate) == 3 &&

              int(RequestMode::EndGuard_) == 4,

              "RequestMode values are as expected");

static_assert(int(RequestCredentials::Omit) == 0 &&

              int(RequestCredentials::Same_origin) == 1 &&

              int(RequestCredentials::Include) == 2 &&

              int(RequestCredentials::EndGuard_) == 3,

              "RequestCredentials values are as expected");

static_assert(int(RequestCache::Default) == 0 &&

              int(RequestCache::No_store) == 1 &&

              int(RequestCache::Reload) == 2 &&

              int(RequestCache::No_cache) == 3 &&

              int(RequestCache::Force_cache) == 4 &&

              int(RequestCache::Only_if_cached) == 5 &&

              int(RequestCache::EndGuard_) == 6,

              "RequestCache values are as expected");

static_assert(int(RequestRedirect::Follow) == 0 &&

              int(RequestRedirect::Error) == 1 &&

              int(RequestRedirect::Manual) == 2 &&

              int(RequestRedirect::EndGuard_) == 3,

              "RequestRedirect values are as expected");

static_assert(int(ResponseType::Basic) == 0 &&

              int(ResponseType::Cors) == 1 &&

              int(ResponseType::Default) == 2 &&

              int(ResponseType::Error) == 3 &&

              int(ResponseType::Opaque) == 4 &&

              int(ResponseType::Opaqueredirect) == 5 &&

              int(ResponseType::EndGuard_) == 6,

              "ResponseType values are as expected");

// If the static_asserts below fails, it means that you have changed the

// Namespace enum in a way that may be incompatible with the existing data

// stored in the DOM Cache.  You would need to update the Cache database schema

// accordingly and adjust the failing static_assert.

static_assert(DEFAULT_NAMESPACE == 0 &&

              CHROME_ONLY_NAMESPACE == 1 &&

              NUMBER_OF_NAMESPACES == 2,

              "Namespace values are as expected");

// If the static_asserts below fails, it means that you have changed the

// nsContentPolicy enum in a way that may be incompatible with the existing data

// stored in the DOM Cache.  You would need to update the Cache database schema

// accordingly and adjust the failing static_assert.

static_assert(nsIContentPolicy::TYPE_INVALID == 0 &&

              nsIContentPolicy::TYPE_OTHER == 1 &&

              nsIContentPolicy::TYPE_SCRIPT == 2 &&

              nsIContentPolicy::TYPE_IMAGE == 3 &&

              nsIContentPolicy::TYPE_STYLESHEET == 4 &&

              nsIContentPolicy::TYPE_OBJECT == 5 &&

              nsIContentPolicy::TYPE_DOCUMENT == 6 &&

              nsIContentPolicy::TYPE_SUBDOCUMENT == 7 &&

              nsIContentPolicy::TYPE_REFRESH == 8 &&

              nsIContentPolicy::TYPE_XBL == 9 &&

              nsIContentPolicy::TYPE_PING == 10 &&

              nsIContentPolicy::TYPE_XMLHTTPREQUEST == 11 &&

              nsIContentPolicy::TYPE_DATAREQUEST == 11 &&

              nsIContentPolicy::TYPE_OBJECT_SUBREQUEST == 12 &&

              nsIContentPolicy::TYPE_DTD == 13 &&

              nsIContentPolicy::TYPE_FONT == 14 &&

              nsIContentPolicy::TYPE_MEDIA == 15 &&

              nsIContentPolicy::TYPE_WEBSOCKET == 16 &&

              nsIContentPolicy::TYPE_CSP_REPORT == 17 &&

              nsIContentPolicy::TYPE_XSLT == 18 &&

              nsIContentPolicy::TYPE_BEACON == 19 &&

              nsIContentPolicy::TYPE_FETCH == 20 &&

              nsIContentPolicy::TYPE_IMAGESET == 21 &&

              nsIContentPolicy::TYPE_WEB_MANIFEST == 22 &&

              nsIContentPolicy::TYPE_INTERNAL_SCRIPT == 23 &&

              nsIContentPolicy::TYPE_INTERNAL_WORKER == 24 &&

              nsIContentPolicy::TYPE_INTERNAL_SHARED_WORKER == 25 &&

              nsIContentPolicy::TYPE_INTERNAL_EMBED == 26 &&

              nsIContentPolicy::TYPE_INTERNAL_OBJECT == 27 &&

              nsIContentPolicy::TYPE_INTERNAL_FRAME == 28 &&

              nsIContentPolicy::TYPE_INTERNAL_IFRAME == 29 &&

              nsIContentPolicy::TYPE_INTERNAL_AUDIO == 30 &&

              nsIContentPolicy::TYPE_INTERNAL_VIDEO == 31 &&

              nsIContentPolicy::TYPE_INTERNAL_TRACK == 32 &&

              nsIContentPolicy::TYPE_INTERNAL_XMLHTTPREQUEST == 33 &&

              nsIContentPolicy::TYPE_INTERNAL_EVENTSOURCE == 34 &&

              nsIContentPolicy::TYPE_INTERNAL_SERVICE_WORKER == 35 &&

              nsIContentPolicy::TYPE_INTERNAL_SCRIPT_PRELOAD == 36 &&

              nsIContentPolicy::TYPE_INTERNAL_IMAGE == 37 &&

              nsIContentPolicy::TYPE_INTERNAL_IMAGE_PRELOAD == 38 &&

              nsIContentPolicy::TYPE_INTERNAL_STYLESHEET == 39 &&

              nsIContentPolicy::TYPE_INTERNAL_STYLESHEET_PRELOAD == 40 &&

              nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON == 41 &&

              nsIContentPolicy::TYPE_INTERNAL_WORKER_IMPORT_SCRIPTS == 42 &&

              nsIContentPolicy::TYPE_SAVEAS_DOWNLOAD == 43 &&

              nsIContentPolicy::TYPE_SPECULATIVE == 44,

              "nsContentPolicyType values are as expected");

namespace {

typedef int32_t EntryId;

struct IdCount

{

  explicit IdCount(int32_t aId) : mId(aId), mCount(1) { }

  int32_t mId;

  int32_t mCount;

};

static nsresult QueryAll(mozIStorageConnection* aConn, CacheId aCacheId,

                         nsTArray<EntryId>& aEntryIdListOut);

static nsresult QueryCache(mozIStorageConnection* aConn, CacheId aCacheId,

                           const CacheRequest& aRequest,

                           const CacheQueryParams& aParams,

                           nsTArray<EntryId>& aEntryIdListOut,

                           uint32_t aMaxResults = UINT32_MAX);

static nsresult MatchByVaryHeader(mozIStorageConnection* aConn,

                                  const CacheRequest& aRequest,

                                  EntryId entryId, bool* aSuccessOut);

static nsresult DeleteEntries(mozIStorageConnection* aConn,

                              const nsTArray<EntryId>& aEntryIdList,

                              nsTArray<nsID>& aDeletedBodyIdListOut,

                              nsTArray<IdCount>& aDeletedSecurityIdListOut,

                              int64_t* aDeletedPaddingSizeOut,

                              uint32_t aPos=0, int32_t aLen=-1);

static nsresult InsertSecurityInfo(mozIStorageConnection* aConn,

                                   nsICryptoHash* aCrypto,

                                   const nsACString& aData, int32_t *aIdOut);

static nsresult DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId,

                                   int32_t aCount);

static nsresult DeleteSecurityInfoList(mozIStorageConnection* aConn,

                                       const nsTArray<IdCount>& aDeletedStorageIdList);

static nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,

                            const CacheRequest& aRequest,

                            const nsID* aRequestBodyId,

                            const CacheResponse& aResponse,

                            const nsID* aResponseBodyId);

static nsresult ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId,

                             SavedResponse* aSavedResponseOut);

static nsresult ReadRequest(mozIStorageConnection* aConn, EntryId aEntryId,

                            SavedRequest* aSavedRequestOut);

static void AppendListParamsToQuery(nsACString& aQuery,

                                    const nsTArray<EntryId>& aEntryIdList,

                                    uint32_t aPos, int32_t aLen);

static nsresult BindListParamsToQuery(mozIStorageStatement* aState,

                                      const nsTArray<EntryId>& aEntryIdList,

                                      uint32_t aPos, int32_t aLen);

static nsresult BindId(mozIStorageStatement* aState, const nsACString& aName,

                       const nsID* aId);

static nsresult ExtractId(mozIStorageStatement* aState, uint32_t aPos,

                          nsID* aIdOut);

static nsresult CreateAndBindKeyStatement(mozIStorageConnection* aConn,

                                          const char* aQueryFormat,

                                          const nsAString& aKey,

                                          mozIStorageStatement** aStateOut);

static nsresult HashCString(nsICryptoHash* aCrypto, const nsACString& aIn,

                            nsACString& aOut);

nsresult GetEffectiveSchemaVersion(mozIStorageConnection* aConn,

                                   int32_t& schemaVersion);

nsresult Validate(mozIStorageConnection* aConn);

nsresult Migrate(mozIStorageConnection* aConn);

} // namespace

class MOZ_RAII AutoDisableForeignKeyChecking

{

public:

  explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)

    : mConn(aConn)

    , mForeignKeyCheckingDisabled(false)

  {

    nsCOMPtr<mozIStorageStatement> state;

    nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING(

      "PRAGMA foreign_keys;"

    ), getter_AddRefs(state));

    if (NS_WARN_IF(NS_FAILED(rv))) { return; }

    bool hasMoreData = false;

    rv = state->ExecuteStep(&hasMoreData);

    if (NS_WARN_IF(NS_FAILED(rv))) { return; }

    int32_t mode;

    rv = state->GetInt32(0, &mode);

    if (NS_WARN_IF(NS_FAILED(rv))) { return; }

    if (mode) {

      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(

        "PRAGMA foreign_keys = OFF;"

      ));

      if (NS_WARN_IF(NS_FAILED(rv))) { return; }

      mForeignKeyCheckingDisabled = true;

    }

  }

  ~AutoDisableForeignKeyChecking()

  {

    if (mForeignKeyCheckingDisabled) {

      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(

        "PRAGMA foreign_keys = ON;"

      ));

      if (NS_WARN_IF(NS_FAILED(rv))) { return; }

    }

  }

private:

  nsCOMPtr<mozIStorageConnection> mConn;

  bool mForeignKeyCheckingDisabled;

};

nsresult

CreateOrMigrateSchema(mozIStorageConnection* aConn)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  int32_t schemaVersion;

  nsresult rv = GetEffectiveSchemaVersion(aConn, schemaVersion);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  if (schemaVersion == kLatestSchemaVersion) {

    // We already have the correct schema version.  Validate it matches

    // our expected schema and then proceed.

    rv = Validate(aConn);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    return rv;

  }

  // Turn off checking foreign keys before starting a transaction, and restore

  // it once we're done.

  AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn);

  mozStorageTransaction trans(aConn, false,

                              mozIStorageConnection::TRANSACTION_IMMEDIATE);

  bool needVacuum = false;

  if (schemaVersion) {

    // A schema exists, but its not the current version.  Attempt to

    // migrate it to our new schema.

    rv = Migrate(aConn);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    // Migrations happen infrequently and reflect a chance in DB structure.

    // This is a good time to rebuild the database.  It also helps catch

    // if a new migration is incorrect by fast failing on the corruption.

    needVacuum = true;

  } else {

    // There is no schema installed.  Create the database from scratch.

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableCaches));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableSecurityInfo));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexSecurityInfoHash));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableEntries));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableRequestHeaders));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableResponseHeaders));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexResponseHeadersName));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableResponseUrlList));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableStorage));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = aConn->SetSchemaVersion(kHackyDowngradeSchemaVersion);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    rv = GetEffectiveSchemaVersion(aConn, schemaVersion);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  }

  rv = Validate(aConn);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = trans.Commit();

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  if (needVacuum) {

    // Unfortunately, this must be performed outside of the transaction.

    aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM"));

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  }

  return rv;

}

nsresult

InitializeConnection(mozIStorageConnection* aConn)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  // This function needs to perform per-connection initialization tasks that

  // need to happen regardless of the schema.

  nsPrintfCString pragmas(

    // Use a smaller page size to improve perf/footprint; default is too large

    "PRAGMA page_size = %u; "

    // Enable auto_vacuum; this must happen after page_size and before WAL

    "PRAGMA auto_vacuum = INCREMENTAL; "

    "PRAGMA foreign_keys = ON; ",

    kPageSize

  );

  // Note, the default encoding of UTF-8 is preferred.  mozStorage does all

  // the work necessary to convert UTF-16 nsString values for us.  We don't

  // need ordering and the binary equality operations are correct.  So, do

  // NOT set PRAGMA encoding to UTF-16.

  nsresult rv = aConn->ExecuteSimpleSQL(pragmas);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Limit fragmentation by growing the database by many pages at once.

  rv = aConn->SetGrowthIncrement(kGrowthSize, EmptyCString());

  if (rv == NS_ERROR_FILE_TOO_BIG) {

    NS_WARNING("Not enough disk space to set sqlite growth increment.");

    rv = NS_OK;

  }

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Enable WAL journaling.  This must be performed in a separate transaction

  // after changing the page_size and enabling auto_vacuum.

  nsPrintfCString wal(

    // WAL journal can grow to given number of *pages*

    "PRAGMA wal_autocheckpoint = %u; "

    // Always truncate the journal back to given number of *bytes*

    "PRAGMA journal_size_limit = %u; "

    // WAL must be enabled at the end to allow page size to be changed, etc.

    "PRAGMA journal_mode = WAL; ",

    kWalAutoCheckpointPages,

    kWalAutoCheckpointSize

  );

  rv = aConn->ExecuteSimpleSQL(wal);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Verify that we successfully set the vacuum mode to incremental.  It

  // is very easy to put the database in a state where the auto_vacuum

  // pragma above fails silently.

#ifdef DEBUG

  nsCOMPtr<mozIStorageStatement> state;

  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "PRAGMA auto_vacuum;"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  bool hasMoreData = false;

  rv = state->ExecuteStep(&hasMoreData);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  int32_t mode;

  rv = state->GetInt32(0, &mode);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // integer value 2 is incremental mode

  if (NS_WARN_IF(mode != 2)) { return NS_ERROR_UNEXPECTED; }

#endif

  return NS_OK;

}

nsresult

CreateCacheId(mozIStorageConnection* aConn, CacheId* aCacheIdOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aCacheIdOut);

  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(

    "INSERT INTO caches DEFAULT VALUES;"

  ));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  nsCOMPtr<mozIStorageStatement> state;

  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "SELECT last_insert_rowid()"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  bool hasMoreData = false;

  rv = state->ExecuteStep(&hasMoreData);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  if (NS_WARN_IF(!hasMoreData)) { return NS_ERROR_UNEXPECTED; }

  rv = state->GetInt64(0, aCacheIdOut);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  return rv;

}

nsresult

DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId,

              nsTArray<nsID>& aDeletedBodyIdListOut,

              int64_t* aDeletedPaddingSizeOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aDeletedPaddingSizeOut);

  // Delete the bodies explicitly as we need to read out the body IDs

  // anyway.  These body IDs must be deleted one-by-one as content may

  // still be referencing them invidivually.

  AutoTArray<EntryId, 256> matches;

  nsresult rv = QueryAll(aConn, aCacheId, matches);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  AutoTArray<IdCount, 16> deletedSecurityIdList;

  int64_t deletedPaddingSize = 0;

  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,

                     deletedSecurityIdList, &deletedPaddingSize);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aDeletedPaddingSizeOut = deletedPaddingSize;

  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Delete the remainder of the cache using cascade semantics.

  nsCOMPtr<mozIStorageStatement> state;

  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "DELETE FROM caches WHERE id=:id;"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = state->BindInt64ByName(NS_LITERAL_CSTRING("id"), aCacheId);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = state->Execute();

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  return rv;

}

nsresult

IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId,

                bool* aOrphanedOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aOrphanedOut);

  // err on the side of not deleting user data

  *aOrphanedOut = false;

  nsCOMPtr<mozIStorageStatement> state;

  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "SELECT COUNT(*) FROM storage WHERE cache_id=:cache_id;"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  bool hasMoreData = false;

  rv = state->ExecuteStep(&hasMoreData);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  MOZ_DIAGNOSTIC_ASSERT(hasMoreData);

  int32_t refCount;

  rv = state->GetInt32(0, &refCount);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aOrphanedOut = refCount == 0;

  return rv;

}

nsresult

FindOrphanedCacheIds(mozIStorageConnection* aConn,

                     nsTArray<CacheId>& aOrphanedListOut)

{

  nsCOMPtr<mozIStorageStatement> state;

  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "SELECT id FROM caches "

    "WHERE id NOT IN (SELECT cache_id from storage);"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  bool hasMoreData = false;

  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {

    CacheId cacheId = INVALID_CACHE_ID;

    rv = state->GetInt64(0, &cacheId);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    aOrphanedListOut.AppendElement(cacheId);

  }

  return rv;

}

nsresult

FindOverallPaddingSize(mozIStorageConnection* aConn,

                       int64_t* aOverallPaddingSizeOut)

{

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aOverallPaddingSizeOut);

  nsCOMPtr<mozIStorageStatement> state;

  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "SELECT response_padding_size FROM entries "

    "WHERE response_padding_size IS NOT NULL;"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  int64_t overallPaddingSize = 0;

  bool hasMoreData = false;

  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {

    int64_t padding_size = 0;

    rv = state->GetInt64(0, &padding_size);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    MOZ_DIAGNOSTIC_ASSERT(padding_size >= 0);

    MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - padding_size >= overallPaddingSize);

    overallPaddingSize += padding_size;

  }

  *aOverallPaddingSizeOut = overallPaddingSize;

  return rv;

}

nsresult

GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  nsCOMPtr<mozIStorageStatement> state;

  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(

    "SELECT request_body_id, response_body_id FROM entries;"

  ), getter_AddRefs(state));

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  bool hasMoreData = false;

  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {

    // extract 0 to 2 nsID structs per row

    for (uint32_t i = 0; i < 2; ++i) {

      bool isNull = false;

      rv = state->GetIsNull(i, &isNull);

      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

      if (!isNull) {

        nsID id;

        rv = ExtractId(state, i, &id);

        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

        aBodyIdListOut.AppendElement(id);

      }

    }

  }

  return rv;

}

nsresult

CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId,

           const CacheRequest& aRequest,

           const CacheQueryParams& aParams,

           bool* aFoundResponseOut,

           SavedResponse* aSavedResponseOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aFoundResponseOut);

  MOZ_DIAGNOSTIC_ASSERT(aSavedResponseOut);

  *aFoundResponseOut = false;

  AutoTArray<EntryId, 1> matches;

  nsresult rv = QueryCache(aConn, aCacheId, aRequest, aParams, matches, 1);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  if (matches.IsEmpty()) {

    return rv;

  }

  rv = ReadResponse(aConn, matches[0], aSavedResponseOut);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  aSavedResponseOut->mCacheId = aCacheId;

  *aFoundResponseOut = true;

  return rv;

}

nsresult

CacheMatchAll(mozIStorageConnection* aConn, CacheId aCacheId,

              const CacheRequestOrVoid& aRequestOrVoid,

              const CacheQueryParams& aParams,

              nsTArray<SavedResponse>& aSavedResponsesOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  nsresult rv;

  AutoTArray<EntryId, 256> matches;

  if (aRequestOrVoid.type() == CacheRequestOrVoid::Tvoid_t) {

    rv = QueryAll(aConn, aCacheId, matches);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  } else {

    rv = QueryCache(aConn, aCacheId, aRequestOrVoid, aParams, matches);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  }

  // TODO: replace this with a bulk load using SQL IN clause (bug 1110458)

  for (uint32_t i = 0; i < matches.Length(); ++i) {

    SavedResponse savedResponse;

    rv = ReadResponse(aConn, matches[i], &savedResponse);

    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

    savedResponse.mCacheId = aCacheId;

    aSavedResponsesOut.AppendElement(savedResponse);

  }

  return rv;

}

nsresult

CachePut(mozIStorageConnection* aConn, CacheId aCacheId,

         const CacheRequest& aRequest,

         const nsID* aRequestBodyId,

         const CacheResponse& aResponse,

         const nsID* aResponseBodyId,

         nsTArray<nsID>& aDeletedBodyIdListOut,

         int64_t* aDeletedPaddingSizeOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aDeletedPaddingSizeOut);

  CacheQueryParams params(false, false, false, false,

                           NS_LITERAL_STRING(""));

  AutoTArray<EntryId, 256> matches;

  nsresult rv = QueryCache(aConn, aCacheId, aRequest, params, matches);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  AutoTArray<IdCount, 16> deletedSecurityIdList;

  int64_t deletedPaddingSize = 0;

  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,

                     deletedSecurityIdList, &deletedPaddingSize);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, aResponse,

                   aResponseBodyId);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Delete the security values after doing the insert to avoid churning

  // the security table when its not necessary.

  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aDeletedPaddingSizeOut = deletedPaddingSize;

  return rv;

}

nsresult

CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId,

            const CacheRequest& aRequest,

            const CacheQueryParams& aParams,

            nsTArray<nsID>& aDeletedBodyIdListOut,

            int64_t* aDeletedPaddingSizeOut, bool* aSuccessOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  MOZ_DIAGNOSTIC_ASSERT(aDeletedPaddingSizeOut);

  MOZ_DIAGNOSTIC_ASSERT(aSuccessOut);

  *aSuccessOut = false;

  AutoTArray<EntryId, 256> matches;

  nsresult rv = QueryCache(aConn, aCacheId, aRequest, aParams, matches);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  if (matches.IsEmpty()) {

    return rv;

  }

  AutoTArray<IdCount, 16> deletedSecurityIdList;

  int64_t deletedPaddingSize = 0;

  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,

                     deletedSecurityIdList, &deletedPaddingSize);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aDeletedPaddingSizeOut = deletedPaddingSize;

  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);

  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aSuccessOut = true;

  return rv;

}

nsresult

CacheKeys(mozIStorageConnection* aConn, CacheId aCacheId,

          const CacheRequestOrVoid& aRequestOrVoid,

          const CacheQueryParams& aParams,

          nsTArray<SavedRequest>& aSavedRequestsOut)

{

  MOZ_ASSERT(!NS_IsMainThread());

  MOZ_DIAGNOSTIC_ASSERT(aConn);

  nsresult rv;

  AutoTArray<EntryId, 256> matches;

  if (aRequestOrVoid.type() == CacheRequestOrVoid::Tvoid_t) {

    rv = QueryAll(aConn, aCacheId, matches);