/******************************************************************************

 *

 * Project:  GDAL

 * Purpose:  gdal "pipeline" subcommand

 * Author:   Even Rouault <even dot rouault at spatialys.com>

 *

 ******************************************************************************

 * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>

 *

 * SPDX-License-Identifier: MIT

 ****************************************************************************/

//! @cond Doxygen_Suppress

#include "gdalalg_pipeline.h"

#include "cpl_error.h"

#include "gdal_priv.h"

#include "gdalalg_external.h"

#include "gdalalg_raster_read.h"

#include "gdalalg_raster_mosaic.h"

#include "gdalalg_raster_stack.h"

#include "gdalalg_raster_write.h"

#include "gdalalg_raster_zonal_stats.h"

#include "gdalalg_vector_read.h"

#include "gdalalg_vector_write.h"

#include "gdalalg_raster_as_features.h"

#include "gdalalg_raster_compare.h"

#include "gdalalg_raster_contour.h"

#include "gdalalg_raster_footprint.h"

#include "gdalalg_raster_polygonize.h"

#include "gdalalg_raster_info.h"

#include "gdalalg_raster_pixel_info.h"

#include "gdalalg_raster_tile.h"

#include "gdalalg_vector_grid.h"

#include "gdalalg_vector_info.h"

#include "gdalalg_vector_rasterize.h"

#include <algorithm>

#include <cassert>

#ifndef _

#define _(x) (x)

#endif

/************************************************************************/

/*                     GDALPipelineStepAlgorithm()                      */

/************************************************************************/

GDALPipelineStepAlgorithm::GDALPipelineStepAlgorithm(

    const std::string &name, const std::string &description,

    const std::string &helpURL, const ConstructorOptions &options)

    : GDALAlgorithm(name, description, helpURL),

      m_standaloneStep(options.standaloneStep), m_constructorOptions(options)

{

}

/************************************************************************/

/*     GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()      */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()

{

    AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER, false)

        .SetMinCount(0)

        .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

        .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

        .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)

        .SetHidden();

}

/************************************************************************/

/*           GDALPipelineStepAlgorithm::AddRasterInputArgs()            */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddRasterInputArgs(

    bool openForMixedRasterVector, bool hiddenForCLI)

{

    AddInputFormatsArg(&m_inputFormats)

        .AddMetadataItem(

            GAAMDI_REQUIRED_CAPABILITIES,

            openForMixedRasterVector

                ? std::vector<std::string>{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR}

                : std::vector<std::string>{GDAL_DCAP_RASTER})

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOpenOptionsArg(&m_openOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    const int nDatasetType = openForMixedRasterVector

                                 ? (GDAL_OF_RASTER | GDAL_OF_VECTOR)

                                 : GDAL_OF_RASTER;

    auto &arg =

        AddInputDatasetArg(

            &m_inputDataset, nDatasetType, false,

            m_constructorOptions.inputDatasetHelpMsg.empty() &&

                    m_constructorOptions.inputDatasetMaxCount == 1

                ? CPLSPrintf(

                      "Input %s dataset",

                      GDALAlgorithmArgDatasetTypeName(nDatasetType).c_str())

                : m_constructorOptions.inputDatasetHelpMsg.c_str())

            .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)

            .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)

            .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

            .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

            .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)

            .SetHiddenForCLI(hiddenForCLI)

            .SetAvailableInPipelineStep(false);

    if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)

        arg.SetPositional();

    if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)

        arg.SetRequired();

    if (!m_constructorOptions.inputDatasetAlias.empty())

        arg.AddAlias(m_constructorOptions.inputDatasetAlias);

}

/************************************************************************/

/*           GDALPipelineStepAlgorithm::AddRasterOutputArgs()           */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddRasterOutputArgs(bool hiddenForCLI)

{

    m_outputFormatArg =

        &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,

                             /* bGDALGAllowed = */ true)

              .AddMetadataItem(

                  GAAMDI_REQUIRED_CAPABILITIES,

                  {GDAL_DCAP_RASTER,

                   m_constructorOptions.outputFormatCreateCapability.c_str()})

              .SetHiddenForCLI(hiddenForCLI))

             .SetAvailableInPipelineStep(false);

    AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,

                        /* positionalAndRequired = */ !hiddenForCLI,

                        m_constructorOptions.outputDatasetHelpMsg.c_str())

        .SetHiddenForCLI(hiddenForCLI)

        .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)

        .SetAvailableInPipelineStep(false);

    AddCreationOptionsArg(&m_creationOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    constexpr const char *MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND =

        "overwrite-append";

    AddOverwriteArg(&m_overwrite)

        .SetHiddenForCLI(hiddenForCLI)

        .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)

        .SetAvailableInPipelineStep(false);

    AddArg(GDAL_ARG_NAME_APPEND, 0,

           _("Append as a subdataset to existing output"), &m_appendRaster)

        .SetDefault(false)

        .SetHiddenForCLI(hiddenForCLI)

        .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)

        .SetAvailableInPipelineStep(false);

}

/************************************************************************/

/*     GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()      */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()

{

    AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)

        .SetMinCount(0)

        .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

        .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

        .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)

        .SetHidden();

}

/************************************************************************/

/*           GDALPipelineStepAlgorithm::AddVectorInputArgs()            */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddVectorInputArgs(bool hiddenForCLI)

{

    AddInputFormatsArg(&m_inputFormats)

        .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, {GDAL_DCAP_VECTOR})

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOpenOptionsArg(&m_openOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    auto &datasetArg =

        AddInputDatasetArg(

            &m_inputDataset, GDAL_OF_VECTOR, false,

            m_constructorOptions.inputDatasetHelpMsg.empty() &&

                    m_constructorOptions.inputDatasetMaxCount == 1

                ? "Input vector dataset"

                : m_constructorOptions.inputDatasetHelpMsg.c_str())

            .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)

            .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

            .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)

            .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

            .SetHiddenForCLI(hiddenForCLI)

            .SetAvailableInPipelineStep(false);

    if (!m_constructorOptions.inputDatasetAlias.empty())

        datasetArg.AddAlias(m_constructorOptions.inputDatasetAlias);

    if (!m_constructorOptions.inputDatasetMetaVar.empty())

        datasetArg.SetMetaVar(m_constructorOptions.inputDatasetMetaVar);

    if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)

        datasetArg.SetPositional();

    if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)

        datasetArg.SetRequired();

    if (m_constructorOptions.addInputLayerNameArgument)

    {

        auto &layerArg = AddArg(GDAL_ARG_NAME_INPUT_LAYER, 'l',

                                _("Input layer name(s)"), &m_inputLayerNames)

                             .AddAlias("layer")

                             .SetHiddenForCLI(hiddenForCLI)

                             .SetAvailableInPipelineStep(false);

        SetAutoCompleteFunctionForLayerName(layerArg, datasetArg);

    }

}

/************************************************************************/

/*           GDALPipelineStepAlgorithm::AddVectorOutputArgs()           */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddVectorOutputArgs(

    bool hiddenForCLI, bool shortNameOutputLayerAllowed)

{

    AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,

                       /* bGDALGAllowed = */ true)

        .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,

                         {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE})

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOutputOpenOptionsArg(&m_outputOpenOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    auto &outputDatasetArg =

        AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,

                            /* positionalAndRequired = */ false)

            .SetHiddenForCLI(hiddenForCLI)

            .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)

            .SetAvailableInPipelineStep(false);

    if (!hiddenForCLI)

        outputDatasetArg.SetPositional();

    if (!hiddenForCLI && m_constructorOptions.outputDatasetRequired)

        outputDatasetArg.SetRequired();

    AddCreationOptionsArg(&m_creationOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddLayerCreationOptionsArg(&m_layerCreationOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOverwriteArg(&m_overwrite)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    GDALInConstructionAlgorithmArg *updateArg = nullptr;

    if (m_constructorOptions.addUpdateArgument)

    {

        updateArg = &AddUpdateArg(&m_update)

                         .SetHiddenForCLI(hiddenForCLI)

                         .SetAvailableInPipelineStep(false);

    }

    if (m_constructorOptions.addOverwriteLayerArgument)

    {

        AddOverwriteLayerArg(&m_overwriteLayer)

            .SetHiddenForCLI(hiddenForCLI)

            .SetAvailableInPipelineStep(false);

    }

    constexpr const char *MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT =

        "append-upsert";

    if (m_constructorOptions.addAppendLayerArgument)

    {

        AddAppendLayerArg(&m_appendLayer)

            .SetHiddenForCLI(hiddenForCLI)

            .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)

            .SetAvailableInPipelineStep(false);

    }

    if (m_constructorOptions.addUpsertArgument)

    {

        AddArg("upsert", 0, _("Upsert features (implies 'append')"), &m_upsert)

            .SetHiddenForCLI(hiddenForCLI)

            .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)

            .SetAvailableInPipelineStep(false)

            .AddAction(

                [updateArg, this]()

                {

                    if (m_upsert && updateArg)

                        updateArg->Set(true);

                })

            .SetCategory(GAAC_ADVANCED);

    }

    if (m_constructorOptions.addOutputLayerNameArgument)

    {

        AddOutputLayerNameArg(hiddenForCLI, shortNameOutputLayerAllowed);

    }

    if (m_constructorOptions.addSkipErrorsArgument)

    {

        AddArg("skip-errors", 0, _("Skip errors when writing features"),

               &m_skipErrors)

            .AddHiddenAlias("skip-failures")  // For ogr2ogr nostalgic people

            .SetAvailableInPipelineStep(false);

    }

    if (m_constructorOptions.addNoCreateEmptyLayersArgument)

    {

        AddArg("no-create-empty-layers", 0,

               _("Avoid creating layers to which no features will be written"),

               &m_noCreateEmptyLayers)

            .SetAvailableInPipelineStep(false);

    }

}

/************************************************************************/

/*          GDALPipelineStepAlgorithm::AddOutputLayerNameArg()          */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddOutputLayerNameArg(

    bool hiddenForCLI, bool shortNameOutputLayerAllowed)

{

    AddArg(GDAL_ARG_NAME_OUTPUT_LAYER, shortNameOutputLayerAllowed ? 'l' : 0,

           _("Output layer name"),

           &m_outputLayerName)

        .AddHiddenAlias("nln")  // For ogr2ogr nostalgic people

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(

            m_constructorOptions.outputLayerNameAvailableInPipelineStep);

}

/************************************************************************/

/*      GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()       */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()

{

    AddInputDatasetArg(&m_inputDataset, GDAL_OF_MULTIDIM_RASTER, false)

        .SetMinCount(0)

        .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

        .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

        .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)

        .SetHidden();

}

/************************************************************************/

/*            GDALPipelineStepAlgorithm::AddMdimInputArgs()             */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddMdimInputArgs(bool openForMixedMdimVector,

                                                 bool hiddenForCLI,

                                                 bool acceptRaster)

{

    AddInputFormatsArg(&m_inputFormats)

        .AddMetadataItem(

            GAAMDI_REQUIRED_CAPABILITIES,

            openForMixedMdimVector

                ? std::vector<

                      std::string>{acceptRaster

                                       ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER

                                       : GDAL_DCAP_MULTIDIM_RASTER,

                                   GDAL_DCAP_VECTOR}

                : std::vector<

                      std::string>{acceptRaster

                                       ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER

                                       : GDAL_DCAP_MULTIDIM_RASTER})

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOpenOptionsArg(&m_openOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    auto &arg =

        AddInputDatasetArg(&m_inputDataset,

                           (acceptRaster ? GDAL_OF_RASTER : 0) |

                               (openForMixedMdimVector

                                    ? (GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VECTOR)

                                    : GDAL_OF_MULTIDIM_RASTER),

                           false,

                           m_constructorOptions.inputDatasetHelpMsg.c_str())

            .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)

            .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)

            .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)

            .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)

            .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)

            .SetHiddenForCLI(hiddenForCLI)

            .SetAvailableInPipelineStep(false);

    if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)

        arg.SetPositional();

    if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)

        arg.SetRequired();

    if (!m_constructorOptions.inputDatasetAlias.empty())

        arg.AddAlias(m_constructorOptions.inputDatasetAlias);

}

/************************************************************************/

/*            GDALPipelineStepAlgorithm::AddMdimOutputArgs()            */

/************************************************************************/

void GDALPipelineStepAlgorithm::AddMdimOutputArgs(bool hiddenForCLI)

{

    m_outputFormatArg =

        &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,

                             /* bGDALGAllowed = */ true)

              .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,

                               {GDAL_DCAP_CREATE_MULTIDIMENSIONAL})

              .SetHiddenForCLI(hiddenForCLI))

             .SetAvailableInPipelineStep(false);

    AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER,

                        /* positionalAndRequired = */ !hiddenForCLI,

                        m_constructorOptions.outputDatasetHelpMsg.c_str())

        .SetHiddenForCLI(hiddenForCLI)

        .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)

        .SetAvailableInPipelineStep(false);

    AddCreationOptionsArg(&m_creationOptions)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

    AddOverwriteArg(&m_overwrite)

        .SetHiddenForCLI(hiddenForCLI)

        .SetAvailableInPipelineStep(false);

}

/************************************************************************/

/*                 GDALPipelineStepAlgorithm::RunImpl()                 */

/************************************************************************/

bool GDALPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,

                                        void *pProgressData)

{

    if (m_standaloneStep)

    {

        std::unique_ptr<GDALPipelineStepAlgorithm> readAlg;

        if (GetInputType() == GDAL_OF_RASTER)

            readAlg = std::make_unique<GDALRasterReadAlgorithm>();

        else

            readAlg = std::make_unique<GDALVectorReadAlgorithm>();

        for (auto &arg : readAlg->GetArgs())

        {

            auto stepArg = GetArg(arg->GetName());

            if (stepArg && stepArg->IsExplicitlySet() &&

                stepArg->GetType() == arg->GetType())

            {

                arg->SetSkipIfAlreadySet(true);

                arg->SetFrom(*stepArg);

            }

        }

        std::unique_ptr<GDALPipelineStepAlgorithm> writeAlg;

        if (GetOutputType() == GDAL_OF_RASTER)

        {

            if (GetName() == GDALRasterInfoAlgorithm::NAME)

                writeAlg = std::make_unique<GDALRasterInfoAlgorithm>();

            else if (GetName() == GDALRasterCompareAlgorithm::NAME)

                writeAlg = std::make_unique<GDALRasterCompareAlgorithm>();

            else

                writeAlg = std::make_unique<GDALRasterWriteAlgorithm>();

        }

        else

        {

            if (GetName() == GDALVectorInfoAlgorithm::NAME)

                writeAlg = std::make_unique<GDALVectorInfoAlgorithm>();

            else

                writeAlg = std::make_unique<GDALVectorWriteAlgorithm>();

        }

        for (auto &arg : writeAlg->GetArgs())

        {

            auto stepArg = GetArg(arg->GetName());

            if (stepArg && stepArg->IsExplicitlySet() &&

                stepArg->GetType() == arg->GetType())

            {

                arg->SetSkipIfAlreadySet(true);

                arg->SetFrom(*stepArg);

            }

        }

        const bool bIsStreaming = m_format == "stream";

        // Already checked by GDALAlgorithm::Run()

        CPLAssert(!m_executionForStreamOutput || bIsStreaming);

        bool ret = false;

        if (!m_outputVRTCompatible &&

            (EQUAL(m_format.c_str(), "VRT") ||

             (m_format.empty() &&

              EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str())

                        .c_str(),

                    "VRT"))))

        {

            ReportError(CE_Failure, CPLE_NotSupported,

                        "VRT output is not supported. Consider using the "

                        "GDALG driver instead (files with .gdalg.json "

                        "extension)");

        }

        else if (readAlg->Run())

        {

            auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);

            const bool bOutputSpecified =

                outputArg && outputArg->IsExplicitlySet() &&

                (outputArg->GetType() == GAAT_DATASET ||

                 outputArg->GetType() == GAAT_DATASET_LIST);

            m_inputDataset.clear();

            m_inputDataset.resize(1);

            m_inputDataset[0].Set(readAlg->m_outputDataset.GetDatasetRef());

            if (bOutputSpecified)

                m_outputDataset.Set(nullptr);

            std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>

                pScaledData(nullptr, GDALDestroyScaledProgress);

            const bool bCanHandleNextStep =

                !bIsStreaming && CanHandleNextStep(writeAlg.get());

            GDALPipelineStepRunContext stepCtxt;

            if (pfnProgress && GetName() == GDALRasterCompareAlgorithm::NAME)

            {

                stepCtxt.m_pfnProgress = pfnProgress;

                stepCtxt.m_pProgressData = pProgressData;

            }

            else if (pfnProgress &&

                     (bCanHandleNextStep || !IsNativelyStreamingCompatible()))

            {

                pScaledData.reset(GDALCreateScaledProgress(

                    0.0, bIsStreaming || bCanHandleNextStep ? 1.0 : 0.5,

                    pfnProgress, pProgressData));

                stepCtxt.m_pfnProgress =

                    pScaledData ? GDALScaledProgress : nullptr;

                stepCtxt.m_pProgressData = pScaledData.get();

            }

            if (bCanHandleNextStep)

                stepCtxt.m_poNextUsableStep = writeAlg.get();

            if (RunPreStepPipelineValidations() && RunStep(stepCtxt))

            {

                if (bCanHandleNextStep || !bOutputSpecified)

                {

                    ret = true;

                }

                else

                {

                    writeAlg->m_outputVRTCompatible = m_outputVRTCompatible;

                    writeAlg->m_quiet = m_quiet;

                    std::vector<GDALArgDatasetValue> inputDataset(1);

                    inputDataset[0].Set(m_outputDataset.GetDatasetRef());

                    auto inputArg = writeAlg->GetArg(GDAL_ARG_NAME_INPUT);

                    CPLAssert(inputArg);

                    inputArg->Set(std::move(inputDataset));

                    if (pfnProgress)

                    {

                        pScaledData.reset(GDALCreateScaledProgress(

                            IsNativelyStreamingCompatible() ? 0.0 : 0.5, 1.0,

                            pfnProgress, pProgressData));

                    }

                    stepCtxt.m_pfnProgress =

                        pScaledData ? GDALScaledProgress : nullptr;

                    stepCtxt.m_pProgressData = pScaledData.get();

                    if (writeAlg->ValidateArguments() &&

                        writeAlg->RunStep(stepCtxt))

                    {

                        if (pfnProgress)

                            pfnProgress(1.0, "", pProgressData);

                        m_outputDataset.Set(

                            writeAlg->m_outputDataset.GetDatasetRef());

                        ret = true;

                    }

                }

            }

        }

        return ret;

    }

    else

    {

        GDALPipelineStepRunContext stepCtxt;

        stepCtxt.m_pfnProgress = pfnProgress;

        stepCtxt.m_pProgressData = pProgressData;

        return RunPreStepPipelineValidations() && RunStep(stepCtxt);

    }

}

/************************************************************************/

/*                          SetInputDataset()                           */

/************************************************************************/

void GDALPipelineStepAlgorithm::SetInputDataset(GDALDataset *poDS)

{

    auto arg = GetArg(GDAL_ARG_NAME_INPUT);

    if (arg)

    {

        auto &val = arg->Get<std::vector<GDALArgDatasetValue>>();

        val.resize(1);

        val[0].Set(poDS);

        arg->NotifyValueSet();

        arg->SetSkipIfAlreadySet();

    }

}

/************************************************************************/

/*                         ProcessGDALGOutput()                         */

/************************************************************************/

GDALAlgorithm::ProcessGDALGOutputRet

GDALPipelineStepAlgorithm::ProcessGDALGOutput()

{

    if (m_standaloneStep)

    {

        return GDALAlgorithm::ProcessGDALGOutput();

    }

    else

    {

        // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might

        // actually detect a GDALG output request and process it.

        return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;

    }

}

/************************************************************************/

/*        GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()         */

/************************************************************************/

bool GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()

{

    if (m_standaloneStep)

    {

        return GDALAlgorithm::CheckSafeForStreamOutput();

    }

    else

    {

        // The check is actually done in

        // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()

        // so return true for now.

        return true;

    }

}

/************************************************************************/

/*                GDALPipelineStepAlgorithm::Finalize()                 */

/************************************************************************/

bool GDALPipelineStepAlgorithm::Finalize()

{

    bool ret = GDALAlgorithm::Finalize();

    for (auto &argValue : m_inputDataset)

        ret = argValue.Close() && ret;

    ret = m_outputDataset.Close() && ret;

    return ret;

}

/************************************************************************/

/*               CreateDatasetSingleOutputLayerIfNeeded()               */

/************************************************************************/

bool GDALPipelineStepAlgorithm::CreateDatasetSingleOutputLayerIfNeeded(

    GDALPipelineStepRunContext &ctxt, const std::string &defaultLayerName,

    GDALDataset *&poDstDS, bool &bTemporaryFile,

    std::unique_ptr<GDALDataset> &poNewRetDS, std::string &outputLayerName,

    OGRLayer *&poDstLayer)

{

    auto poWriteStep = ctxt.m_poNextUsableStep ? ctxt.m_poNextUsableStep : this;

    std::string outputFilename = poWriteStep->GetOutputDataset().GetName();

    poDstDS = poWriteStep->GetOutputDataset().GetDatasetRef();

    poNewRetDS.reset();

    bTemporaryFile = false;

    poDstLayer = nullptr;

    GDALDriver *poDstDriver = nullptr;

    if (!poDstDS)

    {

        auto poDriverManager = GetGDALDriverManager();

        std::string format = poWriteStep->GetOutputFormat();

        if (m_standaloneStep || (ctxt.m_poNextUsableStep && format.empty()))

        {

            if (format.empty())

            {

                const auto aosFormats =

                    CPLStringList(GDALGetOutputDriversForDatasetName(

                        outputFilename.c_str(), GDAL_OF_VECTOR,

                        /* bSingleMatch = */ true,

                        /* bWarn = */ true));

                if (aosFormats.size() != 1)

                {

                    ReportError(CE_Failure, CPLE_AppDefined,

                                "Cannot guess driver for %s",

                                outputFilename.c_str());

                    return false;

                }

                format = aosFormats[0];

            }

        }

        else if (!ctxt.m_poNextUsableStep)

        {

            poDstDriver = poDriverManager->GetDriverByName("GPKG");

            if (poDstDriver)

            {

                bTemporaryFile = true;

                outputFilename =

                    CPLGenerateTempFilenameSafe(

                        std::string("_").append(defaultLayerName).c_str()) +

                    ".gpkg";

                format = "GPKG";

            }

            else

                format = "MEM";

        }

        if (!poDstDriver)

            poDstDriver = poDriverManager->GetDriverByName(format.c_str());

        if (!poDstDriver)

        {

            ReportError(CE_Failure, CPLE_AppDefined, "Cannot find driver %s",

                        format.c_str());

            return false;

        }

        poNewRetDS.reset(poDstDriver->Create(

            outputFilename.c_str(), 0, 0, 0, GDT_Unknown,

            CPLStringList(poWriteStep->GetCreationOptions()).List()));

        if (!poNewRetDS)

            return false;

        if (bTemporaryFile)

            poNewRetDS->MarkSuppressOnClose();

        poDstDS = poNewRetDS.get();

    }

    else

    {

        poDstDriver = poDstDS->GetDriver();

    }

    outputLayerName = poWriteStep->GetOutputLayerName();

    if (outputLayerName.empty() && poDstDS->GetLayerCount() > 1)

    {

        ReportError(CE_Failure, CPLE_AppDefined, "--%s must be specified",

                    GDAL_ARG_NAME_OUTPUT_LAYER);

        return false;

    }

    if (poDstDriver && EQUAL(poDstDriver->GetDescription(), "ESRI Shapefile") &&

        (EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(), "shp") ||

         EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(),

               "shz")) &&

        poDstDS->GetLayerCount() <= 1)

    {

        outputLayerName = CPLGetBasenameSafe(poDstDS->GetDescription());

    }

    if (outputLayerName.empty())

        outputLayerName = defaultLayerName;

    poDstLayer = poDstDS->GetLayerByName(outputLayerName.c_str());

    if (poDstLayer)

    {

        if (poWriteStep->GetOverwriteLayer())

        {

            int iLayer = -1;

            const int nLayerCount = poDstDS->GetLayerCount();

            for (iLayer = 0; iLayer < nLayerCount; iLayer++)

            {

                if (poDstDS->GetLayer(iLayer) == poDstLayer)

                    break;

            }

            if (iLayer < nLayerCount)

            {

                if (poDstDS->DeleteLayer(iLayer) != OGRERR_NONE)

                {

                    ReportError(CE_Failure, CPLE_AppDefined,

                                "Cannot delete layer '%s'",

                                outputLayerName.c_str());

                    return false;

                }

            }

            poDstLayer = nullptr;

        }

        else if (!poWriteStep->GetAppendLayer())

        {

            ReportError(CE_Failure, CPLE_AppDefined,

                        "Layer '%s' already exists. Specify the "

                        "--%s option to overwrite it, or --%s "

                        "to append to it.",

                        outputLayerName.c_str(), GDAL_ARG_NAME_OVERWRITE_LAYER,

                        GDAL_ARG_NAME_APPEND);

            return false;

        }

    }

    else if (poWriteStep->GetAppendLayer() || poWriteStep->GetOverwriteLayer())

    {

        ReportError(CE_Failure, CPLE_AppDefined, "Cannot find layer '%s'",

                    outputLayerName.c_str());

        return false;

    }

    return true;

}

GDALAlgorithmStepRegistry::~GDALAlgorithmStepRegistry() = default;

/************************************************************************/

/*                        GDALPipelineAlgorithm                         */

/************************************************************************/

GDALPipelineAlgorithm::GDALPipelineAlgorithm()

    : GDALAbstractPipelineAlgorithm(

          NAME, DESCRIPTION, HELP_URL,

          ConstructorOptions().SetStandaloneStep(false))

{

    m_supportsStreamedOutput = true;

    AddProgressArg();

    AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,

                       /* positionalAndRequired = */ false)

        .SetMinCount(1)

        .SetMaxCount(INT_MAX)

        .SetHiddenForCLI();

    AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,

                        /* positionalAndRequired = */ false)

        .SetHiddenForCLI()

        .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT);

    AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,

                       /* bGDALGAllowed = */ true)

        .SetHiddenForCLI();

    AddArg("pipeline", 0, _("Pipeline string or filename"), &m_pipeline)

        .SetHiddenForCLI()

        .SetPositional();

    AddOutputStringArg(&m_output).SetHiddenForCLI();

    AddStdoutArg(&m_stdout);

    AllowArbitraryLongNameArgs();

    GDALRasterPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);

    GDALVectorPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);

    m_stepRegistry.Register<GDALExternalRasterOrVectorAlgorithm>();

    m_stepRegistry.Register<GDALRasterAsFeaturesAlgorithm>();

    m_stepRegistry.Register<GDALRasterContourAlgorithm>();

    m_stepRegistry.Register<GDALRasterFootprintAlgorithm>();

    m_stepRegistry.Register<GDALRasterPixelInfoAlgorithm>();

    m_stepRegistry.Register<GDALRasterPolygonizeAlgorithm>();

    m_stepRegistry.Register<GDALRasterZonalStatsAlgorithm>();

    m_stepRegistry.Register<GDALVectorGridAlgorithm>();

    m_stepRegistry.Register<GDALVectorRasterizeAlgorithm>();

}

/************************************************************************/

/*               GDALPipelineAlgorithm::GetUsageForCLI()                */

/************************************************************************/

std::string

GDALPipelineAlgorithm::GetUsageForCLI(bool shortUsage,

                                      const UsageOptions &usageOptions) const

{

    UsageOptions stepUsageOptions;

    stepUsageOptions.isPipelineStep = true;

    if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")

    {

        auto alg = GetStepAlg(m_helpDocCategory);

        if (alg)

        {

            alg->SetCallPath({CPLString(m_helpDocCategory)

                                  .replaceAll(RASTER_SUFFIX, "")

                                  .replaceAll(VECTOR_SUFFIX, "")});

            alg->GetArg("help-doc")->Set(true);

            return alg->GetUsageForCLI(shortUsage, stepUsageOptions);

        }

        else

        {

            fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",

                    m_helpDocCategory.c_str());

            return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",

                              m_helpDocCategory.c_str());

        }

    }