20 changed files
devops | ||
infrascode/coreinfra | ||
main.bicep + | ||
pipelines/coreinfra | ||
variables | ||
variables-d.yml + | ||
variables-p.yml + | ||
variables.yml + | ||
coreinfra-cd-infrascode.yml + | ||
resources/adf/name_of_company-recenthistory-d-adf-01 | ||
deployment | ||
config-adfcoreinfra-d.csv + | ||
config-adfcoreinfra-p.csv + | ||
config-recenthistory-d.csv + | ||
config-recenthistory-p.csv + | ||
config-tsIngestion-d.csv + | ||
config-tsIngestion-p.csv + | ||
filter-adfcoreinfra.txt + | ||
filter-recenthistory.txt + | ||
filter-tsIngestion.txt + | ||
integrationRuntime | ||
irt-azure-managed.json + | ||
managedVirtualNetwork | ||
default.json + | ||
pipeline | ||
pl_exec_recenthistory.json + | ||
pl_exec_tsingestion.json + | ||
publish_config.json + | ||
templates/bicep | ||
adf.bicep + | ||
Add comment 1 Plus //******************************************************************
Add comment 2 Plus //INPUT PARAMETERS
Add comment 3 Plus //******************************************************************
Add comment 4 Plus param environment string = 'd'
Add comment 5 Plus param azTenantId string = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
Add comment 6 Plus param azResourceGroupName string = '<name_of_development_resource_group>'
Add comment 7 Plus param azResourceGroupLocation string = '<azure_location>'
Add comment 8 Plus param azDataFactoryName string = 'name_of_company-recenthistory-d-adf-01'
Add comment 9 Plus param azDataFactoryRootPath string = '/resources/adf'
Add comment 10 Plus
Add comment 11 Plus //******************************************************************
Add comment 12 Plus //VARIABLES
Add comment 13 Plus //******************************************************************
Add comment 14 Plus var azDevopsRepoConfiguration = {
Add comment 15 Plus accountName: 'techtacofriday'
Add comment 16 Plus projectName: 'RTD' //Research & Technological Development (RTD)
Add comment 17 Plus repositoryName: 'selective-deployment-adf-components'
Add comment 18 Plus collaborationBranch: 'main'
Add comment 19 Plus rootFolder: '${azDataFactoryRootPath}/${azDataFactoryName}'
Add comment 20 Plus type: 'FactoryVSTSConfiguration'
Add comment 21 Plus tenantId: azTenantId
Add comment 22 Plus }
Add comment 23 Plus
Add comment 24 Plus //******************************************************************
Add comment 25 Plus //RESOURCES & MODULES
Add comment 26 Plus //******************************************************************
Add comment 27 Plus targetScope = 'subscription'
Add comment 28 Plus
Add comment 29 Plus resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = {
Add comment 30 Plus name: azResourceGroupName
Add comment 31 Plus location: azResourceGroupLocation
Add comment 32 Plus }
Add comment 33 Plus
Add comment 34 Plus module dataFactory '../../../templates/bicep/adf.bicep' = {
Add comment 35 Plus name: azDataFactoryName
Add comment 36 Plus scope: resourceGroup
Add comment 37 Plus params: {
Add comment 38 Plus dataFactoryName: azDataFactoryName
Add comment 39 Plus azDevopsRepoConfiguration: (environment == 'd') ? azDevopsRepoConfiguration : null
Add comment 40 Plus }
Add comment 41 Plus }
Add comment 42 Plus
variables-d.yml
/devops/pipelines/coreinfra/variables/variables-d.yml+13/devops/pipelines/coreinfra/variables/variables-d.yml
Add comment 1 Plus variables:
Add comment 2 Plus #Azure DevOps (azdo) variables
Add comment 3 Plus azdoServiceConnection: azdo-serviceconnection-arm-d
Add comment 4 Plus azdoEnvironment: bicep_infrastructure-d
Add comment 5 Plus
Add comment 6 Plus #Azure Account (az) variables
Add comment 7 Plus azSubscriptionId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
Add comment 8 Plus azTenantId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
Add comment 9 Plus azResourceGroupName: name_of_development_resource_group
Add comment 10 Plus azResourceGroupLocation: azure_location
Add comment 11 Plus azDataFactoryName: name_of_company-recenthistory-d-adf-01
Add comment 12 Plus azDataFactoryRootPath: /resources/adf
Add comment 13 Plus
variables-p.yml
/devops/pipelines/coreinfra/variables/variables-p.yml+11/devops/pipelines/coreinfra/variables/variables-p.yml
Add comment 1 Plus variables:
Add comment 2 Plus #Azure DevOps (azdo) variables
Add comment 3 Plus azdoServiceConnection: azdo-serviceconnection-arm-p
Add comment 4 Plus azdoEnvironment: bicep_infrastructure-p
Add comment 5 Plus
Add comment 6 Plus #Azure Account (az) variables
Add comment 7 Plus azSubscriptionId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
Add comment 8 Plus azTenantId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
Add comment 9 Plus azResourceGroupName: name_of_production_resource_group
Add comment 10 Plus azResourceGroupLocation: azure_location
Add comment 11 Plus azDataFactoryName: name_of_company-recenthistory-p-adf-01
variables.yml
/devops/pipelines/coreinfra/variables/variables.yml+5/devops/pipelines/coreinfra/variables/variables.yml
Add comment 1 Plus variables:
Add comment 2 Plus alertActionGroupEmailReceivers: '{"alertActionGroupEmailReceivers":{"type":"array","value":[
Add comment 3 Plus {"emailAddress":"svenchio@techtacofriday.com","name":"Hector Sven","useCommonAlertSchema":true}
Add comment 4 Plus ]}}'
Add comment 5 Plus alertActionGroupArmRoleReceivers: '{"alertActionGroupArmRoleReceivers":{"type":"array","value":[]}}'
coreinfra-cd-infrascode.yml
/devops/pipelines/coreinfra/coreinfra-cd-infrascode.yml+141/devops/pipelines/coreinfra/coreinfra-cd-infrascode.yml
Add comment 1 Plus trigger: none
Add comment 2 Plus
Add comment 3 Plus pool:
Add comment 4 Plus vmImage: windows-latest
Add comment 5 Plus
Add comment 6 Plus parameters:
Add comment 7 Plus - name: environment
Add comment 8 Plus type: object
Add comment 9 Plus default: ['d','p']
Add comment 10 Plus - name: dataProduct
Add comment 11 Plus type: string
Add comment 12 Plus default: 'recenthistory'
Add comment 13 Plus
Add comment 14 Plus variables:
Add comment 15 Plus - name: environmentList
Add comment 16 Plus ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
Add comment 17 Plus value: p
Add comment 18 Plus ${{ else }}:
Add comment 19 Plus value: ${{ join(',',parameters.environment) }}
Add comment 20 Plus
Add comment 21 Plus stages:
Add comment 22 Plus - stage: build_stage
Add comment 23 Plus displayName: Build Stage
Add comment 24 Plus variables:
Add comment 25 Plus - template: ./variables/variables-d.yml
Add comment 26 Plus - name: azDataFactoryRootFolder
Add comment 27 Plus value: '${{variables.azDataFactoryRootPath}}/${{variables.azDataFactoryName}}/'
Add comment 28 Plus - name: azDataFactoryResourceId
Add comment 29 Plus value: '/subscriptions/${{variables.azSubscriptionId}}/resourceGroups/${{variables.azResourceGroupName}}/providers/Microsoft.DataFactory/factories/${{variables.azDataFactoryName}}'
Add comment 30 Plus jobs:
Add comment 31 Plus - job: build_iac
Add comment 32 Plus displayName: Build IaC
Add comment 33 Plus steps:
Add comment 34 Plus - checkout: self
Add comment 35 Plus
Add comment 36 Plus - task: AzureCLI@2
Add comment 37 Plus displayName: 'Build infrastructure ARM template from BICEP'
Add comment 38 Plus inputs:
Add comment 39 Plus azureSubscription: ${{variables.azdoServiceConnection}}
Add comment 40 Plus scriptType: 'pscore'
Add comment 41 Plus scriptLocation: 'inlineScript'
Add comment 42 Plus inlineScript: 'az bicep build --file ./devops/infrascode/coreinfra/main.bicep'
Add comment 43 Plus
Add comment 44 Plus - task: PublishPipelineArtifact@1
Add comment 45 Plus displayName: 'Publish built ARM Template (main.json) as a Pipeline Artifact'
Add comment 46 Plus inputs:
Add comment 47 Plus artifactName: 'armDeployment'
Add comment 48 Plus targetPath: '$(Build.SourcesDirectory)/devops/infrascode/coreinfra/main.json'
Add comment 49 Plus
Add comment 50 Plus - task: CopyFiles@2
Add comment 51 Plus displayName: 'Copy ADF source JSON files from repo/branch'
Add comment 52 Plus inputs:
Add comment 53 Plus SourceFolder: '$(Build.SourcesDirectory)/resources/adf/name_of_company-recenthistory-d-adf-01'
Add comment 54 Plus Contents: '**'
Add comment 55 Plus TargetFolder: '$(Build.ArtifactStagingDirectory)'
Add comment 56 Plus OverWrite: true
Add comment 57 Plus CleanTargetFolder: true
Add comment 58 Plus
Add comment 59 Plus - task: PublishPipelineArtifact@1
Add comment 60 Plus displayName: 'Publish ADF source JSON files into Artifact'
Add comment 61 Plus inputs:
Add comment 62 Plus artifact: 'adfSourceCode'
Add comment 63 Plus targetPath: '$(Build.ArtifactStagingDirectory)'
Add comment 64 Plus
Add comment 65 Plus - ${{ each environmentItem in split(variables.environmentList, ',')}}:
Add comment 66 Plus - stage: predeployment_stage_${{environmentItem}}
Add comment 67 Plus displayName: 'Validate IaC before deployment to target ${{environmentItem}}'
Add comment 68 Plus variables:
Add comment 69 Plus - template: ./variables/variables-${{environmentItem}}.yml
Add comment 70 Plus jobs:
Add comment 71 Plus - job: whatif_iac
Add comment 72 Plus displayName: WhatIf IaC
Add comment 73 Plus steps:
Add comment 74 Plus - checkout: none
Add comment 75 Plus
Add comment 76 Plus - stage: deploying_stage_${{environmentItem}}
Add comment 77 Plus displayName: 'Deploy Stage to target environment ${{environmentItem}}'
Add comment 78 Plus variables:
Add comment 79 Plus - template: ./variables/variables-${{environmentItem}}.yml
Add comment 80 Plus jobs:
Add comment 81 Plus - deployment: deploy_iac
Add comment 82 Plus workspace:
Add comment 83 Plus clean: all
Add comment 84 Plus displayName: Deploy to ${{environmentItem}}
Add comment 85 Plus environment: ${{ variables.azdoEnvironment }}
Add comment 86 Plus strategy:
Add comment 87 Plus runOnce:
Add comment 88 Plus deploy:
Add comment 89 Plus steps:
Add comment 90 Plus - checkout: none
Add comment 91 Plus - task: AzureResourceManagerTemplateDeployment@3
Add comment 92 Plus displayName: 'ARM Template deployment'
Add comment 93 Plus inputs:
Add comment 94 Plus deploymentScope: 'Subscription'
Add comment 95 Plus azureResourceManagerConnection: ${{variables.azdoServiceConnection}}
Add comment 96 Plus subscriptionId: ${{variables.azSubscriptionId}}
Add comment 97 Plus location: ${{variables.azResourceGroupLocation}}
Add comment 98 Plus csmFile: '$(Pipeline.Workspace)/armDeployment/main.json'
Add comment 99 Plus overrideParameters: '-environment ${{environmentItem}} -azTenantId ${{variables.azTenantId}} -azResourceGroupName ${{variables.azResourceGroupName}} -azResourceGroupLocation ${{variables.azResourceGroupLocation}} -azDataFactoryName ${{variables.azDataFactoryName}}'
Add comment 100 Plus deploymentOutputs: armDeploymentOutputs
Add comment 101 Plus continueOnError: true
Add comment 102 Plus
Add comment 103 Plus - stage: postdeployment_stage_${{environmentItem}}
Add comment 104 Plus displayName: 'Additional actions over deployed resources ${{environmentItem}}'
Add comment 105 Plus variables:
Add comment 106 Plus - template: ./variables/variables-${{environmentItem}}.yml
Add comment 107 Plus jobs:
Add comment 108 Plus - job: deploye_adf
Add comment 109 Plus displayName: Deploy Azue Data Factory Pipelines
Add comment 110 Plus steps:
Add comment 111 Plus - checkout: none
Add comment 112 Plus
Add comment 113 Plus - task: DownloadPipelineArtifact@2
Add comment 114 Plus displayName: 'Downloading Artifact with ADF source JSON files'
Add comment 115 Plus inputs:
Add comment 116 Plus buildType: 'current'
Add comment 117 Plus artifactName: 'adfSourceCode'
Add comment 118 Plus targetPath: '$(Build.ArtifactStagingDirectory)/adfSourceCode'
Add comment 119 Plus
Add comment 120 Plus - task: PublishADFTask@1
Add comment 121 Plus displayName: 'ADF Selective deployment'
Add comment 122 Plus inputs:
Add comment 123 Plus azureSubscription: ${{variables.azdoServiceConnection}}
Add comment 124 Plus ResourceGroupName: ${{variables.azResourceGroupName}}
Add comment 125 Plus DataFactoryName: ${{variables.azDataFactoryName}}
Add comment 126 Plus DataFactoryCodePath: '$(Build.ArtifactStagingDirectory)/adfSourceCode/' #Artifact generated in build stage
Add comment 127 Plus Location: ${{variables.azResourceGroupLocation}}
Add comment 128 Plus DeleteNotInSource: false #CAUTION WITH THIS PARAMETER
Add comment 129 Plus StageCode: '${{parameters.dataProduct}}-${{environmentItem}}' #If parameter is specified, a CSV file named './deployment/config-{stage}.csv' must exists in repo
Add comment 130 Plus CreateNewInstance: false #Always
Add comment 131 Plus ##Filtering options include: pipeline, dataset, trigger
Add comment 132 Plus ##If char (+/-) is not provided – an inclusion rule (+) would be applied.
Add comment 133 Plus FilteringType: 'FilePath' #The file should use UTF-8 encoding
Add comment 134 Plus FilterTextFile: '$(Build.ArtifactStagingDirectory)/adfSourceCode/deployment/filter-${{parameters.dataProduct}}.txt' #Use when FilteringType: 'FilePath'
Add comment 135 Plus PublishMethod: 'AzResource'
Add comment 136 Plus DoNotStopStartExcludedTriggers: true #Specifies whether excluded triggers will be stopped before deployment (default: false)
Add comment 137 Plus DoNotDeleteExcludedObjects: true #Specifies whether excluded objects can be removed (default: true). Applies when DeleteNotInSource is set to True only
Add comment 138 Plus DeployGlobalParams: true
Add comment 139 Plus
Add comment 140 Plus
Add comment 141 Plus
config-adfcoreinfra-d.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-adfcoreinfra-d.csv+4/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-adfcoreinfra-d.csv
Add comment 1 Plus type,name,path,value
Add comment 2 Plus integrationRuntime,irt-azure-managed,typeProperties.computeProperties.pipelineExternalComputeScaleProperties.numberOfPipelineNodes,2
Add comment 3 Plus
Add comment 4 Plus
config-adfcoreinfra-p.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-adfcoreinfra-p.csv/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-adfcoreinfra-p.csv
config-recenthistory-d.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-recenthistory-d.csv/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-recenthistory-d.csv
config-recenthistory-p.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-recenthistory-p.csv/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-recenthistory-p.csv
config-tsIngestion-d.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-tsIngestion-d.csv/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-tsIngestion-d.csv
config-tsIngestion-p.csv
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-tsIngestion-p.csv/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/config-tsIngestion-p.csv
filter-adfcoreinfra.txt
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-adfcoreinfra.txt/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-adfcoreinfra.txt
filter-recenthistory.txt
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-recenthistory.txt/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-recenthistory.txt
filter-tsIngestion.txt
/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-tsIngestion.txt/resources/adf/name_of_company-recenthistory-d-adf-01/deployment/filter-tsIngestion.txt
irt-azure-managed.json
/resources/adf/name_of_company-recenthistory-d-adf-01/integrationRuntime/irt-azure-managed.json/resources/adf/name_of_company-recenthistory-d-adf-01/integrationRuntime/irt-azure-managed.json
default.json
/resources/adf/name_of_company-recenthistory-d-adf-01/managedVirtualNetwork/default.json/resources/adf/name_of_company-recenthistory-d-adf-01/managedVirtualNetwork/default.json
pl_exec_recenthistory.json
/resources/adf/name_of_company-recenthistory-d-adf-01/pipeline/pl_exec_recenthistory.json/resources/adf/name_of_company-recenthistory-d-adf-01/pipeline/pl_exec_recenthistory.json
pl_exec_tsingestion.json
/resources/adf/name_of_company-recenthistory-d-adf-01/pipeline/pl_exec_tsingestion.json/resources/adf/name_of_company-recenthistory-d-adf-01/pipeline/pl_exec_tsingestion.json
publish_config.json
/resources/adf/name_of_company-recenthistory-d-adf-01/publish_config.json/resources/adf/name_of_company-recenthistory-d-adf-01/publish_config.json