From 0dbf243ad35adf5771f5a73113e913e29902bc19 Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Thu, 2 Nov 2023 18:32:26 +0545 Subject: [PATCH 1/8] Adding azure account integration --- .gitignore | 4 +++- shell/sg-cli | 64 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 145d9f6..132f979 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,6 @@ dmypy.json .pyre/ data.json -sg.json \ No newline at end of file +sg.json + +*paylod.json \ No newline at end of file diff --git a/shell/sg-cli b/shell/sg-cli index c50287d..b9216c5 100755 --- a/shell/sg-cli +++ b/shell/sg-cli @@ -120,6 +120,7 @@ Available commands: workflow Manage workflow resources artifacts Manage artifacts aws Manage aws integration + azure Manage azure integration Usage: ./$(basename "$0") [options] -- [payload] @@ -313,16 +314,20 @@ EOF } #}}} -aws_help() { #{{{ +cloud_help() { #{{{ cat < Date: Thu, 2 Nov 2023 18:41:01 +0545 Subject: [PATCH 2/8] Updating Readme --- shell/README.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++- shell/sg-cli | 4 ++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/shell/README.md b/shell/README.md index 1307db6..64c6fca 100644 --- a/shell/README.md +++ b/shell/README.md @@ -138,11 +138,13 @@ Payload will look like the follwing: ``` Example 6: Bulk onboard cloud accounts + +Integrating AWS Accounts ``` ./sg-cli aws integrate --org demo-org -- payload.json ``` -Payload will look like the follwing: +Payload will look like the following for integrating AWS accounts: > It should contain an array of AWS account objects under the key `awsAccounts` ``` { @@ -179,6 +181,60 @@ Payload will look like the follwing: } ``` +Integrating Azure Subscriptions +``` +./sg-cli azure integrate --org demo-org -- payload.json +``` + +Payload will look like the following for integrating Azure Subscriptions: +> It should contain an array of AWS account objects under the key `azureSubscription` +``` +{ + "azureSubscription": [ + { + "ResourceName": "AZURE-DUMMY-101", + "Description": "Dummy Azure Account 101.", + "Settings": { + "kind": "AZURE_STATIC", + "config": [ + { + "armClientSecret": "dummy-client-secret101", + "armClientId":"dummy-client-id101", + "armSubscriptionId":"dummy-subscription-id101", + "armTenantId": "dummy-tenant-id101" + } + ] + }, + "Tags": [ + "azure", + "sg-cli", + "integration" + ] + }, + { + "ResourceName": "AZURE-DUMMY-102", + "Description": "Dummy Azure Account 102.", + "Settings": { + "kind": "AZURE_STATIC", + "config": [ + { + "armClientSecret": "dummy-client-secret102", + "armClientId":"dummy-client-id102", + "armSubscriptionId":"dummy-subscription-id102", + "armTenantId": "dummy-tenant-id102" + } + ] + }, + "Tags": [ + "azure", + "sg-cli", + "integration" + ] + } + ] +} +``` + Example 7: Bulk create workflows with tfstate files ``` ./sg-cli workflow create --bulk --org demo-org --workflow-group demo-grp -- payload.json diff --git a/shell/sg-cli b/shell/sg-cli index b9216c5..0e1e257 100755 --- a/shell/sg-cli +++ b/shell/sg-cli @@ -1269,10 +1269,10 @@ integrate_azure() { #{{{ exit 1 fi - array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("azureAccounts") then .azureAccounts else empty end | length')" + array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("azureSubscription") then .azureSubscription else empty end | length')" ! exists_or_notnull \ "array_length" \ - "Invalid payload. \`azureAccounts\` key missing in global object." && \ + "Invalid payload. \`azureSubscription\` key missing in global object." && \ exit 1 i=0 From dc4da109433aefbdc1cea5c99b6453142131cb39 Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Thu, 2 Nov 2023 18:43:36 +0545 Subject: [PATCH 3/8] Typo correction --- shell/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shell/README.md b/shell/README.md index 64c6fca..b8a978b 100644 --- a/shell/README.md +++ b/shell/README.md @@ -139,7 +139,8 @@ Payload will look like the follwing: Example 6: Bulk onboard cloud accounts -Integrating AWS Accounts +Integrating AWS Accounts: + ``` ./sg-cli aws integrate --org demo-org -- payload.json ``` @@ -181,13 +182,14 @@ Payload will look like the following for integrating AWS accounts: } ``` -Integrating Azure Subscriptions +Integrating Azure Subscriptions: + ``` ./sg-cli azure integrate --org demo-org -- payload.json ``` Payload will look like the following for integrating Azure Subscriptions: -> It should contain an array of AWS account objects under the key `azureSubscription` +> It should contain an array of Azure subscriptions objects under the key `azureSubscription` ``` { "azureSubscription": [ From 4f0a36320b943d1a9aa8b29aef51aef3711cdf10 Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Thu, 2 Nov 2023 19:45:30 +0545 Subject: [PATCH 4/8] Updating readme with rbac aws integration --- shell/README.md | 70 ++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/shell/README.md b/shell/README.md index b8a978b..f3a0040 100644 --- a/shell/README.md +++ b/shell/README.md @@ -149,36 +149,46 @@ Payload will look like the following for integrating AWS accounts: > It should contain an array of AWS account objects under the key `awsAccounts` ``` { - "awsAccounts": [ - { - "ResourceName": "Dummy123", - "Description": "dummy account", - "Settings": { - "kind": "AWS_STATIC", - "config": [ - { - "awsAccessKeyId": "hi-its-me-a-dummy-account", - "awsSecretAccessKey": "keep-your-secrets-safe", - "awsDefaultRegion": "us-east-1" - } - ] - } - }, - { - "ResourceName": "Dummy11345", - "Description": "dummy account", - "Settings": { - "kind": "AWS_STATIC", - "config": [ - { - "awsAccessKeyId": "hi-its-me-a-dummy-account", - "awsSecretAccessKey": "keep-your-secrets-safe", - "awsDefaultRegion": "us-east-1" - } - ] - } - } - ] + "awsAccounts": [ + { + "ResourceName": "AWS-STATIC-101", + "Description": "Dummy AWS Account integration using Access Key.", + "Settings": { + "kind": "AWS_STATIC", + "config": [ + { + "awsAccessKeyId": "dummy-accesskey-id", + "awsSecretAccessKey":"keep-your-secret-safe", + "awsDefaultRegion":"us-east-1" + } + ] + }, + "Tags": [ + "aws", + "sg-cli", + "STATIC" + ] + }, + { + "ResourceName": "AWS-RBAC-101", + "Description": "Dummy AWS Account integration using RBAC.", + "Settings": { + "kind": "AWS_RBAC", + "config": [ + { + "externalId": "demo-org:1234567890", + "durationSeconds":"3600", + "roleArn":"arn:aws:iam::account-id:role/role-name" + } + ] + }, + "Tags": [ + "aws", + "sg-cli", + "RBAC" + ] + } + ] } ``` From 0f60ca33cc5773dfed89fef146003f4573d2b644 Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Mon, 6 Nov 2023 18:58:49 +0545 Subject: [PATCH 5/8] Added VCS integration functionality and refractoring --- shell/README.md | 44 ++++++++++++++++++++++++++++++++------------ shell/sg-cli | 27 +++++++++++++-------------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/shell/README.md b/shell/README.md index f3a0040..305e191 100644 --- a/shell/README.md +++ b/shell/README.md @@ -127,7 +127,7 @@ Example 5: (add new key) ``` ./sg-cli stack create --org demo-org --workflow-group integration-wfgrp --patch-payload '{"custom_key": "custom_value"}' -- payload.json ``` -Payload will look like the follwing: +Payload will look like the following: > new key/value will be added to payload ``` { @@ -137,19 +137,19 @@ Payload will look like the follwing: } ``` -Example 6: Bulk onboard cloud accounts +Example 6: Bulk onboard Cloud accounts -Integrating AWS Accounts: +Integrating Cloud Accounts: ``` -./sg-cli aws integrate --org demo-org -- payload.json +./sg-cli cloud integrate --org demo-org -- payload.json ``` -Payload will look like the following for integrating AWS accounts: -> It should contain an array of AWS account objects under the key `awsAccounts` +Payload will look like the following for integrating accounts: +> It should contain an array of account objects under the key `cloudProvider` ``` { - "awsAccounts": [ + "cloudProvider": [ { "ResourceName": "AWS-STATIC-101", "Description": "Dummy AWS Account integration using Access Key.", @@ -187,22 +187,42 @@ Payload will look like the following for integrating AWS accounts: "sg-cli", "RBAC" ] + }, + { + "ResourceName": "AZURE-DUMMY-101", + "Description": "Dummy Azure Account.", + "Settings": { + "kind": "AZURE_STATIC", + "config": [ + { + "armClientSecret": "SECRET-101", + "armClientId":"cscsdcsdcsdcscs101", + "armSubscriptionId":"sdcscdscsdc101", + "armTenantId": "cscsdcsdcsdcscs101" + } + ] + }, + "Tags": [ + "azure", + "sg-cli", + "integration" + ] } ] } ``` -Integrating Azure Subscriptions: +Integrating VCS Provider: ``` -./sg-cli azure integrate --org demo-org -- payload.json +./sg-cli vcs integrate --org demo-org -- payload.json ``` -Payload will look like the following for integrating Azure Subscriptions: -> It should contain an array of Azure subscriptions objects under the key `azureSubscription` +Payload will look like the following for integrating VCS Provider: +> It should contain an array of Azure subscriptions objects under the key `vcsProvider` ``` { - "azureSubscription": [ + "vcsProvider": [ { "ResourceName": "AZURE-DUMMY-101", "Description": "Dummy Azure Account 101.", diff --git a/shell/sg-cli b/shell/sg-cli index 0e1e257..470ef00 100755 --- a/shell/sg-cli +++ b/shell/sg-cli @@ -1219,7 +1219,7 @@ list_artifacts() { #{{{ #}}} ####################################### -# AWS Integrate +# Cloud / VCS Integrate # Arguments: # org_id # payload @@ -1227,17 +1227,18 @@ list_artifacts() { #{{{ # Write to STDIN/STDERR # if successfull/error. ####################################### -integrate_aws() { #{{{ + +integrate_cloud() { #{{{ return_response= if [ -z "${payload}" ] || [ -z "${org_id}" ]; then err "Parameters --org and payload must be provided" exit 1 fi - array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("awsAccounts") then .awsAccounts else empty end | length')" + array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("cloudProvider") then .cloudProvider else empty end | length')" ! exists_or_notnull \ "array_length" \ - "Invalid payload. \`awsAccounts\` key missing in global object." && \ + "Invalid payload. \`cloudProvider\` key missing in global object." && \ exit 1 i=0 @@ -1262,17 +1263,17 @@ integrate_aws() { #{{{ } #}}} -integrate_azure() { #{{{ +integrate_vcs() { return_response= if [ -z "${payload}" ] || [ -z "${org_id}" ]; then err "Parameters --org and payload must be provided" exit 1 fi - array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("azureSubscription") then .azureSubscription else empty end | length')" + array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("vcsProvider") then .vcsProvider else empty end | length')" ! exists_or_notnull \ "array_length" \ - "Invalid payload. \`azureSubscription\` key missing in global object." && \ + "Invalid payload. \`vcsProvider\` key missing in global object." && \ exit 1 i=0 @@ -1293,9 +1294,10 @@ integrate_azure() { #{{{ warn_reponse "Failed to create integration: $(echo "$object_payload" | jq -r '.ResourceName')" fi i=$(( $i + 1 )) - done + done } + get_workflow_outputs() { #{{{ url="${API_URL}/orgs/${org_id}/wfgrps/${wfgrp_id}/wfs/${wf_id}/outputs" response=$(curl -i -s --http1.1 -X GET \ @@ -1805,14 +1807,11 @@ list_artifacts_parse_response() { #{{{ } #}}} -integrate_aws_parse_response() { #{{{ - # echo "${response}" | jq +integrate_vcs_parse_response() { exit 0 } -#}}} -integrate_azure_parse_response() { #{{{ - # echo "${response}" | jq +integrate_cloud_parse_response() { exit 0 } @@ -2008,7 +2007,7 @@ case "$1" in exit 1 esac ;; - azure|aws) + vcs|cloud) service="$1" case "$2" in integrate) From 1b2362a27c390a8eae5348eb716bb3e7e6db39b3 Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Mon, 6 Nov 2023 21:01:45 +0545 Subject: [PATCH 6/8] Added help menu for vcs integration --- shell/sg-cli | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/shell/sg-cli b/shell/sg-cli index 470ef00..1e03752 100755 --- a/shell/sg-cli +++ b/shell/sg-cli @@ -119,8 +119,8 @@ Available commands: stack Manage stack resources workflow Manage workflow resources artifacts Manage artifacts - aws Manage aws integration - azure Manage azure integration + cloud Manage cloud provider integration + vcs Manage vcs provider integration Usage: ./$(basename "$0") [options] -- [payload] @@ -320,11 +320,8 @@ cloud_help() { #{{{ Configure Cloud integration for Stackguardian platform. Examples: - [Integrate AWS Account] - ./$(basename "$0") aws integrate --org demo-org -- payload.json - - [Integrate Azure Subscription] - ./$(basename "$0") azure integrate --org demo-org -- payload.json + [Integrate Cloud Accounts] + ./$(basename "$0") cloud integrate --org demo-org -- payload.json Sub-commands: integrate Create Cloud integration @@ -341,6 +338,31 @@ EOF } #}}} +vcs_help() { #{{{ + cat < --org org-name -- payload + +Use "./$(basename "$0") options" for a list of global command-line options (appiles to all commands). +EOF +} +#}}} + spinner() { #{{{ spinner_pid=$1 # delay=0.15 @@ -2015,7 +2037,11 @@ case "$1" in shift 2 ;; help | --help | -h) - cloud_help + if [ "$service" = "cloud" ]; then + cloud_help + elif [ "$service" = "vcs" ]; then + vcs_help + fi exit 0 ;; *) From 3ac004031076c5295d39eef12cd284bb6e3dd52d Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Fri, 17 Nov 2023 16:01:18 +0545 Subject: [PATCH 7/8] Created Integrations command --- shell/sg-cli | 102 ++++++++------------------------------------------- 1 file changed, 16 insertions(+), 86 deletions(-) diff --git a/shell/sg-cli b/shell/sg-cli index 1e03752..0283d25 100755 --- a/shell/sg-cli +++ b/shell/sg-cli @@ -119,8 +119,7 @@ Available commands: stack Manage stack resources workflow Manage workflow resources artifacts Manage artifacts - cloud Manage cloud provider integration - vcs Manage vcs provider integration + integrations Manage cloud/vcs provider integration Usage: ./$(basename "$0") [options] -- [payload] @@ -314,49 +313,24 @@ EOF } #}}} -cloud_help() { #{{{ +integrations_help() { #{{{ cat < --org org-name -- payload - -Use "./$(basename "$0") options" for a list of global command-line options (appiles to all commands). -EOF -} -#}}} - -vcs_help() { #{{{ - cat < --org org-name -- payload + ./$(basename "$0") integrations --org org-name -- payload Use "./$(basename "$0") options" for a list of global command-line options (appiles to all commands). EOF @@ -388,6 +362,7 @@ parse_response() { #{{{ # get first status code from response status_code="$(echo "$response" \ | awk '/^HTTP/ {print $2; exit}')" + echo $status_code # actual response data response="$(echo "$response" \ @@ -1249,29 +1224,29 @@ list_artifacts() { #{{{ # Write to STDIN/STDERR # if successfull/error. ####################################### - -integrate_cloud() { #{{{ - return_response= +create_integrations() { + return_response= if [ -z "${payload}" ] || [ -z "${org_id}" ]; then err "Parameters --org and payload must be provided" exit 1 fi - array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("cloudProvider") then .cloudProvider else empty end | length')" + array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("Integrations") then .Integrations else empty end | length')" ! exists_or_notnull \ "array_length" \ - "Invalid payload. \`cloudProvider\` key missing in global object." && \ + "Invalid payload. \`Integrations\` key missing in global object." && \ exit 1 i=0 while [ ${i} -lt "${array_length}" ]; do object_payload=$(echo "${payload}" | jq ".[][$i]") - url="${API_URL}/orgs/${org_id}/integrations" + url="${API_URL}/orgs/${org_id}/integrations/" response=$(curl -i -s --http1.1 -X POST \ -H 'PrincipalId: "cli"' \ -H "Authorization: apikey ${API_TOKEN}" \ -H "Content-Type: application/json" \ --data-raw "${object_payload}" "${url}") + echo $response if parse_response; then exists_or_notnull \ "response" && \ @@ -1283,42 +1258,6 @@ integrate_cloud() { #{{{ i=$(( $i + 1 )) done } -#}}} - -integrate_vcs() { - return_response= - if [ -z "${payload}" ] || [ -z "${org_id}" ]; then - err "Parameters --org and payload must be provided" - exit 1 - fi - - array_length="$(echo "${payload}" | jq 'if type == "object" then . else empty end | if has("vcsProvider") then .vcsProvider else empty end | length')" - ! exists_or_notnull \ - "array_length" \ - "Invalid payload. \`vcsProvider\` key missing in global object." && \ - exit 1 - - i=0 - while [ ${i} -lt "${array_length}" ]; do - object_payload=$(echo "${payload}" | jq ".[][$i]") - url="${API_URL}/orgs/${org_id}/integrations" - response=$(curl -i -s --http1.1 -X POST \ - -H 'PrincipalId: "cli"' \ - -H "Authorization: apikey ${API_TOKEN}" \ - -H "Content-Type: application/json" \ - --data-raw "${object_payload}" "${url}") - if parse_response; then - exists_or_notnull \ - "response" && \ - return_response="${return_response}\n${response}" - info "Created integration: $(echo "$object_payload" | jq -r '.ResourceName')" - else - warn_reponse "Failed to create integration: $(echo "$object_payload" | jq -r '.ResourceName')" - fi - i=$(( $i + 1 )) - done -} - get_workflow_outputs() { #{{{ url="${API_URL}/orgs/${org_id}/wfgrps/${wfgrp_id}/wfs/${wf_id}/outputs" @@ -1829,15 +1768,10 @@ list_artifacts_parse_response() { #{{{ } #}}} -integrate_vcs_parse_response() { +create_integrations_parse_response() { exit 0 } -integrate_cloud_parse_response() { - exit 0 -} - - ####################################### # Other support functions ####################################### @@ -2029,19 +1963,15 @@ case "$1" in exit 1 esac ;; - vcs|cloud) + integrations) service="$1" case "$2" in - integrate) + create) service_option="$2" shift 2 ;; help | --help | -h) - if [ "$service" = "cloud" ]; then - cloud_help - elif [ "$service" = "vcs" ]; then - vcs_help - fi + integrations_help exit 0 ;; *) From 71681d44f7be3a17aa3dacb24553469db48f608a Mon Sep 17 00:00:00 2001 From: Binaya Sharma Date: Fri, 17 Nov 2023 16:05:47 +0545 Subject: [PATCH 8/8] Updating Readme --- shell/README.md | 100 +++++++++++++++++++----------------------------- 1 file changed, 39 insertions(+), 61 deletions(-) diff --git a/shell/README.md b/shell/README.md index 305e191..52da2a6 100644 --- a/shell/README.md +++ b/shell/README.md @@ -137,19 +137,52 @@ Payload will look like the following: } ``` -Example 6: Bulk onboard Cloud accounts +Example 6: Bulk onboard Cloud/VCS accounts -Integrating Cloud Accounts: +Integrating Cloud/VCS Accounts: ``` -./sg-cli cloud integrate --org demo-org -- payload.json +./sg-cli integrations create --org org-name -- payload.json ``` Payload will look like the following for integrating accounts: -> It should contain an array of account objects under the key `cloudProvider` +> It should contain an array of account objects under the key `Integrations` ``` { - "cloudProvider": [ + "Integrations": [ + { + "ResourceName": "gitlab-sgcli", + "Settings": { + "kind": "GITLAB_COM", + "config": [ + { + "gitlabCreds": "secretUsername:secretAccessToken" + } + ] + } + }, + { + "ResourceName": "azuredevops-sgcli", + "Settings": { + "kind": "AZURE_DEVOPS", + "config": [ + { + "azureCreds": "personalAccessToken" + } + ] + } + }, + { + "ResourceName": "bitbucket-sgcli", + "Settings": { + "kind": "BITBUCKET_ORG", + "config": [ + { + "bitbucketCreds": "userName:appPassword" + } + ] + } + }, { "ResourceName": "AWS-STATIC-101", "Description": "Dummy AWS Account integration using Access Key.", @@ -207,62 +240,7 @@ Payload will look like the following for integrating accounts: "sg-cli", "integration" ] - } - ] -} -``` - -Integrating VCS Provider: - -``` -./sg-cli vcs integrate --org demo-org -- payload.json -``` - -Payload will look like the following for integrating VCS Provider: -> It should contain an array of Azure subscriptions objects under the key `vcsProvider` -``` -{ - "vcsProvider": [ - { - "ResourceName": "AZURE-DUMMY-101", - "Description": "Dummy Azure Account 101.", - "Settings": { - "kind": "AZURE_STATIC", - "config": [ - { - "armClientSecret": "dummy-client-secret101", - "armClientId":"dummy-client-id101", - "armSubscriptionId":"dummy-subscription-id101", - "armTenantId": "dummy-tenant-id101" - } - ] - }, - "Tags": [ - "azure", - "sg-cli", - "integration" - ] - }, - { - "ResourceName": "AZURE-DUMMY-102", - "Description": "Dummy Azure Account 102.", - "Settings": { - "kind": "AZURE_STATIC", - "config": [ - { - "armClientSecret": "dummy-client-secret102", - "armClientId":"dummy-client-id102", - "armSubscriptionId":"dummy-subscription-id102", - "armTenantId": "dummy-tenant-id102" - } - ] - }, - "Tags": [ - "azure", - "sg-cli", - "integration" - ] - } + } ] } ```