diff --git a/Makefile b/Makefile index 4d4a813321e22ed619c557e783f0c7a50bb603ae..40969d61a36a0a087e1f2bed665408aef6a9a257 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ ENVIRONMENT ?= dev # Hostname for testing CSL_HOSTNAME ?= gpfood.controlshiftlabs.com +# Terraform SA +TF_EMAIL := terraform@$(PROJECT).iam.gserviceaccount.com + # ============================================================================= RELEASE_SOURCE_BUCKET := global-data-csl-pipeline-source @@ -57,6 +60,26 @@ lint-js: src/node_modules # ============================================================================= # DEVELOPMENT TARGETS +set: + gcloud config set auth/impersonate_service_account $(TF_EMAIL) + +unset: + gcloud config unset auth/impersonate_service_account + +secrets-export: + $(eval SENTRY_DSN=$(shell gcloud secrets versions access latest --secret="sentry_dsn_csl_incoming_webhook")) + $(eval CSL_GPFOOD=$(shell gcloud secrets versions access latest --secret="csl_gpfood-controlshiftlabs-com_hmac_dev")) + +secrets: set secrets-export unset + +pubsub-export: + $(eval FULL_TABLE_EXPORTED=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-nightly-read --message "test")) + $(eval NOPE_REALTIME=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-realtime-source --message "test")) + $(eval NOPE_REALTIME_ALL=$(shell gcloud pubsub topics publish csl-realtime-cosmos-dev-source --message "test")) + +pubsub: set pubsub-export unset + +secrets-pubsub: set secrets-export pubsub-export unset src/node_modules: ifdef CI @@ -65,18 +88,18 @@ else cd src && npm install endif -dev: src/node_modules +dev: secrets src/node_modules @echo "Entity: $(ENTITY)" @echo "Env: $(ENVIRONMENT)" cd src && npm start -debug: src/node_modules +debug: secrets src/node_modules cd src && npm run debug -test: src/node_modules +test: secrets-pubsub src/node_modules cd src && npm test -testQuiet: src/node_modules +testQuiet: secrets-pubsub src/node_modules cd src && npm run testQuiet snyk: snyk-auth src/node_modules @@ -91,10 +114,10 @@ snykWizard: src/node_modules testWatch: src/node_modules cd src && npm run testWatch -coverage: src/node_modules +coverage: secrets-pubsub src/node_modules cd src && npm run coverage -coverageWatch: src/node_modules +coverageWatch: secrets-pubsub src/node_modules cd src && npm run coverageWatch fossa: src/node_modules @@ -149,7 +172,7 @@ clean-local-files: # ============================================================================= release: terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip - gcloud config set auth/impersonate_service_account $(TF_EMAIL_DEV) + gcloud config set auth/impersonate_service_account $(TF_EMAIL) gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-$(CI_COMMIT_REF_NAME).zip gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-latest.zip diff --git a/README.md b/README.md index d8d0409e5b438629708eddc26924042ddb4c7fbe..96027189aaa8f80f8e857cdfa37b48251d3625ad 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,4 @@ Requirements: Configure max watches to prevent node.js crashes : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p` -Export your google cloud service account with `export GOOGLE_APPLICATION_CREDENTIALS="PATH"` - running `make dev` will install all npm dependencies and run the node.js express app on http://localhost:8080/ diff --git a/src/index.js b/src/index.js index 2decf183cd5790fa7b7f36661b5e6a627d1aa311..d0d662e2149f15a3eb6d6fab1005985d822c69aa 100644 --- a/src/index.js +++ b/src/index.js @@ -45,6 +45,10 @@ const TOPIC_REALTIME_ALL = process.env.TOPIC_REALTIME_ALL || `projects/${process.env.PROJECT}/topics/csl-realtime-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`; +const IS_IMPERSONATED = + process.env.GOOGLE_OAUTH_ACCESS_TOKEN || + process.env.GOOGLE_APPLICATION_CREDENTIALS; + const {SecretManagerServiceClient} = require('@google-cloud/secret-manager'); const crypto = require('crypto'); @@ -69,9 +73,11 @@ async function getSecret(name) { const Sentry = require('@sentry/node'); (async () => { Sentry.init({ - dsn: await getSecret( - `projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest` - ), + dsn: + process.env.SENTRY_DSN || + (await getSecret( + `projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest` + )), release: `${appName}@${appVersion}`, }); })(); @@ -167,6 +173,7 @@ app.post('/', async (req, res, next) => { // Fetch HMAC PSK from SecretManager, cache for re-use sharedSecret = sharedSecret || + process.env.CSL_GPFOOD || (await getSecret( `projects/${ process.env.PROJECT @@ -248,8 +255,19 @@ app.post('/', async function (req, res, _next) { // NIGHTLY_READ if (path === 'data' && action === 'full_table_exported') { - return await publishOne(TOPIC_NIGHTLY, req, res); - // return res.status(200).send({status: 'OK'}); + if (IS_IMPERSONATED) { + if (process.env.FULL_TABLE_EXPORTED) { + logger.debug(`SUCCESS ${TOPIC_NIGHTLY}`); + return res.send({status: 'OK'}); + } else { + return res + .status(400) + .send({error: `Publishing to ${TOPIC_NIGHTLY} failed`}); + } + } else { + return await publishOne(TOPIC_NIGHTLY, req, res); + // return res.status(200).send({status: 'OK'}); + } } // INCREMENTAL_READ @@ -259,7 +277,18 @@ app.post('/', async function (req, res, _next) { } // REALTIME_READ - await publishRealtime(TOPIC_REALTIME, req, res); + if (IS_IMPERSONATED) { + if (process.env.NOPE_REALTIME && process.env.NOPE_REALTIME_ALL) { + logger.debug(`SUCCESS: [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ]`); + res.send({status: 'OK'}); + } else { + res.status(400).send({ + error: `Publishing to [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ] failed`, + }); + } + } else { + await publishRealtime(TOPIC_REALTIME, req, res); + } }); // Root requests to this endpoint