Makefile

@@ -19,6 +19,9 @@ ENVIRONMENT ?= dev
 # Hostname for testing
 CSL_HOSTNAME ?= gpfood.controlshiftlabs.com
 
+# Terraform SA
+TF_EMAIL := terraform@$(PROJECT).iam.gserviceaccount.com
+
 # =============================================================================
 
 RELEASE_SOURCE_BUCKET := global-data-csl-pipeline-source
@@ -57,6 +60,26 @@ lint-js: src/node_modules
 # =============================================================================
 
 #  DEVELOPMENT TARGETS
+set:
+	gcloud config set auth/impersonate_service_account $(TF_EMAIL)
+
+unset:
+	gcloud config unset auth/impersonate_service_account
+
+secrets-export:
+	$(eval SENTRY_DSN=$(shell gcloud secrets versions access latest --secret="sentry_dsn_csl_incoming_webhook"))
+	$(eval CSL_GPFOOD=$(shell gcloud secrets versions access latest --secret="csl_gpfood-controlshiftlabs-com_hmac_dev"))
+
+secrets: set secrets-export unset
+
+pubsub-export:
+	$(eval FULL_TABLE_EXPORTED=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-nightly-read --message "test"))
+	$(eval NOPE_REALTIME=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-realtime-source --message "test"))
+	$(eval NOPE_REALTIME_ALL=$(shell gcloud pubsub topics publish csl-realtime-cosmos-dev-source --message "test"))
+
+pubsub: set pubsub-export unset
+
+secrets-pubsub: set secrets-export pubsub-export unset
 
 src/node_modules:
 ifdef CI
@@ -65,18 +88,18 @@ else
 	cd src && npm install
 endif
 
-dev: src/node_modules
+dev: secrets src/node_modules
 	@echo "Entity: $(ENTITY)"
 	@echo "Env:    $(ENVIRONMENT)"
 	cd src && npm start
 
-debug: src/node_modules
+debug: secrets src/node_modules
 	cd src && npm run debug
 
-test: src/node_modules
+test: secrets-pubsub src/node_modules
 	cd src && npm test
 
-testQuiet: src/node_modules
+testQuiet: secrets-pubsub src/node_modules
 	cd src && npm run testQuiet
 
 snyk: snyk-auth src/node_modules
@@ -91,10 +114,10 @@ snykWizard: src/node_modules
 testWatch: src/node_modules
 	cd src && npm run testWatch
 
-coverage: src/node_modules
+coverage: secrets-pubsub src/node_modules
 	cd src && npm run coverage
 
-coverageWatch: src/node_modules
+coverageWatch: secrets-pubsub src/node_modules
 	cd src && npm run coverageWatch
 
 fossa: src/node_modules
@@ -149,7 +172,7 @@ clean-local-files:
 # =============================================================================
 
 release: terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip
-	gcloud config set auth/impersonate_service_account $(TF_EMAIL_DEV)
+	gcloud config set auth/impersonate_service_account $(TF_EMAIL)
 	gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-$(CI_COMMIT_REF_NAME).zip
 	gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-latest.zip
 

README.md

@@ -20,6 +20,4 @@ Requirements:
 
 Configure max watches to prevent node.js crashes : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
 
-Export your google cloud service account with `export GOOGLE_APPLICATION_CREDENTIALS="PATH"`
-
 running `make dev` will install all npm dependencies and run the node.js express app on http://localhost:8080/

src/index.js

@@ -45,6 +45,10 @@ const TOPIC_REALTIME_ALL =
   process.env.TOPIC_REALTIME_ALL ||
   `projects/${process.env.PROJECT}/topics/csl-realtime-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`;
 
+const IS_IMPERSONATED =
+  process.env.GOOGLE_OAUTH_ACCESS_TOKEN ||
+  process.env.GOOGLE_APPLICATION_CREDENTIALS;
+
 const {SecretManagerServiceClient} = require('@google-cloud/secret-manager');
 
 const crypto = require('crypto');
@@ -69,9 +73,11 @@ async function getSecret(name) {
 const Sentry = require('@sentry/node');
 (async () => {
   Sentry.init({
-    dsn: await getSecret(
+    dsn:
+      process.env.SENTRY_DSN ||
+      (await getSecret(
         `projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest`
-    ),
+      )),
     release: `${appName}@${appVersion}`,
   });
 })();
@@ -167,6 +173,7 @@ app.post('/', async (req, res, next) => {
   // Fetch HMAC PSK from SecretManager, cache for re-use
   sharedSecret =
     sharedSecret ||
+    process.env.CSL_GPFOOD ||
     (await getSecret(
       `projects/${
         process.env.PROJECT
@@ -248,9 +255,20 @@ app.post('/', async function (req, res, _next) {
 
   // NIGHTLY_READ
   if (path === 'data' && action === 'full_table_exported') {
+    if (IS_IMPERSONATED) {
+      if (process.env.FULL_TABLE_EXPORTED) {
+        logger.debug(`SUCCESS ${TOPIC_NIGHTLY}`);
+        return res.send({status: 'OK'});
+      } else {
+        return res
+          .status(400)
+          .send({error: `Publishing to ${TOPIC_NIGHTLY} failed`});
+      }
+    } else {
       return await publishOne(TOPIC_NIGHTLY, req, res);
       // return res.status(200).send({status: 'OK'});
     }
+  }
 
   // INCREMENTAL_READ
   if (path === 'data' && action === 'incremental_table_exported') {
@@ -259,7 +277,18 @@ app.post('/', async function (req, res, _next) {
   }
 
   // REALTIME_READ
+  if (IS_IMPERSONATED) {
+    if (process.env.NOPE_REALTIME && process.env.NOPE_REALTIME_ALL) {
+      logger.debug(`SUCCESS: [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ]`);
+      res.send({status: 'OK'});
+    } else {
+      res.status(400).send({
+        error: `Publishing to [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ] failed`,
+      });
+    }
+  } else {
     await publishRealtime(TOPIC_REALTIME, req, res);
+  }
 });
 
 // Root requests to this endpoint