.gitlab-ci.yml

@@ -4,7 +4,6 @@ variables:
   DEVELOPMENT_PROJECT: cosmos-dev-286703
   STAGING_PROJECT: cosmos-staging-291101
   REGION: europe-west1
-  SERVICE_ACCOUNT: $SERVICE_ACCOUNT_DEV
 
 cache:
   key: ${CI_PROJECT_NAME}
@@ -12,9 +11,6 @@ cache:
     - .npm/
     - $CI_PROJECT_DIR/src/node_modules/
 
-include:
-  - template: Code-Quality.gitlab-ci.yml
-
 stages:
   - build
   - test
@@ -33,10 +29,12 @@ stages:
   image:
     name: greenpeaceinternational/gitlab-data-builder:latest
   before_script:
-    - gcloud auth activate-service-account --key-file $SERVICE_ACCOUNT
-    - export GOOGLE_APPLICATION_CREDENTIALS=$SERVICE_ACCOUNT
+    - gcloud auth activate-service-account --key-file $SERVICE_ACCOUNT_DEV
+    - export GOOGLE_APPLICATION_CREDENTIALS=$SERVICE_ACCOUNT_DEV
     - gcloud config set project $PROJECT
     - gcloud config set functions/region $REGION
+    - gcloud config set auth/impersonate_service_account $TF_EMAIL_DEV
+    - gcloud config set core/verbosity "error"
     - npm config set cache $CI_PROJECT_DIR/.npm --global
 
 build:
@@ -114,21 +112,6 @@ test-fossa:
 #   script:
 #       - /usr/share/dependency-check/bin/dependency-check.sh --project "csl-webhook" --out . --scan . --enableExperimental --failOnCVSS 7
 
-code_quality:
-  rules:
-    - if: "$CODE_QUALITY_DISABLED"
-      when: never
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH == "develop"'
-  tags:
-    - Global Data
-  stage: analysis
-  needs: []
-  artifacts:
-    expire_in: 6 months
-    paths:
-      - gl-code-quality-report.json
-
 sonarqube:
   rules:
     - if: "$CODE_QUALITY_DISABLED"

Makefile

@@ -19,6 +19,9 @@ ENVIRONMENT ?= dev
 # Hostname for testing
 CSL_HOSTNAME ?= gpfood.controlshiftlabs.com
 
+# Terraform SA
+TF_EMAIL := terraform@$(PROJECT).iam.gserviceaccount.com
+
 # =============================================================================
 
 RELEASE_SOURCE_BUCKET := global-data-csl-pipeline-source
@@ -57,6 +60,26 @@ lint-js: src/node_modules
 # =============================================================================
 
 #  DEVELOPMENT TARGETS
+set:
+	gcloud config set auth/impersonate_service_account $(TF_EMAIL)
+
+unset:
+	gcloud config unset auth/impersonate_service_account
+
+secrets-export:
+	$(eval SENTRY_DSN=$(shell gcloud secrets versions access latest --secret="sentry_dsn_csl_incoming_webhook"))
+	$(eval CSL_GPFOOD=$(shell gcloud secrets versions access latest --secret="csl_gpfood-controlshiftlabs-com_hmac_dev"))
+
+secrets: set secrets-export unset
+
+pubsub-export:
+	$(eval FULL_TABLE_EXPORTED=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-nightly-read --message "test"))
+	$(eval NOPE_REALTIME=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-realtime-source --message "test"))
+	$(eval NOPE_REALTIME_ALL=$(shell gcloud pubsub topics publish csl-realtime-cosmos-dev-source --message "test"))
+
+pubsub: set pubsub-export unset
+
+secrets-pubsub: set secrets-export pubsub-export unset
 
 src/node_modules:
 ifdef CI
@@ -65,18 +88,18 @@ else
 	cd src && npm install
 endif
 
-dev: src/node_modules
+dev: secrets src/node_modules
 	@echo "Entity: $(ENTITY)"
 	@echo "Env:    $(ENVIRONMENT)"
 	cd src && npm start
 
-debug: src/node_modules
+debug: secrets src/node_modules
 	cd src && npm run debug
 
-test: src/node_modules
+test: secrets-pubsub src/node_modules
 	cd src && npm test
 
-testQuiet: src/node_modules
+testQuiet: secrets-pubsub src/node_modules
 	cd src && npm run testQuiet
 
 snyk: snyk-auth src/node_modules
@@ -91,10 +114,10 @@ snykWizard: src/node_modules
 testWatch: src/node_modules
 	cd src && npm run testWatch
 
-coverage: src/node_modules
+coverage: secrets-pubsub src/node_modules
 	cd src && npm run coverage
 
-coverageWatch: src/node_modules
+coverageWatch: secrets-pubsub src/node_modules
 	cd src && npm run coverageWatch
 
 fossa: src/node_modules
@@ -149,6 +172,7 @@ clean-local-files:
 # =============================================================================
 
 release: terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip
+	gcloud config set auth/impersonate_service_account $(TF_EMAIL)
 	gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-$(CI_COMMIT_REF_NAME).zip
 	gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-latest.zip
 

README.md

@@ -20,6 +20,4 @@ Requirements:
 
 Configure max watches to prevent node.js crashes : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
 
-Export your google cloud service account with `export GOOGLE_APPLICATION_CREDENTIALS="PATH"`
-
 running `make dev` will install all npm dependencies and run the node.js express app on http://localhost:8080/

src/.snyk

 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
 version: v1.14.1
-ignore: {}
+# ignores vulnerabilities until expiry date; change duration by modifying expiry date
+ignore:
+  SNYK-JS-Y18N-1021887:
+    - '@google-cloud/pubsub > google-gax > @grpc/grpc-js > @grpc/proto-loader > yargs > y18n':
+        reason: None given
+        expires: '2021-01-06T04:04:27.486Z'
+    - '@google-cloud/secret-manager > google-gax > @grpc/grpc-js > @grpc/proto-loader > yargs > y18n':
+        reason: None given
+        expires: '2021-01-06T04:04:27.486Z'
+    - snyk > snyk-config > nconf > yargs > y18n:
+        reason: None given
+        expires: '2021-01-06T04:04:27.486Z'
 patch: {}

src/index.js

@@ -3,7 +3,7 @@
 'use strict';
 
 const appName = 'csl-incoming-webhook';
-const appVersion = '2.3.0';
+const appVersion = '3.0.0';
 
 const COSMOS_API_VERSION = 'v1';
 
@@ -45,6 +45,10 @@ const TOPIC_REALTIME_ALL =
   process.env.TOPIC_REALTIME_ALL ||
   `projects/${process.env.PROJECT}/topics/csl-realtime-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`;
 
+const IS_IMPERSONATED =
+  process.env.GOOGLE_OAUTH_ACCESS_TOKEN ||
+  process.env.GOOGLE_APPLICATION_CREDENTIALS;
+
 const {SecretManagerServiceClient} = require('@google-cloud/secret-manager');
 
 const crypto = require('crypto');
@@ -69,9 +73,11 @@ async function getSecret(name) {
 const Sentry = require('@sentry/node');
 (async () => {
   Sentry.init({
-    dsn: await getSecret(
-      `projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest`
-    ),
+    dsn:
+      process.env.SENTRY_DSN ||
+      (await getSecret(
+        `projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest`
+      )),
     release: `${appName}@${appVersion}`,
   });
 })();
@@ -167,6 +173,7 @@ app.post('/', async (req, res, next) => {
   // Fetch HMAC PSK from SecretManager, cache for re-use
   sharedSecret =
     sharedSecret ||
+    process.env.CSL_GPFOOD ||
     (await getSecret(
       `projects/${
         process.env.PROJECT
@@ -248,8 +255,19 @@ app.post('/', async function (req, res, _next) {
 
   // NIGHTLY_READ
   if (path === 'data' && action === 'full_table_exported') {
-    return await publishOne(TOPIC_NIGHTLY, req, res);
-    // return res.status(200).send({status: 'OK'});
+    if (IS_IMPERSONATED) {
+      if (process.env.FULL_TABLE_EXPORTED) {
+        logger.debug(`SUCCESS ${TOPIC_NIGHTLY}`);
+        return res.send({status: 'OK'});
+      } else {
+        return res
+          .status(400)
+          .send({error: `Publishing to ${TOPIC_NIGHTLY} failed`});
+      }
+    } else {
+      return await publishOne(TOPIC_NIGHTLY, req, res);
+      // return res.status(200).send({status: 'OK'});
+    }
   }
 
   // INCREMENTAL_READ
@@ -259,7 +277,18 @@ app.post('/', async function (req, res, _next) {
   }
 
   // REALTIME_READ
-  await publishRealtime(TOPIC_REALTIME, req, res);
+  if (IS_IMPERSONATED) {
+    if (process.env.NOPE_REALTIME && process.env.NOPE_REALTIME_ALL) {
+      logger.debug(`SUCCESS: [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ]`);
+      res.send({status: 'OK'});
+    } else {
+      res.status(400).send({
+        error: `Publishing to [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ] failed`,
+      });
+    }
+  } else {
+    await publishRealtime(TOPIC_REALTIME, req, res);
+  }
 });
 
 // Root requests to this endpoint

src/package.json

 {
   "name": "csl-webhook",
-  "version": "2.3.0",
+  "version": "3.0.0",
   "description": "ControlShiftLabs Webhook Event Handler",
   "main": "index.js",
   "engines": {
@@ -13,7 +13,7 @@
     "@google-cloud/secret-manager": "^3.2.0",
     "@google-cloud/trace-agent": "^5.1.1",
     "@sentry/node": "^5.27.0",
-    "snyk": "^1.419.1",
+    "snyk": "^1.425.4",
     "winston": "^3.3.3"
   },
   "devDependencies": {
@@ -52,4 +52,4 @@
   "author": "Raymond Walker",
   "license": "MIT",
   "snyk": true
-}
+}
\ No newline at end of file