Merge branch 'release/v2.3.0'

.gitlab-ci.yml

 ---
 variables:
-  PROJECT: global-data-resources
-  PRODUCTION_PROJECT: global-data-resources
-  TESTING_PROJECT: global-data-resources
+  PROJECT: cosmos-dev-286703
+  DEVELOPMENT_PROJECT: cosmos-dev-286703
+  STAGING_PROJECT: cosmos-staging-291101
   REGION: europe-west1
 
 cache:
@@ -48,7 +48,7 @@ build:
   <<: *only_default
   stage: build
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
   script:
     - time make src/node_modules
     - make lint
@@ -62,19 +62,17 @@ test-jest:
   <<: *only_default
   stage: test
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
   script:
     - make test
 
-  allow_failure: true
-
 test-snyk:
   <<: *make_dependencies
   <<: *only_default
   stage: test
   retry: 2
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
   script:
     - make snyk
 
@@ -83,7 +81,7 @@ test-coverage:
   <<: *only_default
   stage: test
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
   artifacts:
     name: "$CI_JOB_NAME:coverage"
     expire_in: 1 weeks
@@ -99,7 +97,7 @@ test-fossa:
   <<: *only_default
   stage: test
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
   script:
     - make fossa
 
@@ -136,7 +134,7 @@ deploy-dev:
   <<: *make_dependencies
   stage: deploy
   variables:
-    PROJECT: $TESTING_PROJECT
+    PROJECT: $DEVELOPMENT_PROJECT
     ENVIRONMENT: dev
   environment:
     name: testing
@@ -153,7 +151,7 @@ deploy-dev:
 #   <<: *make_dependencies
 #   stage: deploy
 #   variables:
-#     PROJECT: $PRODUCTION_PROJECT
+#     PROJECT: $DEVELOPMENT_PROJECT
 #     ENVIRONMENT: prod
 #   environment:
 #     name: testing

README.md

@@ -2,20 +2,23 @@
 
 Responsible for authenticating CSL webhook events, parsing the payload and forwarding the request to PubSub topics.
 
+The repository deploys a cloud function to gcp using a nodejs runtime and is configured to receive http post requests from CSL webhook events, these requests are then authenticated via hmac/sha256 and finally the Data is published to GCP Pub/Sub topics.
+
 ## Release process:
 
 1. `NEW_RELEASE=v1.2.3 make release`
 1. `git push --all --follow-tags`
 
-## Running Dev Environment:
+## Running Dev Environment on Linux:
 
 Requirements:
 
 1. Node.js >= 10
 2. npm >= 6.14.6
 3. GCloud Service Account .json File
+4. Make
 
-Fixing the watch failure : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
+Configure max watches to prevent node.js crashes : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
 
 Export your google cloud service account with `export GOOGLE_APPLICATION_CREDENTIALS="PATH"`
 

src/.snyk

 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
 version: v1.14.1
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-JS-JSONBIGINT-608659:
-    - '@google-cloud/pubsub > @opentelemetry/tracing > @opentelemetry/resources > gcp-metadata > json-bigint':
-        reason: None given
-        expires: '2020-10-13T23:53:27.051Z'
-# patches apply the minimum changes required to fix a vulnerability
-patch:
-  SNYK-JS-LODASH-567746:
-    - snyk > @snyk/dep-graph > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > inquirer > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-config > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-mvn-plugin > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-nodejs-lockfile-parser > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-nuget-plugin > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - winston > async > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/dep-graph > graphlib > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-go-plugin > graphlib > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-nodejs-lockfile-parser > graphlib > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-nuget-plugin > dotnet-deps-parser > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > snyk-php-plugin > @snyk/composer-lockfile-parser > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > graphlib > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/ruby-semver > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - snyk > lodash:
-        patched: '2020-05-01T03:08:53.854Z'
-    - winston > async > lodash:
-        patched: '2020-05-05T00:19:05.987Z'
+ignore: {}
+patch: {}

src/includes/logger.js

@@ -25,11 +25,9 @@ switch (process.env.ENVIRONMENT) {
   case 'prod':
   case 'production':
     level = 'info';
-    silent = false;
     break;
   default:
     level = 'debug';
-    silent = false;
     break;
 }
 

src/index.js

@@ -3,7 +3,7 @@
 'use strict';
 
 const appName = 'csl-incoming-webhook';
-const appVersion = '2.2.1';
+const appVersion = '2.3.0';
 
 const COSMOS_API_VERSION = 'v1';
 
@@ -43,7 +43,7 @@ const TOPIC_REALTIME =
 
 const TOPIC_REALTIME_ALL =
   process.env.TOPIC_REALTIME_ALL ||
-  `projects/${process.env.PROJECT}/topics/csl-realtime-all-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`;
+  `projects/${process.env.PROJECT}/topics/csl-realtime-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`;
 
 const {SecretManagerServiceClient} = require('@google-cloud/secret-manager');
 
@@ -181,9 +181,10 @@ app.post('/', async (req, res, next) => {
   )}`;
 
   // Verify that the payload is signed with the correct key
+
   if (localSign !== req.header('x-controlshift-webhook-signature')) {
     logger.error('Invalid signature!');
-    return res.status(400).send({status: 'Hash Error'});
+    return res.status(400).send({error: 'Hash Error'});
   }
 
   next();
@@ -263,6 +264,7 @@ app.post('/', async function (req, res, _next) {
 
 // Root requests to this endpoint
 app.get('/', function (_req, res, _next) {
+  console.log('Triggered');
   res.end('OK');
 });
 

src/package.json

 {
-  "name": "csl-incoming-webhook",
-  "version": "2.2.1",
-  "description": "ControlShiftLabs Event Webhook",
+  "name": "csl-webhook",
+  "version": "2.3.0",
+  "description": "ControlShiftLabs Webhook Event Handler",
   "main": "index.js",
   "engines": {
     "node": ">=10.0.0",
@@ -9,31 +9,31 @@
   },
   "dependencies": {
     "@google-cloud/functions-framework": "^1.7.1",
-    "@google-cloud/pubsub": "^2.5.0",
-    "@google-cloud/secret-manager": "^3.1.0",
-    "@google-cloud/trace-agent": "^5.1.0",
-    "@sentry/node": "^5.23.0",
-    "snyk": "^1.393.0",
+    "@google-cloud/pubsub": "^2.6.0",
+    "@google-cloud/secret-manager": "^3.2.0",
+    "@google-cloud/trace-agent": "^5.1.1",
+    "@sentry/node": "^5.27.0",
+    "snyk": "^1.419.1",
     "winston": "^3.3.3"
   },
   "devDependencies": {
     "chai": "^4.2.0",
-    "eslint": "^7.9.0",
+    "eslint": "^7.11.0",
     "eslint-config-google": "^0.14.0",
-    "eslint-config-prettier": "^6.11.0",
-    "jest": "^26.4.2",
+    "eslint-config-prettier": "^6.14.0",
+    "jest": "^26.6.0",
     "newman": "^5.2.0",
-    "nodemon": "^2.0.4",
-    "prettier": "^2.1.1",
-    "supertest": "^4.0.2"
+    "nodemon": "^2.0.6",
+    "prettier": "^2.1.2",
+    "supertest": "^5.0.0"
   },
   "scripts": {
     "lint": "eslint '**/*.js'",
     "prettier": "prettier --write '**/*.{js,json,md}' '../*.md'",
-    "snyk": "snyk",
+    "snyk": "snyk test",
     "start": "nodemon -e js --exec 'npm run prettier && npm run lint && npx @google-cloud/functions-framework --target=main'",
     "debug": "nodemon -e js --exec 'npm run prettier && npm run lint && npx --node-arg=--inspect @google-cloud/functions-framework --target=main'",
-    "test": "snyk test && jest",
+    "test": "jest",
     "testQuiet": "jest --silent",
     "coverage": "jest --silent --coverage",
     "coverageWatch": "nodemon --watch '**/*' -e js --exec 'jest --silent --coverage'",
@@ -47,7 +47,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git@gitlab.greenpeace.org:global-data/csl-civis-webhook.git"
+    "url": "git@gitlab.greenpeace.org:cosmos/csl-incoming-webhook.git"
   },
   "author": "Raymond Walker",
   "license": "MIT",

src/tests/index.test.js

@@ -33,6 +33,10 @@ describe('POST Endpoints', () => {
       .post('/')
       .set('Content-Type', 'application/json')
       .set('Accept', 'application/json')
+      .set(
+        'x-controlshift-webhook-signature',
+        'sha256:907f3b3acd3f15c16b15c6a7329f18d6bbbdee6f2215761a8fe1cfc96ca95123'
+      )
       .send({
         type: 'data.full_table_exported',
       });
@@ -46,6 +50,10 @@ describe('POST Endpoints', () => {
       .post('/')
       .set('Content-Type', 'application/json')
       .set('Accept', 'application/json')
+      .set(
+        'x-controlshift-webhook-signature',
+        'sha256:0fdc7de63a7fce465b84bbb4afc6fe74e8b9c4ee9e785f42e2bf0f2847d6bb07'
+      )
       .send({
         type: 'data.incremental_table_exported',
       });
@@ -59,6 +67,10 @@ describe('POST Endpoints', () => {
       .post('/')
       .set('Content-Type', 'application/json')
       .set('Accept', 'application/json')
+      .set(
+        'x-controlshift-webhook-signature',
+        'sha256:3f5e38170ddeb5d7cba6879521a54365230a3bcf694f729d2bd8ccfbb7c7717f'
+      )
       .send({
         type: 'nope.nopenopenope',
       });
@@ -72,11 +84,26 @@ describe('POST Endpoints', () => {
       .post('/')
       .set('Content-Type', 'application/json')
       .set('Accept', 'application/json')
+      .set(
+        'x-controlshift-webhook-signature',
+        'sha256:2cd2b174c5df3c79fdd3fd34f8b6702451c6b7b53b4bcea606660607451951eb'
+      )
       .send({});
 
     expect(res.statusCode).toEqual(400);
     expect(res.body).toHaveProperty('error');
   });
+
+  it('should error out with "Invalid signature"', async () => {
+    const res = await request(app)
+      .post('/')
+      .set('Content-Type', 'application/json')
+      .set('Accept', 'application/json')
+      .send({});
+
+    expect(res.statusCode).toEqual(400);
+    expect(res.body).toHaveProperty('error', 'Hash Error');
+  });
 });
 
 describe('GET Endpoints', () => {

terraform/README.md

@@ -11,10 +11,14 @@ Before running terraform make sure that:
 - The following topics exist:
 
 ```
+# dev
 csl-realtime-all-cosmos-dev-sink
 csl-realtime-all-cosmos-dev-source
 csl-realtime-all-cosmos-test-sink
 csl-realtime-all-cosmos-test-source
+
+# staging
+csl-realtime-all-cosmos-stage-source
 ```
 
 ## TODO

terraform/deployments/prod/app/main.tf → terraform/deployments/stage/app/main.tf

@@ -2,31 +2,47 @@ terraform {
   required_version = ">= 0.12.0"
 
   backend "gcs" {
-    bucket = "global-data-terraform-state"
+    bucket = "cosmos-staging-terraform-state"
     # Structure:
     # state/<application>/<entity>/<component>/<environment>
-    prefix = "state/csl/cosmos/webhook/prod"
+    prefix = "state/csl/cosmos/webhook/staging"
   }
 }
 
 provider "google" {
-  project = "global-data-resources" 
-  region  = "EU"
-  version = "~> 3.38"
+  alias = "initial"
+}
+
+data "google_client_config" "config-default" {
+  provider = google.initial
+}
+
+data "google_service_account_access_token" "default" {
+  provider               = google.initial
+  target_service_account = "terraform@cosmos-staging-291101.iam.gserviceaccount.com"
+  scopes                 = ["cloud-platform"]
+  lifetime               = "300s"
+}
+
+provider "google" {
+  project      = "cosmos-staging-291101"
+  version      = "~> 3.38"
+  region       = "EU"
+  access_token = data.google_service_account_access_token.default.access_token
 }
 
 module "artifacts" {
-  source = "../../../modules/artifacts"
-  project = "global-data-resources"
+  source  = "../../../modules/artifacts"
+  project = "cosmos-staging-291101"
 }
 
 module "deployment" {
   source  = "../../../modules/app"
-  project = "global-data-resources"
+  project = "cosmos-staging-291101"
 
   source_archive_bucket = module.artifacts.source_bucket
   source_archive_object = module.artifacts.source_object
 
   entity      = "cosmos"
-  environment = "prod"
+  environment = "stage"
 }

terraform/modules/app/main.tf

@@ -27,7 +27,7 @@ resource "google_cloudfunctions_function" "function" {
 
   environment_variables = {
     CSL_HOSTNAME       = var.csl_hostname
-    TOPIC_REALTIME_ALL = var.realtime_all_topic != "" ? var.realtime_all_topic : "csl-realtime-all-cosmos-${var.environment}-source"
+    TOPIC_REALTIME_ALL = var.realtime_all_topic != "" ? var.realtime_all_topic : "csl-realtime-cosmos-${var.environment}-source"
     ENTITY             = var.entity
     ENVIRONMENT        = var.environment
     PROJECT            = var.project
@@ -36,6 +36,8 @@ resource "google_cloudfunctions_function" "function" {
 
 resource "google_service_account" "function" {
   account_id = "${local.app_name}-${var.entity}-${var.environment}"
+  display_name = "CSL webhook handler function"
+  project    = var.project
 }
 
 # IAM entry to allow all users to invoke the function (public HTTP endpoint)

terraform/modules/app/pubsub.tf

@@ -61,7 +61,7 @@ resource "google_pubsub_topic_iam_member" "realtime_read_publisher" {
 # Add publisher permissions to the global csl realtime topic
 resource "google_pubsub_topic_iam_member" "realtime_read_all_publisher" {
   project = var.project
-  topic   = var.realtime_all_topic != "" ? var.realtime_all_topic : "csl-realtime-all-cosmos-${var.environment}-source"
+  topic   = var.realtime_all_topic != "" ? var.realtime_all_topic : "csl-realtime-cosmos-${var.environment}-source"
   role    = "roles/pubsub.publisher"
   member  = "serviceAccount:${google_service_account.function.email}"
 }

terraform/modules/app/variables.tf

@@ -30,7 +30,7 @@ variable "realtime_all_topic" {
 # DEFAULTS
 
 variable "environment" {
-  default     = "prod"
+  default     = "stage"
   description = "Application run environment"
 }