Skip to content

GitLab

Commits (14)
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 323 additions and 376 deletions
.gitlab-ci.yml
View file @ 387e4f98
......@@ -4,7 +4,6 @@ variables:
DEVELOPMENT_PROJECT: cosmos-dev-286703
STAGING_PROJECT: cosmos-staging-291101
REGION: europe-west1
SERVICE_ACCOUNT: $SERVICE_ACCOUNT_DEV
cache:
key: ${CI_PROJECT_NAME}
......@@ -12,9 +11,6 @@ cache:
- .npm/
- $CI_PROJECT_DIR/src/node_modules/
include:
- template: Code-Quality.gitlab-ci.yml
stages:
- build
- test
......@@ -33,10 +29,12 @@ stages:
image:
name: greenpeaceinternational/gitlab-data-builder:latest
before_script:
- gcloud auth activate-service-account --key-file $SERVICE_ACCOUNT
- export GOOGLE_APPLICATION_CREDENTIALS=$SERVICE_ACCOUNT
- gcloud auth activate-service-account --key-file $SERVICE_ACCOUNT_DEV
- export GOOGLE_APPLICATION_CREDENTIALS=$SERVICE_ACCOUNT_DEV
- gcloud config set project $PROJECT
- gcloud config set functions/region $REGION
- gcloud config set auth/impersonate_service_account $TF_EMAIL_DEV
- gcloud config set core/verbosity "error"
- npm config set cache $CI_PROJECT_DIR/.npm --global
build:
......@@ -114,21 +112,6 @@ test-fossa:
# script:
# - /usr/share/dependency-check/bin/dependency-check.sh --project "csl-webhook" --out . --scan . --enableExperimental --failOnCVSS 7
code_quality:
rules:
- if: "$CODE_QUALITY_DISABLED"
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "develop"'
tags:
- Global Data
stage: analysis
needs: []
artifacts:
expire_in: 6 months
paths:
- gl-code-quality-report.json
sonarqube:
rules:
- if: "$CODE_QUALITY_DISABLED"
......
Makefile
View file @ 387e4f98
......@@ -19,6 +19,9 @@ ENVIRONMENT ?= dev
# Hostname for testing
CSL_HOSTNAME ?= gpfood.controlshiftlabs.com
# Terraform SA
TF_EMAIL := terraform@$(PROJECT).iam.gserviceaccount.com
# =============================================================================
RELEASE_SOURCE_BUCKET := global-data-csl-pipeline-source
......@@ -57,6 +60,26 @@ lint-js: src/node_modules
# =============================================================================
# DEVELOPMENT TARGETS
set:
gcloud config set auth/impersonate_service_account $(TF_EMAIL)
unset:
gcloud config unset auth/impersonate_service_account
secrets-export:
$(eval SENTRY_DSN=$(shell gcloud secrets versions access latest --secret="sentry_dsn_csl_incoming_webhook"))
$(eval CSL_GPFOOD=$(shell gcloud secrets versions access latest --secret="csl_gpfood-controlshiftlabs-com_hmac_dev"))
secrets: set secrets-export unset
pubsub-export:
$(eval FULL_TABLE_EXPORTED=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-nightly-read --message "test"))
$(eval NOPE_REALTIME=$(shell gcloud pubsub topics publish csl-webhook-cosmos-dev-realtime-source --message "test"))
$(eval NOPE_REALTIME_ALL=$(shell gcloud pubsub topics publish csl-realtime-cosmos-dev-source --message "test"))
pubsub: set pubsub-export unset
secrets-pubsub: set secrets-export pubsub-export unset
src/node_modules:
ifdef CI
......@@ -65,18 +88,18 @@ else
cd src && npm install
endif
dev: src/node_modules
dev: secrets src/node_modules
@echo "Entity: $(ENTITY)"
@echo "Env: $(ENVIRONMENT)"
cd src && npm start
debug: src/node_modules
debug: secrets src/node_modules
cd src && npm run debug
test: src/node_modules
test: secrets-pubsub src/node_modules
cd src && npm test
testQuiet: src/node_modules
testQuiet: secrets-pubsub src/node_modules
cd src && npm run testQuiet
snyk: snyk-auth src/node_modules
......@@ -91,10 +114,10 @@ snykWizard: src/node_modules
testWatch: src/node_modules
cd src && npm run testWatch
coverage: src/node_modules
coverage: secrets-pubsub src/node_modules
cd src && npm run coverage
coverageWatch: src/node_modules
coverageWatch: secrets-pubsub src/node_modules
cd src && npm run coverageWatch
fossa: src/node_modules
......@@ -149,6 +172,7 @@ clean-local-files:
# =============================================================================
release: terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip
gcloud config set auth/impersonate_service_account $(TF_EMAIL)
gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-$(CI_COMMIT_REF_NAME).zip
gsutil cp terraform/deployments/$(ENVIRONMENT)/app/build/$(APP_NAME).zip gs://$(RELEASE_SOURCE_BUCKET)/$(APP_NAME)-latest.zip
......
README.md
View file @ 387e4f98
......@@ -20,6 +20,4 @@ Requirements:
Configure max watches to prevent node.js crashes : `echo fs.inotify.max_user_watches=582222 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
Export your google cloud service account with `export GOOGLE_APPLICATION_CREDENTIALS="PATH"`
running `make dev` will install all npm dependencies and run the node.js express app on http://localhost:8080/
src/.snyk
View file @ 387e4f98
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.14.1
ignore: {}
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
SNYK-JS-Y18N-1021887:
- '@google-cloud/pubsub > google-gax > @grpc/grpc-js > @grpc/proto-loader > yargs > y18n':
reason: None given
expires: '2021-01-06T04:04:27.486Z'
- '@google-cloud/secret-manager > google-gax > @grpc/grpc-js > @grpc/proto-loader > yargs > y18n':
reason: None given
expires: '2021-01-06T04:04:27.486Z'
- snyk > snyk-config > nconf > yargs > y18n:
reason: None given
expires: '2021-01-06T04:04:27.486Z'
patch: {}
src/index.js
View file @ 387e4f98
......@@ -3,7 +3,7 @@
'use strict';
const appName = 'csl-incoming-webhook';
const appVersion = '2.3.0';
const appVersion = '3.0.0';
const COSMOS_API_VERSION = 'v1';
......@@ -45,6 +45,10 @@ const TOPIC_REALTIME_ALL =
process.env.TOPIC_REALTIME_ALL ||
`projects/${process.env.PROJECT}/topics/csl-realtime-${process.env.ENTITY}-${process.env.ENVIRONMENT}-source`;
const IS_IMPERSONATED =
process.env.GOOGLE_OAUTH_ACCESS_TOKEN ||
process.env.GOOGLE_APPLICATION_CREDENTIALS;
const {SecretManagerServiceClient} = require('@google-cloud/secret-manager');
const crypto = require('crypto');
......@@ -69,9 +73,11 @@ async function getSecret(name) {
const Sentry = require('@sentry/node');
(async () => {
Sentry.init({
dsn: await getSecret(
`projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest`
),
dsn:
process.env.SENTRY_DSN ||
(await getSecret(
`projects/${process.env.PROJECT}/secrets/sentry_dsn_csl_incoming_webhook/versions/latest`
)),
release: `${appName}@${appVersion}`,
});
})();
......@@ -167,6 +173,7 @@ app.post('/', async (req, res, next) => {
// Fetch HMAC PSK from SecretManager, cache for re-use
sharedSecret =
sharedSecret ||
process.env.CSL_GPFOOD ||
(await getSecret(
`projects/${
process.env.PROJECT
......@@ -248,8 +255,19 @@ app.post('/', async function (req, res, _next) {
// NIGHTLY_READ
if (path === 'data' && action === 'full_table_exported') {
return await publishOne(TOPIC_NIGHTLY, req, res);
// return res.status(200).send({status: 'OK'});
if (IS_IMPERSONATED) {
if (process.env.FULL_TABLE_EXPORTED) {
logger.debug(`SUCCESS ${TOPIC_NIGHTLY}`);
return res.send({status: 'OK'});
} else {
return res
.status(400)
.send({error: `Publishing to ${TOPIC_NIGHTLY} failed`});
}
} else {
return await publishOne(TOPIC_NIGHTLY, req, res);
// return res.status(200).send({status: 'OK'});
}
}
// INCREMENTAL_READ
......@@ -259,7 +277,18 @@ app.post('/', async function (req, res, _next) {
}
// REALTIME_READ
await publishRealtime(TOPIC_REALTIME, req, res);
if (IS_IMPERSONATED) {
if (process.env.NOPE_REALTIME && process.env.NOPE_REALTIME_ALL) {
logger.debug(`SUCCESS: [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ]`);
res.send({status: 'OK'});
} else {
res.status(400).send({
error: `Publishing to [ "${TOPIC_REALTIME}", "${TOPIC_REALTIME_ALL}" ] failed`,
});
}
} else {
await publishRealtime(TOPIC_REALTIME, req, res);
}
});
// Root requests to this endpoint
......
src/package-lock.json
View file @ 387e4f98
This diff is collapsed.
src/package.json
View file @ 387e4f98
{
"name": "csl-webhook",
"version": "2.3.0",
"version": "3.0.0",
"description": "ControlShiftLabs Webhook Event Handler",
"main": "index.js",
"engines": {
......@@ -13,7 +13,7 @@
"@google-cloud/secret-manager": "^3.2.0",
"@google-cloud/trace-agent": "^5.1.1",
"@sentry/node": "^5.27.0",
"snyk": "^1.419.1",
"snyk": "^1.425.4",
"winston": "^3.3.3"
},
"devDependencies": {
......@@ -52,4 +52,4 @@
"author": "Raymond Walker",
"license": "MIT",
"snyk": true
}
}
\ No newline at end of file