Verified Commit 0c8dbe5b authored by Ray Walker's avatar Ray Walker
Browse files

init

parent 24755735
Pipeline #4455 failed with stages
in 3 minutes and 20 seconds
.sonar
.scannerwork
.secret
account.json
.terraform
*.tfstate
*.tfstate.lock.info
*.tfstate.backup
tf.plan
build/
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
firebase-debug.log*
# Firebase cache
.firebase/
# Firebase config
# Uncomment this if you'd like others to create their own Firebase project.
# For a team working on the same Firebase project(s), it is recommended to leave
# it commented so all members can deploy to the same project(s) in .firebaserc.
# .firebaserc
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
---
variables:
PRODUCTION_PROJECT: global-data-resources
TESTING_PROJECT: greenpeace-testing
REGION: europe-west1
TERRAFORM_VERSION: 0.12.12
# SAST_DISABLE_DIND: "true"
cache:
paths:
- $CI_PROJECT_DIR/src/node_modules/
stages:
- build
- test
- analysis
- deploy
# include:
# - template: Dependency-Scanning.gitlab-ci.yml
# - template: SAST.gitlab-ci.yml
.make_dependencies: &make_dependencies
image:
name: google/cloud-sdk:alpine
before_script:
- apk add make nodejs npm
- cd /usr/local/bin
- wget -qO- https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip | unzip -
- chmod 755 terraform
- cd $CI_PROJECT_DIR
- gcloud auth activate-service-account --key-file $SERVICE_ACCOUNT
- export GOOGLE_APPLICATION_CREDENTIALS=$SERVICE_ACCOUNT
- gcloud config set project $PROJECT
- gcloud config set functions/region $REGION
build:
<<: *make_dependencies
stage: build
variables:
PROJECT: $TESTING_PROJECT
script:
- time make src/node_modules
- make lint
test-jest:
<<: *make_dependencies
stage: test
variables:
PROJECT: $TESTING_PROJECT
script:
- make test
only:
- merge_requests
- branches
test-snyk:
<<: *make_dependencies
stage: test
variables:
PROJECT: $TESTING_PROJECT
script:
- make snyk
only:
- merge_requests
- branches
test-coverage:
<<: *make_dependencies
stage: test
variables:
PROJECT: $TESTING_PROJECT
artifacts:
name: "$CI_JOB_NAME:coverage"
expire_in: 1 weeks
paths:
- src/coverage/
script:
- make coverage
only:
- merge_requests
- branches
test-fossa:
<<: *make_dependencies
stage: test
variables:
PROJECT: $TESTING_PROJECT
script:
- make fossa
only:
- merge_requests
- branches
check-sonarqube:
stage: analysis
variables:
SONAR_HOST_URL: "https://sonarqube.p4.greenpeace.org"
image:
name: sonarsource/sonar-scanner-cli:latest
entrypoint: [""]
before_script:
- git fetch --unshallow || true
script:
- sonar-scanner
only:
- merge_requests
- branches
.deploy:
<<: *make_dependencies
stage: deploy
script: make
deploy-develop:
extends: .deploy
variables:
PROJECT: $TESTING_PROJECT
NODE_ENV: test
environment: Testing
only:
- develop
deploy-production:
variables:
PROJECT: $PRODUCTION_PROJECT
NODE_ENV: prod
TF_VAR_SERVICE_ACCOUNT: $SERVICE_ACCOUNT
RELEASE_VERSION: $CI_COMMIT_TAG
PRODUCTION_SOURCE_BUCKET: global-data-csl-pipeline-source
environment: Production
only:
- /^v[0-9]+(?:.[0-9]+)+$/
script:
- make release
ignore: |
/node_modules/
.terraform
Copyright (c) 2019 Greenpeace International
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
.EXPORT_ALL_VARIABLES:
# =============================================================================
# USER CONFIGURABLE VARIABLES
### REQUIRED, NO DEFAULT ENVIRONMENT VARIABLES
# SENTRY_DSN
# CLIENT_ID
# CLIENT_SECRET
# Project to deploy this webhook
PROJECT ?= greenpeace-testing
# Business unit deploying this webhook
ENTITY ?= dev
# Environment in which this webhook will run
NODE_ENV ?= dev
# Unique identifier for this application
APP_NAME ?= csl-test-nightly
# Name of BigQuery dataset to store results
DATASET_ID ?= csl-test-development
# PubSub topic where parsed CSL events are published
INPUT_TOPIC ?= projects/$(PROJECT)/topics/csl-nightly-read-test-output
# =============================================================================
# SERVICE_ACCOUNT is set in GitLab as a file CI environment variable
# So, for local development, we use the filename of a local json file
# containing service account credentials
SERVICE_ACCOUNT ?= account.json
TF_VAR_SERVICE_ACCOUNT ?= $(SERVICE_ACCOUNT)
RELEASE_SOURCE_BUCKET := global-data-csl-pipeline-source
CI_COMMIT_TAG ?= testing
# =============================================================================
TF_VAR_SENTRY_DSN := $(SENTRY_DSN)
# =============================================================================
YAMLLINT := $(shell command -v yamllint 2> /dev/null)
# =============================================================================
all: app list
# =============================================================================
app: lint terraform
list:
gcloud functions list --project=$(PROJECT) --filter=labels.app=global-data
describe:
gcloud functions describe $(APP_NAME) --project=$(PROJECT) --region=europe-west1
# =============================================================================
lint: lint-yaml src/node_modules lint-js
lint-yaml:
# ifndef YAMLLINT
# $(warning "yamllint is not installed: https://github.com/adrienverge/yamllint")
# else
# find . -type f -name '*.yml' | xargs $(YAMLLINT)
# endif
lint-js:
cd src && npm run lint
# =============================================================================
# DEVELOPMENT TARGETS
src/node_modules:
cd src && npm install
dev: src/node_modules
ifndef SENTRY_DSN
$(error SENTRY_DSN is not set)
endif
cd src && npm start
test: src/node_modules
ifndef SENTRY_DSN
$(error SENTRY_DSN is not set)
endif
cd src && npm test
testQuiet: src/node_modules
ifndef SENTRY_DSN
$(error SENTRY_DSN is not set)
endif
cd src && npm run testQuiet
snyk: src/node_modules
cd src && npm run snyk test
testWatch: src/node_modules
ifndef SENTRY_DSN
$(error SENTRY_DSN is not set)
endif
cd src && npm run testWatch
coverage: src/node_modules
ifndef SENTRY_DSN
$(error SENTRY_DSN is not set)
endif
cd src && npm run coverage
fossa: src/node_modules
ifdef FOSSA_API_KEY
curl -SH 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install.sh | bash
fossa init
fossa analyze
else
$(warning FOSSA_API_KEY not set)
$(warning Can\'t analyze license status)
endif
# =============================================================================
prettier:
@$(MAKE) -j prettier-terraform prettier-npm
prettier-terraform:
terraform fmt
prettier-npm:
cd src && npm run prettier
# =============================================================================
.PHONY: terraform
terraform: terraform-app
#
# terraform-fixtures:
# $(MAKE) -C terraform/deployments/test/fixtures
terraform-app:
$(MAKE) -C terraform/deployments/test/app
output: terraform-output
terraform-output:
@echo "Fixtures:"
@echo "---"
@$(MAKE) -sC terraform/deployments/test/fixtures output
@echo
@echo "App:"
@echo "---"
@$(MAKE) -sC terraform/deployments/test/app output
# =============================================================================
clean:
@$(MAKE) clean-terraform clean-local-files
clean-terraform: clean-terraform-app
clean-terraform-app:
$(MAKE) -C terraform/deployments/test/app destroy
clean-local-files:
rm -fr src/node_modules src/coverage terraform/app/build .sonar .scannerwork tf.plan
# =============================================================================
release:
gsutil cp terraform/deployments/test/app/build/csl-nightly-bq-load.zip gs://$(RELEASE_SOURCE_BUCKET)/csl-nightly-bq-load-$(CI_COMMIT_TAG).zip
gsutil cp terraform/deployments/test/app/build/csl-nightly-bq-load.zip gs://$(RELEASE_SOURCE_BUCKET)/csl-nightly-bq-load-latest.zip
# CSL nightly full table BQ transform + load
1. Listens to GCP PubSub `NIGHTLY_WRITE` topics
1. Loads the CSV data from the payload Google Storage Bucket into BigQuery Dataset
sonar-project.properties
# must be unique in a given SonarQube instance
sonar.projectKey=global-data-csl-nightly-bq-load
sonar.language=js
sonar.sources=src
sonar.exclusions=src/node_modules/*,src/tests/*
sonar.javascript.lcov.reportPaths=src/coverage/lcov.info
module.exports = {
env: {
es6: true,
node: true,
},
extends: ['google', 'prettier'],
globals: {
Atomics: 'readonly',
SharedArrayBuffer: 'readonly',
},
parserOptions: {
ecmaVersion: 2018,
sourceType: 'module',
},
rules: {},
};
node_modules/
{
"trailingComma": "es5",
"bracketSpacing": false,
"singleQuote": true,
"printWidth": 80
}
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.13.5
ignore: {}
patch: {}
const winston = require('winston');
let level;
let silent;
switch (process.env.NODE_ENV) {
case 'production':
level = 'warning';
silent = false;
break;
case 'test':
level = 'emerg';
silent = true;
break;
default:
level = 'debug';
silent = false;
break;
}
const options = {
console: {
level,
silent,
handleExceptions: true,
format: winston.format.combine(
winston.format.colorize(),
winston.format.splat(),
winston.format.printf(
info => `${new Date().toISOString()} ${info.level}: ${info.message}`
)
),
},
};
const logger = winston.createLogger({
levels: winston.config.syslog.levels,
transports: [new winston.transports.Console(options.console)],
exitOnError: false,
});
module.exports = logger;
// Om Namah Shivaya
'use strict';
// Start google tracing
if (process.env.NODE_ENV === 'production') {
require('@google-cloud/trace-agent').start();
}
// Initialise error handling service
global.Sentry = require('@sentry/node');
Sentry.init({dsn: process.env.SENTRY_DSN});
if (typeof process.env.ENTITY === 'undefined') {
throw new Error('ENTITY is not defined');
}
if (typeof process.env.PROJECT === 'undefined') {
throw new Error('PROJECT is not defined');
}
if (typeof process.env.SENTRY_DSN === 'undefined') {
throw new Error('SENTRY_DSN is not defined');
}
if (typeof process.env.DATASET_ID === 'undefined') {
throw new Error('DATASET_ID is not defined');
}
const dataset = process.env.DATASET_ID;
const entity = process.env.ENTITY;
const environment = process.env.NODE_ENV;
const project = process.env.PROJECT;
const {BigQuery} = require('@google-cloud/bigquery');
const {Storage} = require('@google-cloud/storage');
const logger = require('./includes/logger.js');
const bigquery = new BigQuery({
projectId: project,
});
const storage = new Storage({
projectId: project,
});
const metadata = {
sourceFormat: 'CSV',
skipLeadingRows: 1,
autodetect: true,
};
/**
* sentryHandler - wrap function execution with Sentry error handling
*
* @param {type} handler description
* @return {type} description
*/
function sentryHandler(handler) {
return async event => {
try {
return await handler(event);
} catch (error) {
logger.error(error.toString());
Sentry.captureException(error);
await Sentry.flush(2000);
return error;
}
};
}
/**
* Read GCS object, store in BigQuery table
*
* @param {object} pubSubEvent The event pubSubEvent.
* @param {object} context The event metadata.
* @return {bool}
*/
exports.main = sentryHandler(async (pubSubEvent, context) => {
let payload;
let job;
// Parse PubSub event data
if ('dev' === entity && 'dev' === environment) {
// Workaround for local development, as GCP functions framework doesn't
// properly unmarshall events
// See: https://github.com/GoogleCloudPlatform/functions-framework-nodejs/issues/41
const message = pubSubEvent
? JSON.parse(Buffer.from(pubSubEvent, 'base64')).message
: {};
// Parse base64 encoded message data
payload = message.data
? JSON.parse(Buffer.from(message.data, 'base64'))
: {};
} else {
// Parse base64 encoded message data
payload = pubSubEvent
? JSON.parse(Buffer.from(pubSubEvent.data, 'base64'))
: {};
}
const bucket = payload.object.bucket;
const filename = payload.object.filename;
const table = payload.object.table;
// Loads data from a Google Cloud Storage file into the table
await bigquery
.dataset(dataset)
.table(table)
.load(storage.bucket(bucket).file(filename), metadata)
.then(results => {
job = results[0];
logger.info(`LOAD: Job ${job.id} started.`);
// Wait for the job to finish